import os import json import random import io import time from helpers import aws_helper, tar_helper, csv_helper, service_helper from models.solver_run import SolverRun from models.solution import Solution from models.form import Form from services.base import Base class Loft(Base): def process(self): self.solver_run = SolverRun.parse_obj(self.retreive_attributes_from_message()) self.solution = self.generate_solution() self.result = self.stream_to_s3_bucket() def retreive_attributes_from_message(self): # get s3 object key = aws_helper.get_key_from_message(self.source) s3_object = aws_helper.get_object(key, os.environ['SOLVER_INGEST_BUCKET']) # convert to tar tar = tar_helper.raw_to_tar(s3_object) # get attributes file and convert to dict attributes = json.loads(tar_helper.extract_file_from_tar(tar, 'solver_run_attributes.json').read()) # get items file and convert to dict items_csv = tar_helper.extract_file_from_tar(tar, 'items.csv') items_csv_reader = csv_helper.file_stream_reader(items_csv) # add items to attributes dict attributes['items'] = service_helper.items_csv_to_dict(items_csv_reader) return attributes def generate_solution(self): # temporary data for mocks form_count = 10 # items will be generated from real solver process, this is for mock purposes # real solver will return N forms and process a cut score, this is for mock purposes return Solution( response_id=random.randint(100,5000), forms=[ Form( items=[item.id for item in random.sample(self.solver_run.items, self.solver_run.total_form_items)], cut_score=120 ) for x in range(form_count) ] ) def stream_to_s3_bucket(self): # setup writer buffer and write processed forms to file buffer = io.StringIO() solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) # upload generated file to s3 and return result return aws_helper.file_stream_upload(solution_file, f'test_solution_{time.time()}.csv', os.environ['MEASURE_PROCESSED_BUCKET'])