import os, json, random, io, logging from helpers import aws_helper, tar_helper, csv_helper, service_helper from models.solver_run import SolverRun from models.solution import Solution from models.form import Form from services.base import Base class LoftService(Base): def process(self): self.solver_run = SolverRun.parse_obj(self.retreive_attributes_from_message()) self.solution = self.generate_solution() self.result = self.stream_to_s3_bucket() def retreive_attributes_from_message(self): logging.info('Retrieving attributes from message...') # get s3 object self.key = aws_helper.get_key_from_message(self.source) s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET']) # convert to tar self.tar = tar_helper.raw_to_tar(s3_object) # get attributes file and convert to dict attributes = json.loads(tar_helper.extract_file_from_tar(self.tar , 'solver_run_attributes.json').read()) # get items file and convert to dict items_csv = tar_helper.extract_file_from_tar(self.tar , 'items.csv') items_csv_reader = csv_helper.file_stream_reader(items_csv) # add items to attributes dict attributes['items'] = service_helper.items_csv_to_dict(items_csv_reader) logging.info('Processed Attributes...') return attributes def generate_solution(self): logging.info('Processing Solution...') # temporary data for mocks form_count = 10 # items will be generated from real solver process, this is for mock purposes # real solver will return N forms and process a cut score, this is for mock purposes return Solution( response_id=random.randint(100,5000), forms=[ Form( items=[item.id for item in random.sample(self.solver_run.items, self.solver_run.total_form_items)], cut_score=120 ) for x in range(form_count) ] ) def stream_to_s3_bucket(self): self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv' logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['MEASURE_PROCESSED_BUCKET']) # setup writer buffer and write processed forms to file buffer = io.StringIO() solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) # upload generated file to s3 and return result return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['MEASURE_PROCESSED_BUCKET'])