diff --git a/app/services/base.py b/app/services/base.py new file mode 100644 index 0000000..77f72f7 --- /dev/null +++ b/app/services/base.py @@ -0,0 +1,4 @@ +class Base: + def __init__(self, source, ingest_type='message'): + self.ingest_type = ingest_type + self.source = source diff --git a/app/services/loft.py b/app/services/loft.py new file mode 100644 index 0000000..4826fe7 --- /dev/null +++ b/app/services/loft.py @@ -0,0 +1,63 @@ +import os +import json +import random +import io +import time + +from helpers import aws_helper, tar_helper, csv_helper, service_helper + +from models.solver_run import SolverRun +from models.solution import Solution +from models.form import Form + +from services.base import Base + +class Loft(Base): + def process(self): + self.solver_run = SolverRun.parse_obj(self.retreive_attributes_from_message()) + self.solution = self.generate_solution() + self.result = self.stream_to_s3_bucket() + + def retreive_attributes_from_message(self): + # get s3 object + key = aws_helper.get_key_from_message(self.source) + s3_object = aws_helper.get_object(key, os.environ['SOLVER_INGEST_BUCKET']) + + # convert to tar + tar = tar_helper.raw_to_tar(s3_object) + + # get attributes file and convert to dict + attributes = json.loads(tar_helper.extract_file_from_tar(tar, 'solver_run_attributes.json').read()) + + # get items file and convert to dict + items_csv = tar_helper.extract_file_from_tar(tar, 'items.csv') + items_csv_reader = csv_helper.file_stream_reader(items_csv) + + # add items to attributes dict + attributes['items'] = service_helper.items_csv_to_dict(items_csv_reader) + + return attributes + + def generate_solution(self): + # temporary data for mocks + form_count = 10 + + # items will be generated from real solver process, this is for mock purposes + # real solver will return N forms and process a cut score, this is for mock purposes + return Solution( + response_id=random.randint(100,5000), + forms=[ + Form( + items=[item.id for item in random.sample(self.solver_run.items, self.solver_run.total_form_items)], + cut_score=120 + ) for x in range(form_count) + ] + ) + + def stream_to_s3_bucket(self): + # setup writer buffer and write processed forms to file + buffer = io.StringIO() + solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) + + # upload generated file to s3 and return result + return aws_helper.file_stream_upload(solution_file, f'test_solution_{time.time()}.csv', os.environ['MEASURE_PROCESSED_BUCKET'])