From d13574ac081d60f4b478fcd1ecf0b6497f2f83c6 Mon Sep 17 00:00:00 2001 From: Josh Burman Date: Fri, 29 Oct 2021 00:10:30 +0000 Subject: [PATCH] renamed to loft service to be more clear, added solution file generation naming via regex, few qol things --- app/helpers/service_helper.py | 6 ++++++ app/main.py | 9 ++++++--- app/models/item.py | 1 + app/services/{loft.py => loft_service.py} | 15 +++++++-------- 4 files changed, 20 insertions(+), 11 deletions(-) rename app/services/{loft.py => loft_service.py} (75%) diff --git a/app/helpers/service_helper.py b/app/helpers/service_helper.py index 41c3e88..07b6900 100644 --- a/app/helpers/service_helper.py +++ b/app/helpers/service_helper.py @@ -1,5 +1,6 @@ import csv import io +import re def items_csv_to_dict(items_csv_reader): items = [] @@ -15,6 +16,8 @@ def items_csv_to_dict(items_csv_reader): for key, col in enumerate(headers): if key == 0: item[col] = row[key] + elif col == 'b_param': + item[col] = row[key] elif key > 1: item['attributes'].append({ 'id': col, @@ -43,3 +46,6 @@ def solution_to_file(buffer, total_form_items, forms): buff2 = io.BytesIO(buffer.getvalue().encode()) return buff2 + +def key_to_uuid(key): + return re.split("_", key)[0] diff --git a/app/main.py b/app/main.py index d64b551..1308bda 100644 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,14 @@ import os +import sys -from services.loft import Loft +from services.loft_service import LoftService from helpers import aws_helper +from sqs_listener import SqsListener +from sqs_listener.daemon import Daemon print("Starting Solver Service (v0.3.1)...") -# listen to the solver queue +# # listen to the solver queue while True: msg = aws_helper.receive_message(os.environ['SOLVER_SQS_INGEST_QUEUE']) @@ -14,7 +17,7 @@ while True: # for now the solver service only supports Loft types # this is here to allow us to create an extensible way to # gather/manage/process data based on the particular needs - service = Loft(message) + service = LoftService(message) service.process() # delete message once process is complete diff --git a/app/models/item.py b/app/models/item.py index 975593e..8ae3b70 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -6,3 +6,4 @@ from models.attribute import Attribute class Item(BaseModel): id: int attributes: List[Attribute] + b_param: int diff --git a/app/services/loft.py b/app/services/loft_service.py similarity index 75% rename from app/services/loft.py rename to app/services/loft_service.py index 4826fe7..d571a61 100644 --- a/app/services/loft.py +++ b/app/services/loft_service.py @@ -2,7 +2,6 @@ import os import json import random import io -import time from helpers import aws_helper, tar_helper, csv_helper, service_helper @@ -12,7 +11,7 @@ from models.form import Form from services.base import Base -class Loft(Base): +class LoftService(Base): def process(self): self.solver_run = SolverRun.parse_obj(self.retreive_attributes_from_message()) self.solution = self.generate_solution() @@ -20,17 +19,17 @@ class Loft(Base): def retreive_attributes_from_message(self): # get s3 object - key = aws_helper.get_key_from_message(self.source) - s3_object = aws_helper.get_object(key, os.environ['SOLVER_INGEST_BUCKET']) + self.key = aws_helper.get_key_from_message(self.source) + s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET']) # convert to tar - tar = tar_helper.raw_to_tar(s3_object) + self.tar = tar_helper.raw_to_tar(s3_object) # get attributes file and convert to dict - attributes = json.loads(tar_helper.extract_file_from_tar(tar, 'solver_run_attributes.json').read()) + attributes = json.loads(tar_helper.extract_file_from_tar(self.tar , 'solver_run_attributes.json').read()) # get items file and convert to dict - items_csv = tar_helper.extract_file_from_tar(tar, 'items.csv') + items_csv = tar_helper.extract_file_from_tar(self.tar , 'items.csv') items_csv_reader = csv_helper.file_stream_reader(items_csv) # add items to attributes dict @@ -60,4 +59,4 @@ class Loft(Base): solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) # upload generated file to s3 and return result - return aws_helper.file_stream_upload(solution_file, f'test_solution_{time.time()}.csv', os.environ['MEASURE_PROCESSED_BUCKET']) + return aws_helper.file_stream_upload(solution_file, f'{service_helper.key_to_uuid(self.key)}.csv', os.environ['MEASURE_PROCESSED_BUCKET'])