diff --git a/app/main.py b/app/main.py index 9d9f7d1..7636ecc 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,4 @@ -import os, sys, json, time +import os, sys, logging from services.loft_service import LoftService from helpers import aws_helper @@ -6,16 +6,20 @@ from helpers import aws_helper from daemonize import Daemonize from sqs_listener import SqsListener +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(levelname)s %(asctime)s - %(message)s") + class ServiceListener(SqsListener): def handle_message(self, body, attributes, messages_attributes): # gather/manage/process data based on the particular needs + logging.info('Incoming message: %s', body) + service = LoftService(body) service.process() - # log the things + + logging.info('Process complete for %s', service.file_name) def main(): - print("Starting Solver Service (v0.3.2)...") - print("Initializing listener") + logging.info('Starting Solver Service (v0.3.2)...') listener = ServiceListener( 'measure-development-solver-ingest', region_name=os.environ['SOLVER_AWS_REGION'], diff --git a/app/services/loft_service.py b/app/services/loft_service.py index d571a61..aae2ad5 100644 --- a/app/services/loft_service.py +++ b/app/services/loft_service.py @@ -1,7 +1,4 @@ -import os -import json -import random -import io +import os, json, random, io, logging from helpers import aws_helper, tar_helper, csv_helper, service_helper @@ -18,6 +15,7 @@ class LoftService(Base): self.result = self.stream_to_s3_bucket() def retreive_attributes_from_message(self): + logging.info('Retrieving attributes from message...') # get s3 object self.key = aws_helper.get_key_from_message(self.source) s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET']) @@ -34,10 +32,12 @@ class LoftService(Base): # add items to attributes dict attributes['items'] = service_helper.items_csv_to_dict(items_csv_reader) + logging.info('Processed Attributes...') return attributes def generate_solution(self): + logging.info('Processing Solution...') # temporary data for mocks form_count = 10 @@ -54,9 +54,11 @@ class LoftService(Base): ) def stream_to_s3_bucket(self): + self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv' + logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['MEASURE_PROCESSED_BUCKET']) # setup writer buffer and write processed forms to file buffer = io.StringIO() solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) # upload generated file to s3 and return result - return aws_helper.file_stream_upload(solution_file, f'{service_helper.key_to_uuid(self.key)}.csv', os.environ['MEASURE_PROCESSED_BUCKET']) + return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['MEASURE_PROCESSED_BUCKET'])