jshbrmn/irt-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logging all the things...

Browse Source
This commit is contained in:
Josh Burman 2021-11-02 04:07:15 +00:00
parent 5a2871b102
commit 1c50ad5642
2 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/main.py
View File

@ -1,4 +1,4 @@
import os, sys, json, time import os, sys, logging
from services.loft_service import LoftService from services.loft_service import LoftService
from helpers import aws_helper from helpers import aws_helper
@ -6,16 +6,20 @@ from helpers import aws_helper
from daemonize import Daemonize from daemonize import Daemonize
from sqs_listener import SqsListener from sqs_listener import SqsListener
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(levelname)s %(asctime)s - %(message)s")
class ServiceListener(SqsListener): class ServiceListener(SqsListener):
def handle_message(self, body, attributes, messages_attributes): def handle_message(self, body, attributes, messages_attributes):
# gather/manage/process data based on the particular needs # gather/manage/process data based on the particular needs
logging.info('Incoming message: %s', body)
service = LoftService(body) service = LoftService(body)
service.process() service.process()
# log the things
logging.info('Process complete for %s', service.file_name)
def main(): def main():
print("Starting Solver Service (v0.3.2)...") logging.info('Starting Solver Service (v0.3.2)...')
print("Initializing listener")
listener = ServiceListener( listener = ServiceListener(
'measure-development-solver-ingest', 'measure-development-solver-ingest',
region_name=os.environ['SOLVER_AWS_REGION'], region_name=os.environ['SOLVER_AWS_REGION'],

12
app/services/loft_service.py
View File

@ -1,7 +1,4 @@
import os import os, json, random, io, logging
import json
import random
import io
from helpers import aws_helper, tar_helper, csv_helper, service_helper from helpers import aws_helper, tar_helper, csv_helper, service_helper
@ -18,6 +15,7 @@ class LoftService(Base):
self.result = self.stream_to_s3_bucket() self.result = self.stream_to_s3_bucket()
def retreive_attributes_from_message(self): def retreive_attributes_from_message(self):
logging.info('Retrieving attributes from message...')
# get s3 object # get s3 object
self.key = aws_helper.get_key_from_message(self.source) self.key = aws_helper.get_key_from_message(self.source)
s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET']) s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET'])
@ -34,10 +32,12 @@ class LoftService(Base):
# add items to attributes dict # add items to attributes dict
attributes['items'] = service_helper.items_csv_to_dict(items_csv_reader) attributes['items'] = service_helper.items_csv_to_dict(items_csv_reader)
logging.info('Processed Attributes...')
return attributes return attributes
def generate_solution(self): def generate_solution(self):
logging.info('Processing Solution...')
# temporary data for mocks # temporary data for mocks
form_count = 10 form_count = 10
@ -54,9 +54,11 @@ class LoftService(Base):
) )
def stream_to_s3_bucket(self): def stream_to_s3_bucket(self):
self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv'
logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['MEASURE_PROCESSED_BUCKET'])
# setup writer buffer and write processed forms to file # setup writer buffer and write processed forms to file
buffer = io.StringIO() buffer = io.StringIO()
solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms)
# upload generated file to s3 and return result # upload generated file to s3 and return result
return aws_helper.file_stream_upload(solution_file, f'{service_helper.key_to_uuid(self.key)}.csv', os.environ['MEASURE_PROCESSED_BUCKET']) return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['MEASURE_PROCESSED_BUCKET'])