jshbrmn/irt-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/QUANT-1196-solve-all-the-things

Browse Source
This commit is contained in:
Josh Burman
2021-11-03 20:15:19 +00:00
parent 2f86cbecfa d87aa17531
commit 6558875591
3 changed files with 15 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/helpers/aws_helper.py
View File

@ -3,16 +3,19 @@ import os
import json import json
session = boto3.Session( session = boto3.Session(
aws_access_key_id=os.environ['SOLVER_AWS_ACCESS_KEY_ID'], aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['SOLVER_AWS_SECRET_ACCESS_KEY'] aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY']
) )
s3 = session.resource('s3', region_name=os.environ['SOLVER_AWS_REGION']) s3 = session.resource('s3', region_name=os.environ['AWS_REGION'])
sqs = session.client('sqs', region_name=os.environ['SOLVER_AWS_REGION']) sqs = session.client('sqs', region_name=os.environ['AWS_REGION'])
def get_key_from_message(body): def get_key_from_message(body):
return body['Records'][0]['s3']['object']['key'] return body['Records'][0]['s3']['object']['key']
def get_bucket_from_message(body):
return body['Records'][0]['s3']['bucket']['name']
def get_object(key, bucket): def get_object(key, bucket):
return s3.Object( return s3.Object(
bucket_name=bucket, bucket_name=bucket,

10
app/main.py
View File

@ -19,13 +19,13 @@ class ServiceListener(SqsListener):
logging.info('Process complete for %s', service.file_name) logging.info('Process complete for %s', service.file_name)
def main(): def main():
logging.info('Starting Solver Service (v0.3.2)...') logging.info('Starting Solver Service (v0.4.3)...')
listener = ServiceListener( listener = ServiceListener(
'measure-development-solver-ingest', 'measure-development-solver-ingest',
region_name=os.environ['SOLVER_AWS_REGION'], region_name=os.environ['AWS_REGION'],
aws_access_key=os.environ['SOLVER_AWS_ACCESS_KEY_ID'], aws_access_key=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_key=os.environ['SOLVER_AWS_SECRET_ACCESS_KEY'], aws_secret_key=os.environ['AWS_SECRET_ACCESS_KEY'],
queue_url=os.environ['SOLVER_SQS_INGEST_QUEUE'] queue_url=os.environ['SQS_QUEUE']
) )
listener.listen() listener.listen()

6
app/services/loft_service.py
View File

@ -18,7 +18,7 @@ class LoftService(Base):
logging.info('Retrieving attributes from message...') logging.info('Retrieving attributes from message...')
# get s3 object # get s3 object
self.key = aws_helper.get_key_from_message(self.source) self.key = aws_helper.get_key_from_message(self.source)
s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET']) s3_object = aws_helper.get_object(self.key, aws_helper.get_bucket_from_message(self.source))
# convert to tar # convert to tar
self.tar = tar_helper.raw_to_tar(s3_object) self.tar = tar_helper.raw_to_tar(s3_object)
@ -50,10 +50,10 @@ class LoftService(Base):
def stream_to_s3_bucket(self): def stream_to_s3_bucket(self):
self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv' self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv'
logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['MEASURE_PROCESSED_BUCKET']) logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['S3_PROCESSED_BUCKET'])
# setup writer buffer and write processed forms to file # setup writer buffer and write processed forms to file
buffer = io.StringIO() buffer = io.StringIO()
solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms)
# upload generated file to s3 and return result # upload generated file to s3 and return result
return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['MEASURE_PROCESSED_BUCKET']) return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['S3_PROCESSED_BUCKET'])