jshbrmn/irt-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/QUANT-1196-solve-all-the-things

Browse Source
This commit is contained in:
Josh Burman
2021-11-03 20:15:19 +00:00
parent 2f86cbecfa d87aa17531
commit 6558875591
3 changed files with 15 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/helpers/aws_helper.py
View File

@ -3,16 +3,19 @@ import os
import json
session = boto3.Session(
aws_access_key_id=os.environ['SOLVER_AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['SOLVER_AWS_SECRET_ACCESS_KEY']
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY']
)
s3 = session.resource('s3', region_name=os.environ['SOLVER_AWS_REGION'])
sqs = session.client('sqs', region_name=os.environ['SOLVER_AWS_REGION'])
s3 = session.resource('s3', region_name=os.environ['AWS_REGION'])
sqs = session.client('sqs', region_name=os.environ['AWS_REGION'])
def get_key_from_message(body):
return body['Records'][0]['s3']['object']['key']
def get_bucket_from_message(body):
return body['Records'][0]['s3']['bucket']['name']
def get_object(key, bucket):
return s3.Object(
bucket_name=bucket,

10
app/main.py
View File

@ -19,13 +19,13 @@ class ServiceListener(SqsListener):
logging.info('Process complete for %s', service.file_name)
def main():
logging.info('Starting Solver Service (v0.3.2)...')
logging.info('Starting Solver Service (v0.4.3)...')
listener = ServiceListener(
'measure-development-solver-ingest',
region_name=os.environ['SOLVER_AWS_REGION'],
aws_access_key=os.environ['SOLVER_AWS_ACCESS_KEY_ID'],
aws_secret_key=os.environ['SOLVER_AWS_SECRET_ACCESS_KEY'],
queue_url=os.environ['SOLVER_SQS_INGEST_QUEUE']
region_name=os.environ['AWS_REGION'],
aws_access_key=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_key=os.environ['AWS_SECRET_ACCESS_KEY'],
queue_url=os.environ['SQS_QUEUE']
)
listener.listen()

6
app/services/loft_service.py
View File

@ -18,7 +18,7 @@ class LoftService(Base):
logging.info('Retrieving attributes from message...')
# get s3 object
self.key = aws_helper.get_key_from_message(self.source)
s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET'])
s3_object = aws_helper.get_object(self.key, aws_helper.get_bucket_from_message(self.source))
# convert to tar
self.tar = tar_helper.raw_to_tar(s3_object)
@ -50,10 +50,10 @@ class LoftService(Base):
def stream_to_s3_bucket(self):
self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv'
logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['MEASURE_PROCESSED_BUCKET'])
logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['S3_PROCESSED_BUCKET'])
# setup writer buffer and write processed forms to file
buffer = io.StringIO()
solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms)
# upload generated file to s3 and return result
return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['MEASURE_PROCESSED_BUCKET'])
return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['S3_PROCESSED_BUCKET'])