From b005efbe2ec7702440d9ded64e82097874df28c4 Mon Sep 17 00:00:00 2001 From: Josh Burman Date: Wed, 3 Nov 2021 17:18:02 +0000 Subject: [PATCH 1/5] envs fixed --- app/helpers/aws_helper.py | 8 ++++---- app/services/loft_service.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/helpers/aws_helper.py b/app/helpers/aws_helper.py index 89cc893..596f860 100644 --- a/app/helpers/aws_helper.py +++ b/app/helpers/aws_helper.py @@ -3,12 +3,12 @@ import os import json session = boto3.Session( - aws_access_key_id=os.environ['SOLVER_AWS_ACCESS_KEY_ID'], - aws_secret_access_key=os.environ['SOLVER_AWS_SECRET_ACCESS_KEY'] + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'] ) -s3 = session.resource('s3', region_name=os.environ['SOLVER_AWS_REGION']) -sqs = session.client('sqs', region_name=os.environ['SOLVER_AWS_REGION']) +s3 = session.resource('s3', region_name=os.environ['AWS_REGION']) +sqs = session.client('sqs', region_name=os.environ['AWS_REGION']) def get_key_from_message(body): return body['Records'][0]['s3']['object']['key'] diff --git a/app/services/loft_service.py b/app/services/loft_service.py index aae2ad5..c4f3f66 100644 --- a/app/services/loft_service.py +++ b/app/services/loft_service.py @@ -18,7 +18,7 @@ class LoftService(Base): logging.info('Retrieving attributes from message...') # get s3 object self.key = aws_helper.get_key_from_message(self.source) - s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET']) + s3_object = aws_helper.get_object(self.key, os.environ['INGEST_BUCKET']) # convert to tar self.tar = tar_helper.raw_to_tar(s3_object) From 1c7fb41ca5acc80b69e863a2ef2495952f3398c6 Mon Sep 17 00:00:00 2001 From: Josh Burman Date: Wed, 3 Nov 2021 17:22:27 +0000 Subject: [PATCH 2/5] more env var changes and version change --- app/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/main.py b/app/main.py index 7636ecc..d2183d1 100644 --- a/app/main.py +++ b/app/main.py @@ -19,13 +19,13 @@ class ServiceListener(SqsListener): logging.info('Process complete for %s', service.file_name) def main(): - logging.info('Starting Solver Service (v0.3.2)...') + logging.info('Starting Solver Service (v0.4.1)...') listener = ServiceListener( 'measure-development-solver-ingest', - region_name=os.environ['SOLVER_AWS_REGION'], - aws_access_key=os.environ['SOLVER_AWS_ACCESS_KEY_ID'], - aws_secret_key=os.environ['SOLVER_AWS_SECRET_ACCESS_KEY'], - queue_url=os.environ['SOLVER_SQS_INGEST_QUEUE'] + region_name=os.environ['AWS_REGION'], + aws_access_key=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_key=os.environ['AWS_SECRET_ACCESS_KEY'], + queue_url=os.environ['SQS_QUEUE'] ) listener.listen() From a42ae76e7217713cac214b7bd8162c3d3945ccba Mon Sep 17 00:00:00 2001 From: Josh Burman Date: Wed, 3 Nov 2021 17:58:48 +0000 Subject: [PATCH 3/5] use bucket name from payload instead of env var --- app/helpers/aws_helper.py | 3 +++ app/main.py | 2 +- app/services/loft_service.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/helpers/aws_helper.py b/app/helpers/aws_helper.py index 596f860..a7869af 100644 --- a/app/helpers/aws_helper.py +++ b/app/helpers/aws_helper.py @@ -13,6 +13,9 @@ sqs = session.client('sqs', region_name=os.environ['AWS_REGION']) def get_key_from_message(body): return body['Records'][0]['s3']['object']['key'] +def get_bucket_from_message(body): + return body['Records'][0]['s3']['bucket']['name'] + def get_object(key, bucket): return s3.Object( bucket_name=bucket, diff --git a/app/main.py b/app/main.py index d2183d1..d6ff386 100644 --- a/app/main.py +++ b/app/main.py @@ -19,7 +19,7 @@ class ServiceListener(SqsListener): logging.info('Process complete for %s', service.file_name) def main(): - logging.info('Starting Solver Service (v0.4.1)...') + logging.info('Starting Solver Service (v0.4.2)...') listener = ServiceListener( 'measure-development-solver-ingest', region_name=os.environ['AWS_REGION'], diff --git a/app/services/loft_service.py b/app/services/loft_service.py index c4f3f66..76ce231 100644 --- a/app/services/loft_service.py +++ b/app/services/loft_service.py @@ -18,7 +18,7 @@ class LoftService(Base): logging.info('Retrieving attributes from message...') # get s3 object self.key = aws_helper.get_key_from_message(self.source) - s3_object = aws_helper.get_object(self.key, os.environ['INGEST_BUCKET']) + s3_object = aws_helper.get_object(self.key, aws_helper.get_bucket_from_message(self.source)) # convert to tar self.tar = tar_helper.raw_to_tar(s3_object) From 29d8a539742ddd8c7fe3f09280c60b1d5c45b008 Mon Sep 17 00:00:00 2001 From: Josh Burman Date: Wed, 3 Nov 2021 18:16:04 +0000 Subject: [PATCH 4/5] env var rename --- app/services/loft_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/loft_service.py b/app/services/loft_service.py index 76ce231..ec0cb96 100644 --- a/app/services/loft_service.py +++ b/app/services/loft_service.py @@ -55,10 +55,10 @@ class LoftService(Base): def stream_to_s3_bucket(self): self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv' - logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['MEASURE_PROCESSED_BUCKET']) + logging.info('Streaming to %s s3 bucket %s', self.file_name, os.environ['S3_PROCESSED_BUCKET']) # setup writer buffer and write processed forms to file buffer = io.StringIO() solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms) # upload generated file to s3 and return result - return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['MEASURE_PROCESSED_BUCKET']) + return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['S3_PROCESSED_BUCKET']) From d87aa17531c9e0b2138f3df1063294c4cc236f82 Mon Sep 17 00:00:00 2001 From: Josh Burman Date: Wed, 3 Nov 2021 18:16:45 +0000 Subject: [PATCH 5/5] increment semver --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index d6ff386..748323b 100644 --- a/app/main.py +++ b/app/main.py @@ -19,7 +19,7 @@ class ServiceListener(SqsListener): logging.info('Process complete for %s', service.file_name) def main(): - logging.info('Starting Solver Service (v0.4.2)...') + logging.info('Starting Solver Service (v0.4.3)...') listener = ServiceListener( 'measure-development-solver-ingest', region_name=os.environ['AWS_REGION'],