67 lines
2.4 KiB
Python
67 lines
2.4 KiB
Python
import os
|
|
import json
|
|
import random
|
|
import io
|
|
|
|
from helpers import aws_helper, tar_helper, csv_helper, service_helper
|
|
|
|
from models.solver_run import SolverRun
|
|
from models.solution import Solution
|
|
from models.form import Form
|
|
|
|
from lib.irt.test_response_function import TestResponseFunction
|
|
from lib.irt.test_information_function import TestInformationFunction
|
|
|
|
from services.base import Base
|
|
|
|
class LoftService(Base):
|
|
def process(self):
|
|
self.solver_run = SolverRun.parse_obj(self.retreive_attributes_from_message())
|
|
self.solution = self.generate_solution()
|
|
self.result = self.stream_to_s3_bucket()
|
|
|
|
def retreive_attributes_from_message(self):
|
|
# get s3 object
|
|
self.key = aws_helper.get_key_from_message(self.source)
|
|
s3_object = aws_helper.get_object(self.key, os.environ['SOLVER_INGEST_BUCKET'])
|
|
|
|
# convert to tar
|
|
self.tar = tar_helper.raw_to_tar(s3_object)
|
|
|
|
# get attributes file and convert to dict
|
|
attributes = json.loads(tar_helper.extract_file_from_tar(self.tar , 'solver_run_attributes.json').read())
|
|
|
|
# get items file and convert to dict
|
|
items_csv = tar_helper.extract_file_from_tar(self.tar , 'items.csv')
|
|
items_csv_reader = csv_helper.file_stream_reader(items_csv)
|
|
|
|
# add items to attributes dict
|
|
attributes['items'] = service_helper.items_csv_to_dict(items_csv_reader)
|
|
|
|
return attributes
|
|
|
|
def generate_solution(self):
|
|
# temporary data for mocks
|
|
form_count = 10
|
|
|
|
# items will be generated from real solver process, this is for mock purposes
|
|
# real solver will return N forms and process a cut score, this is for mock purposes
|
|
return Solution(
|
|
response_id=random.randint(100,5000),
|
|
forms=[self.generate_forms(random.sample(self.solver_run.items, self.solver_run.total_form_items)) for x in range(form_count)]
|
|
)
|
|
|
|
def stream_to_s3_bucket(self):
|
|
# setup writer buffer and write processed forms to file
|
|
buffer = io.StringIO()
|
|
solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms)
|
|
|
|
# upload generated file to s3 and return result
|
|
return aws_helper.file_stream_upload(solution_file, f'{service_helper.key_to_uuid(self.key)}.csv', os.environ['MEASURE_PROCESSED_BUCKET'])
|
|
|
|
def generate_forms(self, items):
|
|
return Form(
|
|
items=items,
|
|
cut_score=TestResponseFunction(self.solver_run.irt_model).calculate(items, theta=self.solver_run.theta_cut_score)
|
|
)
|