the big format
This commit is contained in:
Joshua Burman
parent
19a37ab33a
commit
deb6b9014e
@ -3,37 +3,34 @@ import os
|
||||
import json
|
||||
|
||||
session = boto3.Session(
|
||||
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY']
|
||||
)
|
||||
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'])
|
||||
|
||||
s3 = session.resource('s3', region_name=os.environ['AWS_REGION'])
|
||||
sqs = session.client('sqs', region_name=os.environ['AWS_REGION'])
|
||||
|
||||
|
||||
def get_key_from_message(body):
|
||||
return body['Records'][0]['s3']['object']['key']
|
||||
return body['Records'][0]['s3']['object']['key']
|
||||
|
||||
|
||||
def get_bucket_from_message(body):
|
||||
return body['Records'][0]['s3']['bucket']['name']
|
||||
return body['Records'][0]['s3']['bucket']['name']
|
||||
|
||||
|
||||
def get_object(key, bucket):
|
||||
return s3.Object(
|
||||
bucket_name=bucket,
|
||||
key=key
|
||||
).get()['Body'].read()
|
||||
return s3.Object(bucket_name=bucket, key=key).get()['Body'].read()
|
||||
|
||||
|
||||
def file_stream_upload(buffer, name, bucket):
|
||||
return s3.Bucket(bucket).upload_fileobj(buffer, name)
|
||||
return s3.Bucket(bucket).upload_fileobj(buffer, name)
|
||||
|
||||
|
||||
def receive_message(queue, message_num=1, wait_time=1):
|
||||
return sqs.receive_message(
|
||||
QueueUrl=queue,
|
||||
MaxNumberOfMessages=message_num,
|
||||
WaitTimeSeconds=wait_time
|
||||
)
|
||||
return sqs.receive_message(QueueUrl=queue,
|
||||
MaxNumberOfMessages=message_num,
|
||||
WaitTimeSeconds=wait_time)
|
||||
|
||||
|
||||
def delete_message(queue, receipt):
|
||||
return sqs.delete_message(
|
||||
QueueUrl=queue,
|
||||
ReceiptHandle=receipt
|
||||
)
|
||||
return sqs.delete_message(QueueUrl=queue, ReceiptHandle=receipt)
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import csv
|
||||
import io
|
||||
|
||||
|
||||
def file_stream_reader(f):
|
||||
return csv.reader(io.StringIO(f.read().decode('ascii')))
|
||||
return csv.reader(io.StringIO(f.read().decode('ascii')))
|
||||
|
||||
@ -3,98 +3,120 @@ import io
|
||||
import re
|
||||
from tokenize import String
|
||||
|
||||
|
||||
def items_csv_to_dict(items_csv_reader, solver_run):
|
||||
items = []
|
||||
headers = []
|
||||
items = []
|
||||
headers = []
|
||||
|
||||
# get headers and items
|
||||
for key, row in enumerate(items_csv_reader):
|
||||
if key == 0:
|
||||
headers = row
|
||||
else:
|
||||
item = { 'attributes': [] }
|
||||
# get headers and items
|
||||
for key, row in enumerate(items_csv_reader):
|
||||
if key == 0:
|
||||
headers = row
|
||||
else:
|
||||
item = {'attributes': []}
|
||||
|
||||
# ensure that the b param is formatted correctly
|
||||
if row[len(headers) - 1] != '' and is_float(row[len(headers) - 1]):
|
||||
for key, col in enumerate(headers):
|
||||
if solver_run.irt_model.formatted_b_param() == col:
|
||||
value = float(row[key])
|
||||
item['b_param'] = value
|
||||
elif solver_run.get_constraint(col) and solver_run.get_constraint(col).reference_attribute.type == 'bundle':
|
||||
if row[key]:
|
||||
item[solver_run.get_constraint(col).reference_attribute.id] = row[key]
|
||||
elif solver_run.get_constraint(col):
|
||||
constraint = solver_run.get_constraint(col)
|
||||
item['attributes'].append({
|
||||
'id': col,
|
||||
'value': row[key],
|
||||
'type': constraint.reference_attribute.type
|
||||
})
|
||||
else:
|
||||
if row[key]:
|
||||
item[col] = row[key]
|
||||
# ensure that the b param is formatted correctly
|
||||
if row[len(headers) - 1] != '' and is_float(row[len(headers) - 1]):
|
||||
for key, col in enumerate(headers):
|
||||
if solver_run.irt_model.formatted_b_param() == col:
|
||||
value = float(row[key])
|
||||
item['b_param'] = value
|
||||
elif solver_run.get_constraint(
|
||||
col) and solver_run.get_constraint(
|
||||
col).reference_attribute.type == 'bundle':
|
||||
if row[key]:
|
||||
item[solver_run.get_constraint(
|
||||
col).reference_attribute.id] = row[key]
|
||||
elif solver_run.get_constraint(col):
|
||||
constraint = solver_run.get_constraint(col)
|
||||
item['attributes'].append({
|
||||
'id':
|
||||
col,
|
||||
'value':
|
||||
row[key],
|
||||
'type':
|
||||
constraint.reference_attribute.type
|
||||
})
|
||||
else:
|
||||
if row[key]:
|
||||
item[col] = row[key]
|
||||
|
||||
items.append(item)
|
||||
items.append(item)
|
||||
|
||||
return items
|
||||
|
||||
return items
|
||||
|
||||
def solution_to_file(buffer, total_form_items, forms):
|
||||
wr = csv.writer(buffer, dialect='excel', delimiter=',')
|
||||
wr = csv.writer(buffer, dialect='excel', delimiter=',')
|
||||
|
||||
# write header row for first row utilizing the total items all forms will have
|
||||
# fill the rows with the targets and cut score then the items
|
||||
header = ['status']
|
||||
# write header row for first row utilizing the total items all forms will have
|
||||
# fill the rows with the targets and cut score then the items
|
||||
header = ['status']
|
||||
|
||||
for result in forms[0].tif_results:
|
||||
header += [f'tif @ {round(result.theta, 2)}']
|
||||
for result in forms[0].tif_results:
|
||||
header += [f'tif @ {round(result.theta, 2)}']
|
||||
|
||||
for result in forms[0].tcc_results:
|
||||
header += [f'tcc @ {round(result.theta, 2)}']
|
||||
for result in forms[0].tcc_results:
|
||||
header += [f'tcc @ {round(result.theta, 2)}']
|
||||
|
||||
header += ['cut score'] + [x + 1 for x in range(total_form_items)]
|
||||
wr.writerow(header)
|
||||
header += ['cut score'] + [x + 1 for x in range(total_form_items)]
|
||||
wr.writerow(header)
|
||||
|
||||
# add each form as row to processed csv
|
||||
for form in forms:
|
||||
row = [form.status]
|
||||
# add each form as row to processed csv
|
||||
for form in forms:
|
||||
row = [form.status]
|
||||
|
||||
for result in form.tif_results + form.tcc_results:
|
||||
row += [f'target - {result.value}\nresult - {round(result.result, 2)}']
|
||||
for result in form.tif_results + form.tcc_results:
|
||||
row += [
|
||||
f'target - {result.value}\nresult - {round(result.result, 2)}'
|
||||
]
|
||||
|
||||
# provide generated items and cut score
|
||||
row += [round(form.cut_score, 2)] + [item.id for item in form.items]
|
||||
wr.writerow(row)
|
||||
# provide generated items and cut score
|
||||
row += [round(form.cut_score, 2)] + [item.id for item in form.items]
|
||||
wr.writerow(row)
|
||||
|
||||
buff2 = io.BytesIO(buffer.getvalue().encode())
|
||||
buff2 = io.BytesIO(buffer.getvalue().encode())
|
||||
|
||||
return buff2
|
||||
|
||||
return buff2
|
||||
|
||||
def error_to_file(buffer, error):
|
||||
wr = csv.writer(buffer, dialect='excel', delimiter=',')
|
||||
wr.writerow(['status'])
|
||||
wr.writerow([error.args[0]])
|
||||
wr = csv.writer(buffer, dialect='excel', delimiter=',')
|
||||
wr.writerow(['status'])
|
||||
wr.writerow([error.args[0]])
|
||||
|
||||
return io.BytesIO(buffer.getvalue().encode())
|
||||
|
||||
return io.BytesIO(buffer.getvalue().encode())
|
||||
|
||||
def key_to_uuid(key):
|
||||
return re.split("_", key)[0]
|
||||
return re.split("_", key)[0]
|
||||
|
||||
|
||||
def solution_items(variables, solver_run):
|
||||
form_items = []
|
||||
form_items = []
|
||||
|
||||
for v in variables:
|
||||
if v.varValue > 0 and 'Item_' in v.name:
|
||||
item_id = v.name.replace('Item_', '')
|
||||
item = solver_run.get_item(item_id)
|
||||
# add item to list and then remove from master item list
|
||||
form_items.append(item)
|
||||
for v in variables:
|
||||
if v.varValue > 0:
|
||||
if 'Item_' in v.name:
|
||||
item_id = v.name.replace('Item_', '')
|
||||
item = solver_run.get_item(item_id)
|
||||
# add item to list and then remove from master item list
|
||||
if item: form_items.append(item)
|
||||
elif 'Bundle_' in v.name:
|
||||
bundle_id = v.name.replace('Bundle_', '')
|
||||
bundle = solver_run.get_bundle(bundle_id)
|
||||
|
||||
if bundle:
|
||||
for item in bundle.items:
|
||||
if item: form_items.append(item)
|
||||
|
||||
return form_items
|
||||
|
||||
return form_items
|
||||
|
||||
# probably a better place for this...
|
||||
def is_float(element: String) -> bool:
|
||||
try:
|
||||
float(element)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
try:
|
||||
float(element)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
@ -9,75 +9,95 @@ from models.item import Item
|
||||
|
||||
from lib.errors.item_generation_error import ItemGenerationError
|
||||
|
||||
def build_constraints(solver_run: SolverRun, problem: LpProblem, items: list[Item]) -> LpProblem:
|
||||
logging.info('Creating Constraints...')
|
||||
|
||||
try:
|
||||
total_form_items = solver_run.total_form_items
|
||||
constraints = solver_run.constraints
|
||||
def build_constraints(solver_run: SolverRun, problem: LpProblem,
|
||||
items: list[Item], bundles: list[Bundle]) -> LpProblem:
|
||||
logging.info('Creating Constraints...')
|
||||
|
||||
for constraint in constraints:
|
||||
attribute = constraint.reference_attribute
|
||||
min = constraint.minimum
|
||||
max = constraint.maximum
|
||||
try:
|
||||
total_form_items = solver_run.total_form_items
|
||||
constraints = solver_run.constraints
|
||||
|
||||
if attribute.type == 'metadata':
|
||||
logging.info('Metadata Constraint Generating...')
|
||||
con = dict(zip([item.id for item in solver_run.items],
|
||||
[item.attribute_exists(attribute)
|
||||
for item in solver_run.items]))
|
||||
problem += lpSum([con[item.id]
|
||||
* items[item.id]
|
||||
for item in solver_run.items]) >= round(total_form_items * (min / 100)), f'{attribute.id} - {attribute.value} - min'
|
||||
problem += lpSum([con[item.id]
|
||||
* items[item.id]
|
||||
for item in solver_run.items]) <= round(total_form_items * (max / 100)), f'{attribute.id} - {attribute.value} - max'
|
||||
elif attribute.type == 'bundle':
|
||||
logging.info('Bundles Constraint Generating...')
|
||||
# TODO: account for many different bundle types, since the id condition in L33 could yield duplicates
|
||||
if solver_run.bundles != None:
|
||||
total_bundle_items = 0
|
||||
selected_bundles = get_random_bundles(solver_run.total_form_items, solver_run.bundles, int(constraint.minimum), int(constraint.maximum))
|
||||
for constraint in constraints:
|
||||
attribute = constraint.reference_attribute
|
||||
min = constraint.minimum
|
||||
max = constraint.maximum
|
||||
|
||||
for bundle in selected_bundles:
|
||||
con = dict(zip([item.id for item in solver_run.items],
|
||||
[(getattr(item, bundle.type, False) == bundle.id)
|
||||
for item in solver_run.items]))
|
||||
problem += lpSum([con[item.id]
|
||||
* items[item.id]
|
||||
for item in solver_run.items]) == bundle.count, f'Bundle constraint for {bundle.type} ({bundle.id})'
|
||||
total_bundle_items += bundle.count
|
||||
if attribute.type == 'metadata':
|
||||
logging.info('Metadata Constraint Generating...')
|
||||
con = dict(
|
||||
zip([item.id for item in solver_run.items], [
|
||||
item.attribute_exists(attribute)
|
||||
for item in solver_run.items
|
||||
]))
|
||||
problem += lpSum([
|
||||
con[item.id] * items[item.id] for item in solver_run.items
|
||||
]) >= round(
|
||||
total_form_items *
|
||||
(min / 100)), f'{attribute.id} - {attribute.value} - min'
|
||||
problem += lpSum([
|
||||
con[item.id] * items[item.id] for item in solver_run.items
|
||||
]) <= round(
|
||||
total_form_items *
|
||||
(max / 100)), f'{attribute.id} - {attribute.value} - max'
|
||||
elif attribute.type == 'bundle':
|
||||
logging.info('Bundles Constraint Generating...')
|
||||
# TODO: account for many different bundle types, since the id condition in L33 could yield duplicates
|
||||
if solver_run.bundles != None:
|
||||
# make sure the total bundles used in generated form is limited between min-max set
|
||||
problem += lpSum([
|
||||
bundles[bundle.id] for bundle in solver_run.bundles
|
||||
]) == randint(int(constraint.minimum),
|
||||
int(constraint.maximum))
|
||||
# total_bundle_items = 0
|
||||
# selected_bundles = get_random_bundles(solver_run.total_form_items, solver_run.bundles, int(constraint.minimum), int(constraint.maximum))
|
||||
|
||||
# make sure all other items added to the form
|
||||
# are not a part of any bundle
|
||||
# currently only supports single bundle constraints, will need refactoring for multiple bundle constraints
|
||||
con = dict(zip([item.id for item in solver_run.items],
|
||||
[(getattr(item, attribute.id, None) == None)
|
||||
for item in solver_run.items]))
|
||||
problem += lpSum([con[item.id]
|
||||
* items[item.id]
|
||||
for item in solver_run.items]) == solver_run.total_form_items - total_bundle_items, f'Remaining items are not of a bundle type'
|
||||
# for bundle in selected_bundles:
|
||||
# con = dict(zip([item.id for item in solver_run.items],
|
||||
# [(getattr(item, bundle.type, False) == bundle.id)
|
||||
# for item in solver_run.items]))
|
||||
# problem += lpSum([con[item.id]
|
||||
# * items[item.id]
|
||||
# for item in solver_run.items]) == bundle.count, f'Bundle constraint for {bundle.type} ({bundle.id})'
|
||||
# total_bundle_items += bundle.count
|
||||
|
||||
logging.info('Constraints Created...')
|
||||
return problem
|
||||
except ValueError as error:
|
||||
logging.error(error)
|
||||
raise ItemGenerationError("Bundle min and/or max larger than bundle amount provided", error.args[0])
|
||||
# # make sure all other items added to the form
|
||||
# # are not a part of any bundle
|
||||
# # currently only supports single bundle constraints, will need refactoring for multiple bundle constraints
|
||||
# con = dict(zip([item.id for item in solver_run.items],
|
||||
# [(getattr(item, attribute.id, None) == None)
|
||||
# for item in solver_run.items]))
|
||||
# problem += lpSum([con[item.id]
|
||||
# * items[item.id]
|
||||
# for item in solver_run.items]) == solver_run.total_form_items - total_bundle_items, f'Remaining items are not of a bundle type'
|
||||
|
||||
def get_random_bundles(total_form_items: int, bundles: list[Bundle], min: int , max: int, found_bundles = False) -> list[Bundle]:
|
||||
selected_bundles = None
|
||||
total_bundle_items = 0
|
||||
total_bundles = randint(min, max)
|
||||
logging.info(f'Selecting Bundles (total of {total_bundles})...')
|
||||
logging.info('Constraints Created...')
|
||||
return problem
|
||||
except ValueError as error:
|
||||
logging.error(error)
|
||||
raise ItemGenerationError(
|
||||
"Bundle min and/or max larger than bundle amount provided",
|
||||
error.args[0])
|
||||
|
||||
while found_bundles == False:
|
||||
selected_bundles = sample(bundles, total_bundles)
|
||||
total_bundle_items = sum(bundle.count for bundle in selected_bundles)
|
||||
|
||||
if total_bundle_items <= total_form_items:
|
||||
found_bundles = True
|
||||
def get_random_bundles(total_form_items: int,
|
||||
bundles: list[Bundle],
|
||||
min: int,
|
||||
max: int,
|
||||
found_bundles=False) -> list[Bundle]:
|
||||
selected_bundles = None
|
||||
total_bundle_items = 0
|
||||
total_bundles = randint(min, max)
|
||||
logging.info(f'Selecting Bundles (total of {total_bundles})...')
|
||||
|
||||
if found_bundles == True:
|
||||
return selected_bundles
|
||||
else:
|
||||
return get_random_bundles(total_form_items, total_bundles - 1, bundles)
|
||||
while found_bundles == False:
|
||||
selected_bundles = sample(bundles, total_bundles)
|
||||
total_bundle_items = sum(bundle.count for bundle in selected_bundles)
|
||||
|
||||
if total_bundle_items <= total_form_items:
|
||||
found_bundles = True
|
||||
|
||||
if found_bundles == True:
|
||||
return selected_bundles
|
||||
else:
|
||||
return get_random_bundles(total_form_items, total_bundles - 1, bundles)
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import io
|
||||
import tarfile
|
||||
|
||||
|
||||
def raw_to_tar(raw_object):
|
||||
tarball = io.BytesIO(raw_object)
|
||||
return tarfile.open(fileobj=tarball, mode='r:gz')
|
||||
tarball = io.BytesIO(raw_object)
|
||||
return tarfile.open(fileobj=tarball, mode='r:gz')
|
||||
|
||||
|
||||
def extract_file_from_tar(tar, file_name):
|
||||
return tar.extractfile(tar.getmember(file_name))
|
||||
return tar.extractfile(tar.getmember(file_name))
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
class ItemGenerationError(Exception):
|
||||
pass
|
||||
pass
|
||||
|
||||
@ -3,22 +3,28 @@ import logging
|
||||
from lib.irt.models.three_parameter_logistic import ThreeParameterLogistic
|
||||
from lib.errors.item_generation_error import ItemGenerationError
|
||||
|
||||
class ItemInformationFunction():
|
||||
def __init__(self, irt_model):
|
||||
self.model_data = irt_model
|
||||
|
||||
# determines the amount of information for a given question at a given theta (ability level)
|
||||
# further detailed on page 161, equation 4 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def calculate(self, **kwargs):
|
||||
try:
|
||||
if self.model_data.model == '3PL':
|
||||
p = ThreeParameterLogistic(self.model_data, kwargs).result()
|
||||
q = 1 - p
|
||||
return (self.model_data.a_param * q * (p - self.model_data.c_param)**2) / (p * ((1 - self.model_data.c_param)**2))
|
||||
else:
|
||||
# potentially error out
|
||||
raise ItemGenerationError("irt model not supported or provided")
|
||||
except ZeroDivisionError as error:
|
||||
logging.error(error)
|
||||
raise ItemGenerationError("params not well formatted", error.args[0])
|
||||
class ItemInformationFunction():
|
||||
|
||||
def __init__(self, irt_model):
|
||||
self.model_data = irt_model
|
||||
|
||||
# determines the amount of information for a given question at a given theta (ability level)
|
||||
# further detailed on page 161, equation 4 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def calculate(self, **kwargs):
|
||||
try:
|
||||
if self.model_data.model == '3PL':
|
||||
p = ThreeParameterLogistic(self.model_data, kwargs).result()
|
||||
q = 1 - p
|
||||
return (self.model_data.a_param * q *
|
||||
(p - self.model_data.c_param)**2) / (p * (
|
||||
(1 - self.model_data.c_param)**2))
|
||||
else:
|
||||
# potentially error out
|
||||
raise ItemGenerationError(
|
||||
"irt model not supported or provided")
|
||||
except ZeroDivisionError as error:
|
||||
logging.error(error)
|
||||
raise ItemGenerationError("params not well formatted",
|
||||
error.args[0])
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
from lib.irt.models.three_parameter_logistic import ThreeParameterLogistic
|
||||
from lib.errors.item_generation_error import ItemGenerationError
|
||||
|
||||
class ItemResponseFunction():
|
||||
def __init__(self, irt_model):
|
||||
self.model_data = irt_model
|
||||
|
||||
def calculate(self, **kwargs):
|
||||
if self.model_data.model == '3PL':
|
||||
return ThreeParameterLogistic(self.model_data, kwargs).result()
|
||||
else:
|
||||
raise ItemGenerationError("irt model not supported or provided")
|
||||
class ItemResponseFunction():
|
||||
|
||||
def __init__(self, irt_model):
|
||||
self.model_data = irt_model
|
||||
|
||||
def calculate(self, **kwargs):
|
||||
if self.model_data.model == '3PL':
|
||||
return ThreeParameterLogistic(self.model_data, kwargs).result()
|
||||
else:
|
||||
raise ItemGenerationError("irt model not supported or provided")
|
||||
|
||||
@ -1,16 +1,18 @@
|
||||
class ThreeParameterLogistic:
|
||||
def __init__(self, model_params, kwargs):
|
||||
self.model_params = model_params
|
||||
# check if exists, if not error out
|
||||
self.b_param = kwargs['b_param']
|
||||
self.e = 2.71828
|
||||
self.theta = kwargs['theta']
|
||||
|
||||
# contains the primary 3pl function, determining the probably of an inidividual
|
||||
# that an individual at a certain theta would get a particular question correct
|
||||
# detailed further on page 161, equation 1 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def result(self):
|
||||
a = self.model_params.a_param
|
||||
c = self.model_params.c_param
|
||||
return c + (1 - c) * (1 / (1 + self.e**(-a * (self.theta - self.b_param))))
|
||||
def __init__(self, model_params, kwargs):
|
||||
self.model_params = model_params
|
||||
# check if exists, if not error out
|
||||
self.b_param = kwargs['b_param']
|
||||
self.e = 2.71828
|
||||
self.theta = kwargs['theta']
|
||||
|
||||
# contains the primary 3pl function, determining the probably of an inidividual
|
||||
# that an individual at a certain theta would get a particular question correct
|
||||
# detailed further on page 161, equation 1 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def result(self):
|
||||
a = self.model_params.a_param
|
||||
c = self.model_params.c_param
|
||||
return c + (1 - c) * (1 / (1 + self.e**(-a *
|
||||
(self.theta - self.b_param))))
|
||||
|
||||
@ -1,19 +1,22 @@
|
||||
from lib.irt.item_information_function import ItemInformationFunction
|
||||
|
||||
|
||||
class TestInformationFunction():
|
||||
def __init__(self, irt_model):
|
||||
self.irt_model = irt_model
|
||||
self.iif = ItemInformationFunction(irt_model)
|
||||
|
||||
# determins the amount of information
|
||||
# at a certain theta (ability level) of the sum of a question set correct
|
||||
# detailed further on page 166, equation 4 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def calculate(self, items, **kwargs):
|
||||
sum = 0
|
||||
def __init__(self, irt_model):
|
||||
self.irt_model = irt_model
|
||||
self.iif = ItemInformationFunction(irt_model)
|
||||
|
||||
for item in items:
|
||||
result = self.iif.calculate(b_param=item.b_param, theta=kwargs['theta'])
|
||||
sum += result
|
||||
# determins the amount of information
|
||||
# at a certain theta (ability level) of the sum of a question set correct
|
||||
# detailed further on page 166, equation 4 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def calculate(self, items, **kwargs):
|
||||
sum = 0
|
||||
|
||||
return sum
|
||||
for item in items:
|
||||
result = self.iif.calculate(b_param=item.b_param,
|
||||
theta=kwargs['theta'])
|
||||
sum += result
|
||||
|
||||
return sum
|
||||
|
||||
@ -1,20 +1,23 @@
|
||||
from lib.irt.item_response_function import ItemResponseFunction
|
||||
|
||||
|
||||
# otherwise known as the Test Characteristic Curve (TCC)
|
||||
class TestResponseFunction():
|
||||
def __init__(self, irt_model):
|
||||
self.irt_model = irt_model
|
||||
self.irf = ItemResponseFunction(irt_model)
|
||||
|
||||
# determins the probably of an inidividual
|
||||
# at a certain theta (ability level) would get a sum of questions correct
|
||||
# detailed further on page 166, equation 3 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def calculate(self, items, **kwargs):
|
||||
sum = 0
|
||||
def __init__(self, irt_model):
|
||||
self.irt_model = irt_model
|
||||
self.irf = ItemResponseFunction(irt_model)
|
||||
|
||||
for item in items:
|
||||
result = self.irf.calculate(b_param=item.b_param, theta=kwargs['theta'])
|
||||
sum += result
|
||||
# determins the probably of an inidividual
|
||||
# at a certain theta (ability level) would get a sum of questions correct
|
||||
# detailed further on page 166, equation 3 here:
|
||||
# https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5978482/pdf/10.1177_0146621615613308.pdf
|
||||
def calculate(self, items, **kwargs):
|
||||
sum = 0
|
||||
|
||||
return sum
|
||||
for item in items:
|
||||
result = self.irf.calculate(b_param=item.b_param,
|
||||
theta=kwargs['theta'])
|
||||
sum += result
|
||||
|
||||
return sum
|
||||
|
||||
45
app/main.py
45
app/main.py
@ -6,31 +6,36 @@ from helpers import aws_helper
|
||||
from daemonize import Daemonize
|
||||
from sqs_listener import SqsListener
|
||||
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(levelname)s %(asctime)s - %(message)s")
|
||||
logging.basicConfig(stream=sys.stdout,
|
||||
level=logging.INFO,
|
||||
format="%(levelname)s %(asctime)s - %(message)s")
|
||||
|
||||
|
||||
class ServiceListener(SqsListener):
|
||||
def handle_message(self, body, attributes, messages_attributes):
|
||||
# gather/manage/process data based on the particular needs
|
||||
logging.info('Incoming message: %s', body)
|
||||
|
||||
service = LoftService(body)
|
||||
service.process()
|
||||
def handle_message(self, body, attributes, messages_attributes):
|
||||
# gather/manage/process data based on the particular needs
|
||||
logging.info('Incoming message: %s', body)
|
||||
|
||||
service = LoftService(body)
|
||||
service.process()
|
||||
|
||||
logging.info('Process complete for %s', service.file_name)
|
||||
|
||||
logging.info('Process complete for %s', service.file_name)
|
||||
|
||||
def main():
|
||||
logging.info('Starting Solver Service (v1.1.2)...')
|
||||
listener = ServiceListener(
|
||||
os.environ['SQS_QUEUE'],
|
||||
region_name=os.environ['AWS_REGION'],
|
||||
aws_access_key=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_key=os.environ['AWS_SECRET_ACCESS_KEY'],
|
||||
queue_url=os.environ['SQS_QUEUE']
|
||||
)
|
||||
listener.listen()
|
||||
logging.info('Starting Solver Service (v1.1.2)...')
|
||||
listener = ServiceListener(
|
||||
os.environ['SQS_QUEUE'],
|
||||
region_name=os.environ['AWS_REGION'],
|
||||
aws_access_key=os.environ['AWS_ACCESS_KEY_ID'],
|
||||
aws_secret_key=os.environ['AWS_SECRET_ACCESS_KEY'],
|
||||
queue_url=os.environ['SQS_QUEUE'])
|
||||
listener.listen()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
myname=os.path.basename(sys.argv[0])
|
||||
pidfile='/tmp/%s' % myname
|
||||
daemon = Daemonize(app=myname,pid=pidfile, action=main, foreground=True)
|
||||
daemon.start()
|
||||
myname = os.path.basename(sys.argv[0])
|
||||
pidfile = '/tmp/%s' % myname
|
||||
daemon = Daemonize(app=myname, pid=pidfile, action=main, foreground=True)
|
||||
daemon.start()
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
|
||||
class AdvancedOptions(BaseModel):
|
||||
linearity_check: Optional[bool]
|
||||
show_progress: Optional[bool]
|
||||
max_solution_time: Optional[int]
|
||||
brand_bound_tolerance: Optional[float]
|
||||
max_forms: Optional[int]
|
||||
precision: Optional[float]
|
||||
extra_param_range: Optional[List[Dict]]
|
||||
linearity_check: Optional[bool]
|
||||
show_progress: Optional[bool]
|
||||
max_solution_time: Optional[int]
|
||||
brand_bound_tolerance: Optional[float]
|
||||
max_forms: Optional[int]
|
||||
precision: Optional[float]
|
||||
extra_param_range: Optional[List[Dict]]
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class Attribute(BaseModel):
|
||||
value: Optional[str]
|
||||
type: Optional[str]
|
||||
id: str
|
||||
value: Optional[str]
|
||||
type: Optional[str]
|
||||
id: str
|
||||
|
||||
@ -1,6 +1,23 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import List
|
||||
|
||||
from lib.irt.test_information_function import TestInformationFunction
|
||||
from lib.irt.test_response_function import TestResponseFunction
|
||||
|
||||
from models.item import Item
|
||||
from models.irt_model import IRTModel
|
||||
|
||||
|
||||
class Bundle(BaseModel):
|
||||
id: int
|
||||
count: int
|
||||
type: str
|
||||
id: int
|
||||
count: int
|
||||
items: List[Item]
|
||||
type: str
|
||||
|
||||
def tif(self, irt_model: IRTModel, theta: float) -> float:
|
||||
return TestInformationFunction(irt_model).calculate(self.items,
|
||||
theta=theta)
|
||||
|
||||
def trf(self, irt_model: IRTModel, theta: float) -> float:
|
||||
return TestResponseFunction(irt_model).calculate(self.items,
|
||||
theta=theta)
|
||||
|
||||
@ -2,7 +2,8 @@ from pydantic import BaseModel
|
||||
|
||||
from models.attribute import Attribute
|
||||
|
||||
|
||||
class Constraint(BaseModel):
|
||||
reference_attribute: Attribute
|
||||
minimum: float
|
||||
maximum: float
|
||||
reference_attribute: Attribute
|
||||
minimum: float
|
||||
maximum: float
|
||||
|
||||
@ -8,19 +8,20 @@ from models.target import Target
|
||||
|
||||
from lib.irt.test_response_function import TestResponseFunction
|
||||
|
||||
class Form(BaseModel):
|
||||
items: List[Item]
|
||||
cut_score: float
|
||||
tif_results: List[Target]
|
||||
tcc_results: List[Target]
|
||||
status: str = 'Not Optimized'
|
||||
|
||||
@classmethod
|
||||
def create(cls, items, solver_run, status):
|
||||
return cls(
|
||||
items=items,
|
||||
cut_score=TestResponseFunction(solver_run.irt_model).calculate(items, theta=solver_run.theta_cut_score),
|
||||
tif_results=irt_helper.generate_tif_results(items, solver_run),
|
||||
tcc_results=irt_helper.generate_tcc_results(items, solver_run),
|
||||
status=status
|
||||
)
|
||||
class Form(BaseModel):
|
||||
items: List[Item]
|
||||
cut_score: float
|
||||
tif_results: List[Target]
|
||||
tcc_results: List[Target]
|
||||
status: str = 'Not Optimized'
|
||||
|
||||
@classmethod
|
||||
def create(cls, items, solver_run, status):
|
||||
return cls(
|
||||
items=items,
|
||||
cut_score=TestResponseFunction(solver_run.irt_model).calculate(
|
||||
items, theta=solver_run.theta_cut_score),
|
||||
tif_results=irt_helper.generate_tif_results(items, solver_run),
|
||||
tcc_results=irt_helper.generate_tcc_results(items, solver_run),
|
||||
status=status)
|
||||
|
||||
@ -1,12 +1,13 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Dict
|
||||
|
||||
|
||||
class IRTModel(BaseModel):
|
||||
a_param: float
|
||||
b_param: Dict = {"schema_bson_id": str, "field_bson_id": str}
|
||||
c_param: float
|
||||
model: str
|
||||
a_param: float
|
||||
b_param: Dict = {"schema_bson_id": str, "field_bson_id": str}
|
||||
c_param: float
|
||||
model: str
|
||||
|
||||
|
||||
def formatted_b_param(self):
|
||||
return self.b_param['schema_bson_id'] + '-' + self.b_param['field_bson_id']
|
||||
def formatted_b_param(self):
|
||||
return self.b_param['schema_bson_id'] + '-' + self.b_param[
|
||||
'field_bson_id']
|
||||
|
||||
@ -6,27 +6,32 @@ from models.attribute import Attribute
|
||||
from lib.irt.item_response_function import ItemResponseFunction
|
||||
from lib.irt.item_information_function import ItemInformationFunction
|
||||
|
||||
|
||||
class Item(BaseModel):
|
||||
id: int
|
||||
passage_id: Optional[int]
|
||||
workflow_state: Optional[str]
|
||||
attributes: List[Attribute]
|
||||
b_param: float = 0.00
|
||||
id: int
|
||||
passage_id: Optional[int]
|
||||
workflow_state: Optional[str]
|
||||
attributes: List[Attribute]
|
||||
b_param: float = 0.00
|
||||
|
||||
def iif(self, solver_run, theta):
|
||||
return ItemInformationFunction(solver_run.irt_model).calculate(b_param=self.b_param,theta=theta)
|
||||
def iif(self, solver_run, theta):
|
||||
return ItemInformationFunction(solver_run.irt_model).calculate(
|
||||
b_param=self.b_param, theta=theta)
|
||||
|
||||
def irf(self, solver_run, theta):
|
||||
return ItemResponseFunction(solver_run.irt_model).calculate(b_param=self.b_param,theta=theta)
|
||||
def irf(self, solver_run, theta):
|
||||
return ItemResponseFunction(solver_run.irt_model).calculate(
|
||||
b_param=self.b_param, theta=theta)
|
||||
|
||||
def get_attribute(self, ref_attribute):
|
||||
for attribute in self.attributes:
|
||||
if attribute.id == ref_attribute.id and attribute.value.lower() == ref_attribute.value.lower():
|
||||
return attribute.value
|
||||
return False
|
||||
def get_attribute(self, ref_attribute):
|
||||
for attribute in self.attributes:
|
||||
if attribute.id == ref_attribute.id and attribute.value.lower(
|
||||
) == ref_attribute.value.lower():
|
||||
return attribute.value
|
||||
return False
|
||||
|
||||
def attribute_exists(self, ref_attribute):
|
||||
for attribute in self.attributes:
|
||||
if attribute.id == ref_attribute.id and attribute.value.lower() == ref_attribute.value.lower():
|
||||
return True
|
||||
return False
|
||||
def attribute_exists(self, ref_attribute):
|
||||
for attribute in self.attributes:
|
||||
if attribute.id == ref_attribute.id and attribute.value.lower(
|
||||
) == ref_attribute.value.lower():
|
||||
return True
|
||||
return False
|
||||
|
||||
@ -3,11 +3,12 @@ from typing import Dict, List, AnyStr
|
||||
|
||||
from models.target import Target
|
||||
|
||||
|
||||
class ObjectiveFunction(BaseModel):
|
||||
# minimizing tif/tcc target value is only option currently
|
||||
# as we add more we can build this out to be more dynamic
|
||||
# likely with models representing each objective function type
|
||||
tif_targets: List[Target]
|
||||
tcc_targets: List[Target]
|
||||
objective: AnyStr = "minimize"
|
||||
weight: Dict = {'tif': 1, 'tcc': 1}
|
||||
# minimizing tif/tcc target value is only option currently
|
||||
# as we add more we can build this out to be more dynamic
|
||||
# likely with models representing each objective function type
|
||||
tif_targets: List[Target]
|
||||
tcc_targets: List[Target]
|
||||
objective: AnyStr = "minimize"
|
||||
weight: Dict = {'tif': 1, 'tcc': 1}
|
||||
|
||||
@ -3,6 +3,7 @@ from typing import List
|
||||
|
||||
from models.form import Form
|
||||
|
||||
|
||||
class Solution(BaseModel):
|
||||
response_id: int
|
||||
forms: List[Form]
|
||||
response_id: int
|
||||
forms: List[Form]
|
||||
|
||||
@ -10,64 +10,87 @@ from models.bundle import Bundle
|
||||
from models.objective_function import ObjectiveFunction
|
||||
from models.advanced_options import AdvancedOptions
|
||||
|
||||
|
||||
class SolverRun(BaseModel):
|
||||
items: List[Item] = []
|
||||
bundles: Optional[Bundle]
|
||||
constraints: List[Constraint]
|
||||
irt_model: IRTModel
|
||||
objective_function: ObjectiveFunction
|
||||
total_form_items: int
|
||||
total_forms: int = 1
|
||||
theta_cut_score: float = 0.00
|
||||
advanced_options: Optional[AdvancedOptions]
|
||||
engine: str
|
||||
items: List[Item] = []
|
||||
bundles: list[Bundle] = []
|
||||
constraints: List[Constraint]
|
||||
irt_model: IRTModel
|
||||
objective_function: ObjectiveFunction
|
||||
total_form_items: int
|
||||
total_forms: int = 1
|
||||
theta_cut_score: float = 0.00
|
||||
advanced_options: Optional[AdvancedOptions]
|
||||
engine: str
|
||||
|
||||
def get_item(self, item_id):
|
||||
for item in self.items:
|
||||
if str(item.id) == item_id:
|
||||
return item
|
||||
return False
|
||||
def get_item(self, item_id: int) -> Item or None:
|
||||
for item in self.items:
|
||||
if str(item.id) == item_id:
|
||||
return item
|
||||
|
||||
def remove_items(self, items):
|
||||
self.items = [item for item in self.items if item not in items]
|
||||
return True
|
||||
def get_bundle(self, bundle_id: int) -> Bundle or None:
|
||||
for bundle in self.bundles:
|
||||
if str(bundle.id) == bundle_id:
|
||||
return bundle
|
||||
|
||||
def generate_bundles(self):
|
||||
logging.info('Generating Bundles...')
|
||||
bundle_constraints = (constraint.reference_attribute for constraint in self.constraints if constraint.reference_attribute.type == 'bundle')
|
||||
def get_constraint_by_type(self, type: str) -> Constraint or None:
|
||||
for constraint in self.constraints:
|
||||
if type == constraint.reference_attribute.type:
|
||||
return constraint
|
||||
|
||||
for bundle_constraint in bundle_constraints:
|
||||
type_attribute = bundle_constraint.id
|
||||
def remove_items(self, items: list[Item]) -> bool:
|
||||
self.items = [item for item in self.items if item not in items]
|
||||
return True
|
||||
|
||||
for item in self.items:
|
||||
attribute_id = getattr(item, type_attribute, None)
|
||||
def generate_bundles(self):
|
||||
logging.info('Generating Bundles...')
|
||||
# confirms bundle constraints exists
|
||||
bundle_constraints = (
|
||||
constraint.reference_attribute for constraint in self.constraints
|
||||
if constraint.reference_attribute.type == 'bundle')
|
||||
|
||||
# make sure the item has said attribute
|
||||
if attribute_id != None:
|
||||
# if there are pre-existing bundles, add new or increment existing
|
||||
# else create array with new bundle
|
||||
if self.bundles != None:
|
||||
# get index of the bundle in the bundles list if exists or None if it doesn't
|
||||
bundle_index = next((index for (index, bundle) in enumerate(self.bundles) if bundle.id == attribute_id and bundle.type == type_attribute), None)
|
||||
for bundle_constraint in bundle_constraints:
|
||||
type_attribute = bundle_constraint.id
|
||||
|
||||
# if the index doesn't exist add the new bundle of whatever type
|
||||
# else increment the count of the current bundle
|
||||
if bundle_index == None:
|
||||
self.bundles.append(Bundle(
|
||||
id=attribute_id,
|
||||
count=1,
|
||||
type=type_attribute
|
||||
))
|
||||
else:
|
||||
self.bundles[bundle_index].count += 1
|
||||
else:
|
||||
self.bundles = [Bundle(
|
||||
id=attribute_id,
|
||||
count=1,
|
||||
type=type_attribute
|
||||
)]
|
||||
for item in self.items:
|
||||
attribute_id = getattr(item, type_attribute, None)
|
||||
|
||||
logging.info('Bundles Generated...')
|
||||
# make sure the item has said attribute
|
||||
if attribute_id != None:
|
||||
# if there are pre-existing bundles, add new or increment existing
|
||||
# else create array with new bundle
|
||||
if self.bundles != None:
|
||||
# get index of the bundle in the bundles list if exists or None if it doesn't
|
||||
bundle_index = next(
|
||||
(index
|
||||
for (index, bundle) in enumerate(self.bundles)
|
||||
if bundle.id == attribute_id
|
||||
and bundle.type == type_attribute), None)
|
||||
|
||||
def get_constraint(self, name):
|
||||
return next((constraint for constraint in self.constraints if constraint.reference_attribute.id == name), None)
|
||||
# if the index doesn't exist add the new bundle of whatever type
|
||||
# else increment the count of the current bundle
|
||||
if bundle_index == None:
|
||||
self.bundles.append(
|
||||
Bundle(id=attribute_id,
|
||||
count=1,
|
||||
items=[item],
|
||||
type=type_attribute))
|
||||
else:
|
||||
self.bundles[bundle_index].count += 1
|
||||
self.bundles[bundle_index].items.append(item)
|
||||
else:
|
||||
self.bundles = [
|
||||
Bundle(id=attribute_id,
|
||||
count=1,
|
||||
items=[item],
|
||||
type=type_attribute)
|
||||
]
|
||||
|
||||
logging.info('Bundles Generated...')
|
||||
|
||||
def get_constraint(self, name: str) -> Constraint:
|
||||
return next((constraint for constraint in self.constraints
|
||||
if constraint.reference_attribute.id == name), None)
|
||||
|
||||
def unbundled_items(self) -> list:
|
||||
return [item for item in self.items if item.passage_id == None]
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class Target(BaseModel):
|
||||
theta: float
|
||||
value: float
|
||||
result: Optional[float]
|
||||
theta: float
|
||||
value: float
|
||||
result: Optional[float]
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
class Base:
|
||||
def __init__(self, source, ingest_type='message'):
|
||||
self.ingest_type = ingest_type
|
||||
self.source = source
|
||||
|
||||
def __init__(self, source, ingest_type='message'):
|
||||
self.ingest_type = ingest_type
|
||||
self.source = source
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import os, json, random, io, logging
|
||||
|
||||
from pulp import LpProblem, LpVariable, LpMinimize, LpStatus, lpSum
|
||||
from pulp import LpProblem, LpVariable, LpMinimize, LpMaximize, LpStatus, lpSum
|
||||
|
||||
from helpers import aws_helper, tar_helper, csv_helper, service_helper, solver_helper
|
||||
from lib.errors.item_generation_error import ItemGenerationError
|
||||
@ -12,134 +12,233 @@ from models.item import Item
|
||||
|
||||
from services.base import Base
|
||||
|
||||
|
||||
class LoftService(Base):
|
||||
def process(self):
|
||||
try:
|
||||
self.solver_run = self.create_solver_run_from_attributes()
|
||||
self.solver_run.generate_bundles()
|
||||
self.solution = self.generate_solution()
|
||||
self.result = self.stream_to_s3_bucket()
|
||||
except ItemGenerationError as error:
|
||||
self.result = self.stream_to_s3_bucket(error)
|
||||
except TypeError as error:
|
||||
logging.error(error)
|
||||
self.result = self.stream_to_s3_bucket(ItemGenerationError("Provided params causing error in calculation results"))
|
||||
|
||||
def create_solver_run_from_attributes(self) -> SolverRun:
|
||||
logging.info('Retrieving attributes from message...')
|
||||
# get s3 object
|
||||
self.key = aws_helper.get_key_from_message(self.source)
|
||||
s3_object = aws_helper.get_object(self.key, aws_helper.get_bucket_from_message(self.source))
|
||||
def process(self):
|
||||
try:
|
||||
self.solver_run = self.create_solver_run_from_attributes()
|
||||
self.solver_run.generate_bundles()
|
||||
self.solution = self.generate_solution()
|
||||
# self.solution = self.generate_test_solution()
|
||||
self.result = self.stream_to_s3_bucket()
|
||||
except ItemGenerationError as error:
|
||||
self.result = self.stream_to_s3_bucket(error)
|
||||
except TypeError as error:
|
||||
logging.error(error)
|
||||
self.result = self.stream_to_s3_bucket(
|
||||
ItemGenerationError(
|
||||
"Provided params causing error in calculation results"))
|
||||
|
||||
# convert to tar
|
||||
self.tar = tar_helper.raw_to_tar(s3_object)
|
||||
def generate_test_solution(self) -> Solution:
|
||||
solution = Solution(response_id=random.randint(100, 5000), forms=[])
|
||||
|
||||
# get attributes file and convert to dict
|
||||
attributes = json.loads(tar_helper.extract_file_from_tar(self.tar , 'solver_run_attributes.json').read())
|
||||
problem = LpProblem("ata-form-generate-with-bundles", LpMinimize)
|
||||
|
||||
# create solver run
|
||||
solver_run = SolverRun.parse_obj(attributes)
|
||||
bundles = LpVariable.dicts(
|
||||
"Bundle", [bundle.id for bundle in self.solver_run.bundles],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
items = LpVariable.dicts("Item",
|
||||
[item.id for item in self.solver_run.items],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
|
||||
# get items file and convert to dict
|
||||
items_csv = tar_helper.extract_file_from_tar(self.tar , 'items.csv')
|
||||
items_csv_reader = csv_helper.file_stream_reader(items_csv)
|
||||
problem += lpSum(
|
||||
[bundles[bundle.id] for bundle in self.solver_run.bundles])
|
||||
# problem += lpSum([items[item.id] for item in self.solver_run.items])
|
||||
|
||||
# add items to solver run
|
||||
for item in service_helper.items_csv_to_dict(items_csv_reader, solver_run):
|
||||
solver_run.items.append(Item.parse_obj(item))
|
||||
# problem += lpSum([bundles[bundle.id] for bundle in self.solver_run.bundles]) <= 3, 'max total bundles used'
|
||||
# problem += lpSum([bundles[bundle.id] for bundle in self.solver_run.bundles]) >= 1, 'min total bundles used'
|
||||
problem += lpSum(
|
||||
[bundles[bundle.id] for bundle in self.solver_run.bundles]) == 3
|
||||
|
||||
logging.info('Processed Attributes...')
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.count * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[1 * items[item.id] for item in self.solver_run.unbundled_items()]
|
||||
) == self.solver_run.total_form_items, 'Total bundle form items for form'
|
||||
|
||||
return solver_run
|
||||
problem.solve()
|
||||
|
||||
def generate_solution(self) -> Solution:
|
||||
logging.info('Generating Solution...')
|
||||
# for v in problem.variables():
|
||||
# print(f'{v.name}: {v.varValue}')
|
||||
|
||||
# unsolved solution
|
||||
solution = Solution(
|
||||
response_id=random.randint(100, 5000),
|
||||
forms=[]
|
||||
)
|
||||
# add return items and create as a form
|
||||
form_items = service_helper.solution_items(problem.variables(),
|
||||
self.solver_run)
|
||||
|
||||
# counter for number of forms
|
||||
f = 0
|
||||
# add form to solution
|
||||
solution.forms.append(
|
||||
Form.create(form_items, self.solver_run, LpStatus[problem.status]))
|
||||
logging.info('Form generated and added to solution...')
|
||||
|
||||
# iterate for number of forms that require creation
|
||||
# currently creates distinc forms with no item overlap
|
||||
while f < self.solver_run.total_forms:
|
||||
# setup vars
|
||||
items = LpVariable.dicts(
|
||||
"Item", [item.id for item in self.solver_run.items], lowBound=1, upBound=1, cat='Binary')
|
||||
# bundles = LpVariable.dicts(
|
||||
# "Bundle", [bundle.id for bundle in self.solver_run.bundles], lowBound=1, upBound=1, cat='Binary')
|
||||
return solution
|
||||
|
||||
problem_objection_functions = []
|
||||
def create_solver_run_from_attributes(self) -> SolverRun:
|
||||
logging.info('Retrieving attributes from message...')
|
||||
# get s3 object
|
||||
self.key = aws_helper.get_key_from_message(self.source)
|
||||
s3_object = aws_helper.get_object(
|
||||
self.key, aws_helper.get_bucket_from_message(self.source))
|
||||
|
||||
# create problem
|
||||
problem = LpProblem("ata-form-generate", LpMinimize)
|
||||
# convert to tar
|
||||
self.tar = tar_helper.raw_to_tar(s3_object)
|
||||
|
||||
# dummy objective function, because it just makes things easier™
|
||||
# problem += lpSum([items[item.id]
|
||||
# for item in self.solver_run.items])
|
||||
# get attributes file and convert to dict
|
||||
attributes = json.loads(
|
||||
tar_helper.extract_file_from_tar(
|
||||
self.tar, 'solver_run_attributes.json').read())
|
||||
|
||||
# constraints
|
||||
problem += lpSum([items[item.id]
|
||||
for item in self.solver_run.items]) == self.solver_run.total_form_items, 'Total form items'
|
||||
# create solver run
|
||||
solver_run = SolverRun.parse_obj(attributes)
|
||||
|
||||
# dynamic constraints
|
||||
problem = solver_helper.build_constraints(self.solver_run, problem, items)
|
||||
# get items file and convert to dict
|
||||
items_csv = tar_helper.extract_file_from_tar(self.tar, 'items.csv')
|
||||
items_csv_reader = csv_helper.file_stream_reader(items_csv)
|
||||
|
||||
# multi-objective constraints
|
||||
logging.info('Creating TIF and TCC constraints')
|
||||
for target in self.solver_run.objective_function.tif_targets:
|
||||
tif = lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items])
|
||||
problem_objection_functions.append(tif)
|
||||
problem += lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) >= target.value - 8, f'max tif theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) <= target.value + 8, f'min tif theta ({target.theta}) target value {target.value}'
|
||||
# add items to solver run
|
||||
for item in service_helper.items_csv_to_dict(items_csv_reader,
|
||||
solver_run):
|
||||
solver_run.items.append(Item.parse_obj(item))
|
||||
|
||||
for target in self.solver_run.objective_function.tcc_targets:
|
||||
tcc = lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items])
|
||||
problem_objection_functions.append(tcc)
|
||||
problem += lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) >= target.value - 20, f'max tcc theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) <= target.value + 20, f'min tcc theta ({target.theta}) target value {target.value}'
|
||||
logging.info('Processed Attributes...')
|
||||
|
||||
# solve problem
|
||||
logging.info('Solving...')
|
||||
# problem.solve()
|
||||
problem.sequentialSolve(problem_objection_functions)
|
||||
logging.info('Solved...generating form and adding to solution')
|
||||
return solver_run
|
||||
|
||||
# add return items and create as a form
|
||||
form_items = service_helper.solution_items(problem.variables(), self.solver_run)
|
||||
def generate_solution(self) -> Solution:
|
||||
logging.info('Generating Solution...')
|
||||
|
||||
# add form to solution
|
||||
solution.forms.append(Form.create(form_items, self.solver_run, LpStatus[problem.status]))
|
||||
logging.info('Form generated and added to solution...')
|
||||
# unsolved solution
|
||||
solution = Solution(response_id=random.randint(100, 5000), forms=[])
|
||||
|
||||
# successfull form, increment
|
||||
f += 1
|
||||
# counter for number of forms
|
||||
f = 0
|
||||
|
||||
logging.info('Solution Generated.')
|
||||
return solution
|
||||
# iterate for number of forms that require creation
|
||||
# currently creates distinc forms with no item overlap
|
||||
while f < self.solver_run.total_forms:
|
||||
# setup vars
|
||||
items = LpVariable.dicts(
|
||||
"Item", [item.id for item in self.solver_run.items],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
bundles = LpVariable.dicts(
|
||||
"Bundle", [bundle.id for bundle in self.solver_run.bundles],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
|
||||
def stream_to_s3_bucket(self, error = None):
|
||||
self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv'
|
||||
solution_file = None
|
||||
# setup writer buffer and write processed forms to file
|
||||
buffer = io.StringIO()
|
||||
# problem_objection_functions = []
|
||||
|
||||
if error:
|
||||
logging.info('Streaming %s error response to s3 bucket - %s', self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.error_to_file(buffer, error)
|
||||
else:
|
||||
logging.info('Streaming %s to s3 bucket - %s', self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms)
|
||||
# create problem
|
||||
problem = LpProblem("ata-form-generate", LpMinimize)
|
||||
|
||||
# upload generated file to s3 and return result
|
||||
return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
# dummy objective function, because it just makes things easier™
|
||||
problem += lpSum(
|
||||
[items[item.id] for item in self.solver_run.items])
|
||||
|
||||
# constraints
|
||||
# problem += lpSum([items[item.id]
|
||||
# for item in self.solver_run.items]) == self.solver_run.total_form_items, 'Total form items'
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.count * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
1 * items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
) == self.solver_run.total_form_items, 'Total bundle form items for form'
|
||||
|
||||
# dynamic constraints
|
||||
problem = solver_helper.build_constraints(self.solver_run, problem,
|
||||
items, bundles)
|
||||
|
||||
# multi-objective constraints
|
||||
logging.info('Creating TIF and TCC constraints')
|
||||
for target in self.solver_run.objective_function.tif_targets:
|
||||
|
||||
# tif = lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
# for item in self.solver_run.items])
|
||||
# problem_objection_functions.append(tif)
|
||||
problem += lpSum([
|
||||
bundle.tif(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.iif(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) >= target.value - 8, f'max tif theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([
|
||||
bundle.tif(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.iif(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) <= target.value + 8, f'min tif theta ({target.theta}) target value {target.value}'
|
||||
|
||||
for target in self.solver_run.objective_function.tcc_targets:
|
||||
# tcc = lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
# for item in self.solver_run.items])
|
||||
# problem_objection_functions.append(tcc)
|
||||
problem += lpSum([
|
||||
bundle.trf(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.irf(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) >= target.value - 20, f'max tcc theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([
|
||||
bundle.trf(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.irf(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) <= target.value + 20, f'min tcc theta ({target.theta}) target value {target.value}'
|
||||
|
||||
# solve problem
|
||||
logging.info('Solving...')
|
||||
problem.solve()
|
||||
# problem.sequentialSolve(problem_objection_functions)
|
||||
logging.info('Solved...generating form and adding to solution')
|
||||
|
||||
# add return items and create as a form
|
||||
form_items = service_helper.solution_items(problem.variables(),
|
||||
self.solver_run)
|
||||
|
||||
# add form to solution
|
||||
solution.forms.append(
|
||||
Form.create(form_items, self.solver_run,
|
||||
LpStatus[problem.status]))
|
||||
logging.info('Form generated and added to solution...')
|
||||
|
||||
# successfull form, increment
|
||||
f += 1
|
||||
|
||||
logging.info('Solution Generated.')
|
||||
return solution
|
||||
|
||||
def stream_to_s3_bucket(self, error=None):
|
||||
self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv'
|
||||
solution_file = None
|
||||
# setup writer buffer and write processed forms to file
|
||||
buffer = io.StringIO()
|
||||
|
||||
if error:
|
||||
logging.info('Streaming %s error response to s3 bucket - %s',
|
||||
self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.error_to_file(buffer, error)
|
||||
else:
|
||||
logging.info('Streaming %s to s3 bucket - %s', self.file_name,
|
||||
os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.solution_to_file(
|
||||
buffer, self.solver_run.total_form_items, self.solution.forms)
|
||||
|
||||
# upload generated file to s3 and return result
|
||||
return aws_helper.file_stream_upload(solution_file, self.file_name,
|
||||
os.environ['S3_PROCESSED_BUCKET'])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user