jshbrmn/irt-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

trial for variability

Browse Source
This commit is contained in:
Joshua Burman 2022-03-25 13:58:31 -04:00
parent 1f00e1e1bc
commit 11a5112812
2 changed files with 147 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app/models/solver_run.py
View File

@ -2,6 +2,7 @@ from pydantic import BaseModel
from typing import List, Literal, Optional from typing import List, Literal, Optional
import logging import logging
import random
from models.item import Item from models.item import Item
from models.constraint import Constraint from models.constraint import Constraint
@ -89,7 +90,7 @@ class SolverRun(BaseModel):
# temporary compensator for bundle item limits, since we shouldn't be using cases with less than 3 items # temporary compensator for bundle item limits, since we shouldn't be using cases with less than 3 items
# ideally this should be in the bundles model as a new attribute to handle "constraints of constraints" # ideally this should be in the bundles model as a new attribute to handle "constraints of constraints"
logging.info('Removing bundles with items < 3') logging.info('Removing bundles with items < 3')
for k,v in enumerate(self.bundles): for k, v in enumerate(self.bundles):
bundle = self.bundles[k] bundle = self.bundles[k]
if bundle.count < 3: del self.bundles[k] if bundle.count < 3: del self.bundles[k]
@ -103,4 +104,26 @@ class SolverRun(BaseModel):
# since the only bundles are based on passage id currently # since the only bundles are based on passage id currently
# in the future when we have more than just passage based bundles # in the future when we have more than just passage based bundles
# we'll need to develop a more sophisticated way of handling this concern # we'll need to develop a more sophisticated way of handling this concern
return [item for item in self.items if item.passage_id == None] bundle_constraints = (
constraint.reference_attribute for constraint in self.constraints
if constraint.reference_attribute.type == 'bundle')
if len(bundle_constraints) > 0:
return [item for item in self.items if item.passage_id == None]
else:
return self.items
def select_items_by_percent(self, percent) -> list[Item]:
items = self.unbundled_items()
total_items = len(items)
selected_items_amount = round(total_items - (total_items *
(percent / 100)))
return random.sample(items, selected_items_amount)
def select_bundles_by_percent(self, percent) -> list[Bundle]:
total_bundles = len(self.bundles)
selected_bundles_amount = round(total_bundles - (total_bundles *
(percent / 100)))
return random.sample(self.bundles, selected_bundles_amount)

199
app/services/loft_service.py
View File

@ -14,6 +14,7 @@ from models.target import Target
from services.base import Base from services.base import Base
class LoftService(Base): class LoftService(Base):
def process(self): def process(self):
@ -53,7 +54,8 @@ class LoftService(Base):
items_csv_reader = csv_helper.file_stream_reader(items_csv) items_csv_reader = csv_helper.file_stream_reader(items_csv)
# add items to solver run # add items to solver run
solver_run.items = service_helper.csv_to_item(items_csv_reader, solver_run) solver_run.items = service_helper.csv_to_item(items_csv_reader,
solver_run)
logging.info('Processed Attributes...') logging.info('Processed Attributes...')
@ -62,115 +64,151 @@ class LoftService(Base):
def generate_solution(self) -> Solution: def generate_solution(self) -> Solution:
logging.info('Generating Solution...') logging.info('Generating Solution...')
solution = Solution(response_id=random.randint(100, 5000), forms=[]) # unsolved solution solution = Solution(response_id=random.randint(100, 5000),
forms=[]) # unsolved solution
# setup common Solver variables
items = LpVariable.dicts("Item", [item.id for item in self.solver_run.unbundled_items()], lowBound=0, upBound=1, cat='Binary')
bundles = LpVariable.dicts("Bundle", [bundle.id for bundle in self.solver_run.bundles], lowBound=0, upBound=1, cat='Binary')
# iterate for number of forms that require creation # iterate for number of forms that require creation
for form_count in range(self.solver_run.total_forms): for form_count in range(self.solver_run.total_forms):
form_number = form_count + 1 form_number = form_count + 1
current_drift = 0 # FF Tokyo Drift current_drift = 0 # FF Tokyo Drift
selected_items = []
selected_bundles = []
if form_count == 1:
selected_items = self.solver_run.unbundled_items()
selected_bundles = self.solver_run.bundles
else:
selected_items = self.solver_run.select_items_by_percent(30)
selected_bundles = self.solver_run.select_bundles_by_percent(
30)
# setup common Solver variables
items = LpVariable.dicts("Item",
[item.id for item in selected_items],
lowBound=0,
upBound=1,
cat='Binary')
bundles = LpVariable.dicts(
"Bundle", [bundle.id for bundle in selected_bundles],
lowBound=0,
upBound=1,
cat='Binary')
logging.info(f'Generating Solution for Form {form_number}') logging.info(f'Generating Solution for Form {form_number}')
while current_drift <= Target.max_drift(): while current_drift <= Target.max_drift():
drift_percent = current_drift / 100 drift_percent = current_drift / 100
self.solver_run.objective_function.update_targets_drift(drift_percent) self.solver_run.objective_function.update_targets_drift(
drift_percent)
# create problem # create problem
problem = LpProblem('ata-form-generate', LpMinimize) problem = LpProblem('ata-form-generate', LpMinimize)
# objective function # objective function
problem += lpSum( problem += lpSum([
[ bundle.count * bundles[bundle.id]
bundle.count * bundles[bundle.id] for bundle in self.solver_run.bundles for bundle in self.solver_run.bundles
] + ] + [
[ items[item.id]
items[item.id] for item in self.solver_run.unbundled_items() for item in self.solver_run.unbundled_items()
] ])
)
# Form Constraints # Form Constraints
problem += lpSum( problem += lpSum(
[ [
bundle.count * bundles[bundle.id] for bundle in self.solver_run.bundles bundle.count * bundles[bundle.id]
] + for bundle in self.solver_run.bundles
[ ] + [
1 * items[item.id] for item in self.solver_run.unbundled_items() 1 * items[item.id]
for item in self.solver_run.unbundled_items()
] ]
) == self.solver_run.total_form_items, f'Total bundle form items for form {form_number}' ) == self.solver_run.total_form_items, f'Total bundle form items for form {form_number}'
# Dynamic constraints.. currently we only support Metadata and Bundles(Cases/Passages) # Dynamic constraints.. currently we only support Metadata and Bundles(Cases/Passages)
problem = solver_helper.build_constraints(self.solver_run, problem, items, bundles) problem = solver_helper.build_constraints(
self.solver_run, problem, items, bundles)
# form uniqueness constraints # form uniqueness constraints
for form in solution.forms: for form in solution.forms:
form_item_options = [ form_item_options = [
bundles[bundle.id] for bundle in self.solver_run.bundles bundles[bundle.id]
for bundle in self.solver_run.bundles
] + [ ] + [
items[item.id] for item in self.solver_run.unbundled_items() items[item.id]
for item in self.solver_run.unbundled_items()
] ]
problem += len(set(form.solver_variables)&set(form_item_options)) / float(len(set(form.solver_variables) | set(form_item_options))) * 100 >= 10 problem += len(
set(form.solver_variables)
& set(form_item_options)) / float(
len(
set(form.solver_variables)
| set(form_item_options))) * 100 >= 10
logging.info('Creating TIF and TCC Elastic constraints') logging.info('Creating TIF and TCC Elastic constraints')
# Behold our very own Elastic constraints! # Behold our very own Elastic constraints!
for tif_target in self.solver_run.objective_function.tif_targets: for tif_target in self.solver_run.objective_function.tif_targets:
problem += lpSum( problem += lpSum([
[ bundle.tif(self.solver_run.irt_model, tif_target.theta)
bundle.tif(self.solver_run.irt_model, tif_target.theta) * bundles[bundle.id] * bundles[bundle.id]
for bundle in self.solver_run.bundles for bundle in self.solver_run.bundles
] + ] + [
[ item.iif(self.solver_run, tif_target.theta) *
item.iif(self.solver_run, tif_target.theta) * items[item.id] items[item.id]
for item in self.solver_run.unbundled_items() for item in self.solver_run.unbundled_items()
] ]) >= tif_target.minimum(
) >= tif_target.minimum(), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%'
problem += lpSum( problem += lpSum([
[ bundle.tif(self.solver_run.irt_model, tif_target.theta)
bundle.tif(self.solver_run.irt_model, tif_target.theta) * bundles[bundle.id] * bundles[bundle.id]
for bundle in self.solver_run.bundles for bundle in self.solver_run.bundles
] + ] + [
[ item.iif(self.solver_run, tif_target.theta) *
item.iif(self.solver_run, tif_target.theta) * items[item.id] items[item.id]
for item in self.solver_run.unbundled_items() for item in self.solver_run.unbundled_items()
] ]) <= tif_target.maximum(
) <= tif_target.maximum(), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%'
for tcc_target in self.solver_run.objective_function.tcc_targets: for tcc_target in self.solver_run.objective_function.tcc_targets:
problem += lpSum( problem += lpSum([
[ bundle.trf(self.solver_run.irt_model, tcc_target.theta)
bundle.trf(self.solver_run.irt_model, tcc_target.theta) * bundles[bundle.id] * bundles[bundle.id]
for bundle in self.solver_run.bundles for bundle in self.solver_run.bundles
] + ] + [
[ item.irf(self.solver_run, tcc_target.theta) *
item.irf(self.solver_run, tcc_target.theta) * items[item.id] items[item.id]
for item in self.solver_run.unbundled_items() for item in self.solver_run.unbundled_items()
] ]) >= tcc_target.minimum(
) >= tcc_target.minimum(), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%'
problem += lpSum( problem += lpSum([
[ bundle.trf(self.solver_run.irt_model, tcc_target.theta)
bundle.trf(self.solver_run.irt_model, tcc_target.theta) * bundles[bundle.id] * bundles[bundle.id]
for bundle in self.solver_run.bundles for bundle in self.solver_run.bundles
] + ] + [
[ item.irf(self.solver_run, tcc_target.theta) *
item.irf(self.solver_run, tcc_target.theta) * items[item.id] items[item.id]
for item in self.solver_run.unbundled_items() for item in self.solver_run.unbundled_items()
] ]) <= tcc_target.maximum(
) <= tcc_target.maximum(), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%'
logging.info(f'Solving for Form {form_number} with a drift of {current_drift}%') logging.info(
f'Solving for Form {form_number} with a drift of {current_drift}%'
)
problem.solve() problem.solve()
if LpStatus[problem.status] == 'Infeasible': if LpStatus[problem.status] == 'Infeasible':
logging.info(f'attempt infeasible for drift of {current_drift}%') logging.info(
f'attempt infeasible for drift of {current_drift}%')
if current_drift >= Target.max_drift(): # this is the last attempt, so lets finalize the solution if current_drift >= Target.max_drift(
if ApplicationConfigs.local_dev_env: service_helper.print_problem_variables(problem); ): # this is the last attempt, so lets finalize the solution
if ApplicationConfigs.local_dev_env:
service_helper.print_problem_variables(problem)
logging.info(f'No feasible solution found for Form {form_number}!') logging.info(
f'No feasible solution found for Form {form_number}!'
)
self.add_form_to_solution(problem, solution) self.add_form_to_solution(problem, solution)
@ -178,9 +216,12 @@ class LoftService(Base):
current_drift += Target.max_drift_increment() current_drift += Target.max_drift_increment()
else: else:
if ApplicationConfigs.local_dev_env: service_helper.print_problem_variables(problem); if ApplicationConfigs.local_dev_env:
service_helper.print_problem_variables(problem)
logging.info(f'Optimal solution found with drift of {current_drift}%!') logging.info(
f'Optimal solution found with drift of {current_drift}%!'
)
self.add_form_to_solution(problem, solution) self.add_form_to_solution(problem, solution)
@ -192,8 +233,10 @@ class LoftService(Base):
def add_form_to_solution(self, problem: LpProblem, solution: Solution): def add_form_to_solution(self, problem: LpProblem, solution: Solution):
# add return items and create as a form # add return items and create as a form
form_items, solver_variables = service_helper.solution_items(problem.variables(), self.solver_run) form_items, solver_variables = service_helper.solution_items(
form = Form.create(form_items, self.solver_run, LpStatus[problem.status], solver_variables) problem.variables(), self.solver_run)
form = Form.create(form_items, self.solver_run,
LpStatus[problem.status], solver_variables)
solution.forms.append(form) solution.forms.append(form)
@ -207,7 +250,8 @@ class LoftService(Base):
if error: if error:
logging.info('Streaming %s error response to s3 bucket - %s', logging.info('Streaming %s error response to s3 bucket - %s',
self.file_name, ApplicationConfigs.s3_processed_bucket) self.file_name,
ApplicationConfigs.s3_processed_bucket)
solution_file = service_helper.error_to_file(buffer, error) solution_file = service_helper.error_to_file(buffer, error)
else: else:
logging.info('Streaming %s to s3 bucket - %s', self.file_name, logging.info('Streaming %s to s3 bucket - %s', self.file_name,
@ -216,5 +260,6 @@ class LoftService(Base):
buffer, self.solver_run.total_form_items, self.solution.forms) buffer, self.solver_run.total_form_items, self.solution.forms)
# upload generated file to s3 and return result # upload generated file to s3 and return result
return aws_helper.file_stream_upload(solution_file, self.file_name, return aws_helper.file_stream_upload(
ApplicationConfigs.s3_processed_bucket) solution_file, self.file_name,
ApplicationConfigs.s3_processed_bucket)