trial for variability
This commit is contained in:
Joshua Burman
parent
1f00e1e1bc
commit
11a5112812
@ -2,6 +2,7 @@ from pydantic import BaseModel
|
||||
from typing import List, Literal, Optional
|
||||
|
||||
import logging
|
||||
import random
|
||||
|
||||
from models.item import Item
|
||||
from models.constraint import Constraint
|
||||
@ -89,7 +90,7 @@ class SolverRun(BaseModel):
|
||||
# temporary compensator for bundle item limits, since we shouldn't be using cases with less than 3 items
|
||||
# ideally this should be in the bundles model as a new attribute to handle "constraints of constraints"
|
||||
logging.info('Removing bundles with items < 3')
|
||||
for k,v in enumerate(self.bundles):
|
||||
for k, v in enumerate(self.bundles):
|
||||
bundle = self.bundles[k]
|
||||
if bundle.count < 3: del self.bundles[k]
|
||||
|
||||
@ -103,4 +104,26 @@ class SolverRun(BaseModel):
|
||||
# since the only bundles are based on passage id currently
|
||||
# in the future when we have more than just passage based bundles
|
||||
# we'll need to develop a more sophisticated way of handling this concern
|
||||
return [item for item in self.items if item.passage_id == None]
|
||||
bundle_constraints = (
|
||||
constraint.reference_attribute for constraint in self.constraints
|
||||
if constraint.reference_attribute.type == 'bundle')
|
||||
|
||||
if len(bundle_constraints) > 0:
|
||||
return [item for item in self.items if item.passage_id == None]
|
||||
else:
|
||||
return self.items
|
||||
|
||||
def select_items_by_percent(self, percent) -> list[Item]:
|
||||
items = self.unbundled_items()
|
||||
total_items = len(items)
|
||||
selected_items_amount = round(total_items - (total_items *
|
||||
(percent / 100)))
|
||||
|
||||
return random.sample(items, selected_items_amount)
|
||||
|
||||
def select_bundles_by_percent(self, percent) -> list[Bundle]:
|
||||
total_bundles = len(self.bundles)
|
||||
selected_bundles_amount = round(total_bundles - (total_bundles *
|
||||
(percent / 100)))
|
||||
|
||||
return random.sample(self.bundles, selected_bundles_amount)
|
||||
|
||||
@ -14,6 +14,7 @@ from models.target import Target
|
||||
|
||||
from services.base import Base
|
||||
|
||||
|
||||
class LoftService(Base):
|
||||
|
||||
def process(self):
|
||||
@ -53,7 +54,8 @@ class LoftService(Base):
|
||||
items_csv_reader = csv_helper.file_stream_reader(items_csv)
|
||||
|
||||
# add items to solver run
|
||||
solver_run.items = service_helper.csv_to_item(items_csv_reader, solver_run)
|
||||
solver_run.items = service_helper.csv_to_item(items_csv_reader,
|
||||
solver_run)
|
||||
|
||||
logging.info('Processed Attributes...')
|
||||
|
||||
@ -62,115 +64,151 @@ class LoftService(Base):
|
||||
def generate_solution(self) -> Solution:
|
||||
logging.info('Generating Solution...')
|
||||
|
||||
solution = Solution(response_id=random.randint(100, 5000), forms=[]) # unsolved solution
|
||||
|
||||
# setup common Solver variables
|
||||
items = LpVariable.dicts("Item", [item.id for item in self.solver_run.unbundled_items()], lowBound=0, upBound=1, cat='Binary')
|
||||
bundles = LpVariable.dicts("Bundle", [bundle.id for bundle in self.solver_run.bundles], lowBound=0, upBound=1, cat='Binary')
|
||||
solution = Solution(response_id=random.randint(100, 5000),
|
||||
forms=[]) # unsolved solution
|
||||
|
||||
# iterate for number of forms that require creation
|
||||
for form_count in range(self.solver_run.total_forms):
|
||||
form_number = form_count + 1
|
||||
current_drift = 0 # FF Tokyo Drift
|
||||
form_number = form_count + 1
|
||||
current_drift = 0 # FF Tokyo Drift
|
||||
|
||||
selected_items = []
|
||||
selected_bundles = []
|
||||
|
||||
if form_count == 1:
|
||||
selected_items = self.solver_run.unbundled_items()
|
||||
selected_bundles = self.solver_run.bundles
|
||||
else:
|
||||
selected_items = self.solver_run.select_items_by_percent(30)
|
||||
selected_bundles = self.solver_run.select_bundles_by_percent(
|
||||
30)
|
||||
|
||||
# setup common Solver variables
|
||||
items = LpVariable.dicts("Item",
|
||||
[item.id for item in selected_items],
|
||||
lowBound=0,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
bundles = LpVariable.dicts(
|
||||
"Bundle", [bundle.id for bundle in selected_bundles],
|
||||
lowBound=0,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
|
||||
logging.info(f'Generating Solution for Form {form_number}')
|
||||
|
||||
while current_drift <= Target.max_drift():
|
||||
drift_percent = current_drift / 100
|
||||
self.solver_run.objective_function.update_targets_drift(drift_percent)
|
||||
self.solver_run.objective_function.update_targets_drift(
|
||||
drift_percent)
|
||||
|
||||
# create problem
|
||||
problem = LpProblem('ata-form-generate', LpMinimize)
|
||||
|
||||
# objective function
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.count * bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[
|
||||
items[item.id] for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
)
|
||||
problem += lpSum([
|
||||
bundle.count * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
])
|
||||
|
||||
# Form Constraints
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.count * bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[
|
||||
1 * items[item.id] for item in self.solver_run.unbundled_items()
|
||||
bundle.count * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
1 * items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
) == self.solver_run.total_form_items, f'Total bundle form items for form {form_number}'
|
||||
|
||||
# Dynamic constraints.. currently we only support Metadata and Bundles(Cases/Passages)
|
||||
problem = solver_helper.build_constraints(self.solver_run, problem, items, bundles)
|
||||
problem = solver_helper.build_constraints(
|
||||
self.solver_run, problem, items, bundles)
|
||||
|
||||
# form uniqueness constraints
|
||||
for form in solution.forms:
|
||||
form_item_options = [
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
items[item.id] for item in self.solver_run.unbundled_items()
|
||||
items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
problem += len(set(form.solver_variables)&set(form_item_options)) / float(len(set(form.solver_variables) | set(form_item_options))) * 100 >= 10
|
||||
problem += len(
|
||||
set(form.solver_variables)
|
||||
& set(form_item_options)) / float(
|
||||
len(
|
||||
set(form.solver_variables)
|
||||
| set(form_item_options))) * 100 >= 10
|
||||
|
||||
logging.info('Creating TIF and TCC Elastic constraints')
|
||||
|
||||
# Behold our very own Elastic constraints!
|
||||
for tif_target in self.solver_run.objective_function.tif_targets:
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.tif(self.solver_run.irt_model, tif_target.theta) * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[
|
||||
item.iif(self.solver_run, tif_target.theta) * items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
) >= tif_target.minimum(), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%'
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.tif(self.solver_run.irt_model, tif_target.theta) * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[
|
||||
item.iif(self.solver_run, tif_target.theta) * items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
) <= tif_target.maximum(), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%'
|
||||
problem += lpSum([
|
||||
bundle.tif(self.solver_run.irt_model, tif_target.theta)
|
||||
* bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.iif(self.solver_run, tif_target.theta) *
|
||||
items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]) >= tif_target.minimum(
|
||||
), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%'
|
||||
problem += lpSum([
|
||||
bundle.tif(self.solver_run.irt_model, tif_target.theta)
|
||||
* bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.iif(self.solver_run, tif_target.theta) *
|
||||
items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]) <= tif_target.maximum(
|
||||
), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%'
|
||||
|
||||
for tcc_target in self.solver_run.objective_function.tcc_targets:
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.trf(self.solver_run.irt_model, tcc_target.theta) * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[
|
||||
item.irf(self.solver_run, tcc_target.theta) * items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
) >= tcc_target.minimum(), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%'
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.trf(self.solver_run.irt_model, tcc_target.theta) * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[
|
||||
item.irf(self.solver_run, tcc_target.theta) * items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
) <= tcc_target.maximum(), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%'
|
||||
problem += lpSum([
|
||||
bundle.trf(self.solver_run.irt_model, tcc_target.theta)
|
||||
* bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.irf(self.solver_run, tcc_target.theta) *
|
||||
items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]) >= tcc_target.minimum(
|
||||
), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%'
|
||||
problem += lpSum([
|
||||
bundle.trf(self.solver_run.irt_model, tcc_target.theta)
|
||||
* bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.irf(self.solver_run, tcc_target.theta) *
|
||||
items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]) <= tcc_target.maximum(
|
||||
), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%'
|
||||
|
||||
logging.info(f'Solving for Form {form_number} with a drift of {current_drift}%')
|
||||
logging.info(
|
||||
f'Solving for Form {form_number} with a drift of {current_drift}%'
|
||||
)
|
||||
problem.solve()
|
||||
|
||||
if LpStatus[problem.status] == 'Infeasible':
|
||||
logging.info(f'attempt infeasible for drift of {current_drift}%')
|
||||
logging.info(
|
||||
f'attempt infeasible for drift of {current_drift}%')
|
||||
|
||||
if current_drift >= Target.max_drift(): # this is the last attempt, so lets finalize the solution
|
||||
if ApplicationConfigs.local_dev_env: service_helper.print_problem_variables(problem);
|
||||
if current_drift >= Target.max_drift(
|
||||
): # this is the last attempt, so lets finalize the solution
|
||||
if ApplicationConfigs.local_dev_env:
|
||||
service_helper.print_problem_variables(problem)
|
||||
|
||||
logging.info(f'No feasible solution found for Form {form_number}!')
|
||||
logging.info(
|
||||
f'No feasible solution found for Form {form_number}!'
|
||||
)
|
||||
|
||||
self.add_form_to_solution(problem, solution)
|
||||
|
||||
@ -178,9 +216,12 @@ class LoftService(Base):
|
||||
|
||||
current_drift += Target.max_drift_increment()
|
||||
else:
|
||||
if ApplicationConfigs.local_dev_env: service_helper.print_problem_variables(problem);
|
||||
if ApplicationConfigs.local_dev_env:
|
||||
service_helper.print_problem_variables(problem)
|
||||
|
||||
logging.info(f'Optimal solution found with drift of {current_drift}%!')
|
||||
logging.info(
|
||||
f'Optimal solution found with drift of {current_drift}%!'
|
||||
)
|
||||
|
||||
self.add_form_to_solution(problem, solution)
|
||||
|
||||
@ -192,8 +233,10 @@ class LoftService(Base):
|
||||
|
||||
def add_form_to_solution(self, problem: LpProblem, solution: Solution):
|
||||
# add return items and create as a form
|
||||
form_items, solver_variables = service_helper.solution_items(problem.variables(), self.solver_run)
|
||||
form = Form.create(form_items, self.solver_run, LpStatus[problem.status], solver_variables)
|
||||
form_items, solver_variables = service_helper.solution_items(
|
||||
problem.variables(), self.solver_run)
|
||||
form = Form.create(form_items, self.solver_run,
|
||||
LpStatus[problem.status], solver_variables)
|
||||
|
||||
solution.forms.append(form)
|
||||
|
||||
@ -207,7 +250,8 @@ class LoftService(Base):
|
||||
|
||||
if error:
|
||||
logging.info('Streaming %s error response to s3 bucket - %s',
|
||||
self.file_name, ApplicationConfigs.s3_processed_bucket)
|
||||
self.file_name,
|
||||
ApplicationConfigs.s3_processed_bucket)
|
||||
solution_file = service_helper.error_to_file(buffer, error)
|
||||
else:
|
||||
logging.info('Streaming %s to s3 bucket - %s', self.file_name,
|
||||
@ -216,5 +260,6 @@ class LoftService(Base):
|
||||
buffer, self.solver_run.total_form_items, self.solution.forms)
|
||||
|
||||
# upload generated file to s3 and return result
|
||||
return aws_helper.file_stream_upload(solution_file, self.file_name,
|
||||
ApplicationConfigs.s3_processed_bucket)
|
||||
return aws_helper.file_stream_upload(
|
||||
solution_file, self.file_name,
|
||||
ApplicationConfigs.s3_processed_bucket)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user