From 11979193de7fac3e82a93b849241c8bcf14430ed Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Thu, 9 Nov 2023 16:12:26 -0500 Subject: [PATCH 01/13] refactor in prep for enemies --- app/helpers/solver_helper.py | 53 ++++++++++- app/main.py | 2 +- app/models/form.py | 7 ++ app/models/irt_problem.py | 3 + app/models/problem.py | 79 ++++++++++++++++ app/models/solution.py | 13 +++ app/services/form_generation_service.py | 117 ++---------------------- 7 files changed, 159 insertions(+), 115 deletions(-) create mode 100644 app/models/irt_problem.py create mode 100644 app/models/problem.py diff --git a/app/helpers/solver_helper.py b/app/helpers/solver_helper.py index 1b1c613..ac2f6a5 100644 --- a/app/helpers/solver_helper.py +++ b/app/helpers/solver_helper.py @@ -11,8 +11,9 @@ from models.item import Item from lib.errors.item_generation_error import ItemGenerationError +# should probably be factored out into a bundle class method or a method in the solver run def build_constraints(solver_run: SolverRun, problem: LpProblem, - items: list[Item], bundles: list[Bundle], selected_items: list[Item], selected_bundles: list[Bundle]) -> LpProblem: + items: list[Item], bundles: list[Bundle], selected_items: list[Item], selected_bundles: list[Bundle], current_drift: int) -> LpProblem: logging.info('Creating Constraints...') try: @@ -47,7 +48,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, elif attribute.type == 'bundle': logging.info('Bundles Constraint Generating...') # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates - if selected_bundles != None: + if selected_bundles != None and selected_bundles > 0: # make sure the total bundles used in generated form is limited between min-max set problem += lpSum([ bundles[bundle.id] for bundle in selected_bundles @@ -55,6 +56,52 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, int(constraint.maximum)) logging.info('Constraints Created...') + + # Behold our very own Elastic constraints! + for tif_target in solver_run.objective_function.tif_targets: + problem += lpSum([ + bundle.tif(solver_run.irt_model, tif_target.theta) + * bundles[bundle.id] + for bundle in selected_bundles + ] + [ + item.iif(solver_run, tif_target.theta) * + items[item.id] + for item in selected_items + ]) >= tif_target.minimum( + ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' + problem += lpSum([ + bundle.tif(solver_run.irt_model, tif_target.theta) + * bundles[bundle.id] + for bundle in selected_bundles + ] + [ + item.iif(solver_run, tif_target.theta) * + items[item.id] + for item in selected_items + ]) <= tif_target.maximum( + ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' + + for tcc_target in solver_run.objective_function.tcc_targets: + problem += lpSum([ + bundle.trf(solver_run.irt_model, tcc_target.theta) + * bundles[bundle.id] + for bundle in selected_bundles + ] + [ + item.irf(solver_run, tcc_target.theta) * + items[item.id] + for item in selected_items + ]) >= tcc_target.minimum( + ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + problem += lpSum([ + bundle.trf(solver_run.irt_model, tcc_target.theta) + * bundles[bundle.id] + for bundle in selected_bundles + ] + [ + item.irf(solver_run, tcc_target.theta) * + items[item.id] + for item in selected_items + ]) <= tcc_target.maximum( + ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + return problem except ValueError as error: logging.error(error) @@ -62,7 +109,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, "Bundle min and/or max larger than bundle amount provided", error.args[0]) - +# should probably be factored out into a bundle class method or a method in the solver run def get_random_bundles(total_form_items: int, bundles: list[Bundle], min: int, diff --git a/app/main.py b/app/main.py index 44b0da9..e9f0143 100644 --- a/app/main.py +++ b/app/main.py @@ -43,7 +43,7 @@ class ServiceListener(Consumer): logging.error(f'action of type {action} does not exist.') def main(): - logging.info('Starting IRT Service: That Was Rasch (v1.5.0)...') + logging.info('Starting IRT Service: The Enemies Within (v1.7.0)...') # ToDo: Figure out a much better way of doing this. # LocalStack wants 'endpoint_url', while prod doesnt :( diff --git a/app/models/form.py b/app/models/form.py index e2842f2..242acdc 100644 --- a/app/models/form.py +++ b/app/models/form.py @@ -29,3 +29,10 @@ class Form(BaseModel): tcc_results=irt_helper.generate_tcc_results(items, solver_run), status=status, solver_variables=solver_variables) + + def has_item(self, item: Item) -> bool: + for i in self.items: + if item == i: + return True + + return False diff --git a/app/models/irt_problem.py b/app/models/irt_problem.py new file mode 100644 index 0000000..8f7897f --- /dev/null +++ b/app/models/irt_problem.py @@ -0,0 +1,3 @@ +from models.problem import Problem + +class IrtProblem(Problem): \ No newline at end of file diff --git a/app/models/problem.py b/app/models/problem.py new file mode 100644 index 0000000..e03ab53 --- /dev/null +++ b/app/models/problem.py @@ -0,0 +1,79 @@ +from pydantic import BaseModel +from typing import Any, List +from pulp import LpProblem, LpVariable, lpSum + +import logging, math + +from helpers import solver_helper + +from models.solver_run import SolverRun +from models.solution import Solution +from models.item import Item +from models.bundle import Bundle + +class Problem(BaseModel): + items: List[Item] + bundles: List[Bundle] + problem: Any + solver_items_var: Any = None + solver_bundles_var: Any = None + + def __init__(self, **data) -> None: + super().__init__(**data) + + # setup common Solver variables + self.solver_items_var = LpVariable.dicts("Item", + [item.id for item in self.items], + lowBound=0, + upBound=1, + cat='Binary') + self.solver_bundles_var = LpVariable.dicts("Bundle", + [bundle.id for bundle in self.bundles], + lowBound=0, + upBound=1, + cat='Binary') + + # objective function + self.problem += lpSum([ + bundle.count * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + self.solver_items_var[item.id] + for item in self.items + ]) + + def solve(self) -> LpProblem: + self.problem.solve() + return self.problem + + def generate(self, solution: Solution, solver_run: SolverRun): + + # Form Constraints + self.problem += lpSum( + [ + bundle.count * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + 1 * self.solver_items_var[item.id] + for item in self.items + ] + ) == solver_run.total_form_items, f'Total bundle form items for form' + + # each time a form is generated, we want to ensure + # that it is unique to all other forms generated before it + self.problem += lpSum( + [ + solution.items_exist_in_forms(bundle.items) * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + solution.items_exist_in_forms([item]) * self.solver_items_var[item.id] + for item in self.items + ] + ) <= solver_run.total_form_items - 1, f'Ensuring uniqueness for form' + + def generate_constraints(self, solver_run: SolverRun, current_drift: int): + self.problem = solver_helper.build_constraints( + solver_run, self.problem, self.solver_items_var, self.solver_bundles_var, self.items, self.bundles, current_drift) + + + diff --git a/app/models/solution.py b/app/models/solution.py index 0f7c2ea..47fbd25 100644 --- a/app/models/solution.py +++ b/app/models/solution.py @@ -2,8 +2,21 @@ from pydantic import BaseModel from typing import List from models.form import Form +from models.item import Item class Solution(BaseModel): response_id: int forms: List[Form] + + def items_exist_in_forms(self, items: [Item]) -> bool: + items_found = 0 + + for item in items: + for form in self.forms: + if form.has_item(item): + items_found += 1 + + return items_found + + diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index bfb191e..6b3080a 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -8,6 +8,7 @@ from lib.errors.item_generation_error import ItemGenerationError from models.solver_run import SolverRun from models.solution import Solution +from models.problem import Problem from models.form import Form from models.target import Target @@ -71,24 +72,6 @@ class FormGenerationService(Base): form_number = form_count + 1 current_drift = 0 # FF Tokyo Drift - # adding an element of randomness to the items and bundles used - # may need to change impl based on limit of items available - selected_items = self.solver_run.select_items_by_percent(30) - selected_bundles = self.solver_run.select_bundles_by_percent( - 30) - - # setup common Solver variables - items = LpVariable.dicts("Item", - [item.id for item in selected_items], - lowBound=0, - upBound=1, - cat='Binary') - bundles = LpVariable.dicts("Bundle", - [bundle.id for bundle in selected_bundles], - lowBound=0, - upBound=1, - cat='Binary') - logging.info(f'Generating Solution for Form {form_number} using the {self.solver_run.irt_model.model} IRT model') while current_drift <= Target.max_drift(): @@ -97,99 +80,11 @@ class FormGenerationService(Base): drift_percent) # create problem - problem = LpProblem('ata-form-generate', LpMinimize) - - # objective function - problem += lpSum([ - bundle.count * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - items[item.id] - for item in selected_items - ]) - - # Form Constraints - problem += lpSum( - [ - bundle.count * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - 1 * items[item.id] - for item in selected_items - ] - ) == self.solver_run.total_form_items, f'Total bundle form items for form {form_number}' - - # Dynamic constraints.. currently we only support Metadata and Bundles(Cases/Passages) - problem = solver_helper.build_constraints( - self.solver_run, problem, items, bundles, selected_items, selected_bundles) - - # form uniqueness constraints - # for form in solution.forms: - # form_item_options = [ - # bundles[bundle.id] - # for bundle in selected_bundles - # ] + [ - # items[item.id] - # for item in selected_items - # ] - # problem += len( - # set(form.solver_variables) - # & set(form_item_options)) / float( - # len( - # set(form.solver_variables) - # | set(form_item_options))) * 100 >= 10 - - logging.info('Creating TIF and TCC Elastic constraints') - - # Behold our very own Elastic constraints! - for tif_target in self.solver_run.objective_function.tif_targets: - problem += lpSum([ - bundle.tif(self.solver_run.irt_model, tif_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.iif(self.solver_run, tif_target.theta) * - items[item.id] - for item in selected_items - ]) >= tif_target.minimum( - ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - problem += lpSum([ - bundle.tif(self.solver_run.irt_model, tif_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.iif(self.solver_run, tif_target.theta) * - items[item.id] - for item in selected_items - ]) <= tif_target.maximum( - ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - - for tcc_target in self.solver_run.objective_function.tcc_targets: - problem += lpSum([ - bundle.trf(self.solver_run.irt_model, tcc_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.irf(self.solver_run, tcc_target.theta) * - items[item.id] - for item in selected_items - ]) >= tcc_target.minimum( - ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - problem += lpSum([ - bundle.trf(self.solver_run.irt_model, tcc_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.irf(self.solver_run, tcc_target.theta) * - items[item.id] - for item in selected_items - ]) <= tcc_target.maximum( - ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - - logging.info( - f'Solving for Form {form_number} with a drift of {current_drift}%' - ) - problem.solve() + problem_handler = Problem(items = self.solver_run.items, bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) + problem_handler.generate(solution, self.solver_run) + problem_handler.generate_constraints(self.solver_run, current_drift) + + problem = problem_handler.solve() if LpStatus[problem.status] == 'Infeasible': logging.info( From bbe82daffd69d2af21fbb08c3e7a98d10b8604c5 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Thu, 9 Nov 2023 18:13:35 -0500 Subject: [PATCH 02/13] remove file --- app/models/irt_problem.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 app/models/irt_problem.py diff --git a/app/models/irt_problem.py b/app/models/irt_problem.py deleted file mode 100644 index 8f7897f..0000000 --- a/app/models/irt_problem.py +++ /dev/null @@ -1,3 +0,0 @@ -from models.problem import Problem - -class IrtProblem(Problem): \ No newline at end of file From 6d3639a0c10152d0e280408c9306124807b98594 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Fri, 10 Nov 2023 15:21:16 -0500 Subject: [PATCH 03/13] moved constraints to objects --- app/helpers/solver_helper.py | 6 +- app/models/attribute.py | 1 - app/models/bundle_constraint.py | 20 ++++++ app/models/constraint.py | 5 +- app/models/form.py | 11 +++- app/models/irt_target_constraint.py | 51 +++++++++++++++ app/models/item.py | 13 ++-- app/models/metadata_constraint.py | 34 ++++++++++ app/models/problem.py | 87 +++++++++++++++++++++---- app/models/solution.py | 7 +- app/models/solver_run.py | 30 ++++++++- app/services/form_generation_service.py | 5 +- 12 files changed, 232 insertions(+), 38 deletions(-) create mode 100644 app/models/bundle_constraint.py create mode 100644 app/models/irt_target_constraint.py create mode 100644 app/models/metadata_constraint.py diff --git a/app/helpers/solver_helper.py b/app/helpers/solver_helper.py index ac2f6a5..5ed124a 100644 --- a/app/helpers/solver_helper.py +++ b/app/helpers/solver_helper.py @@ -48,7 +48,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, elif attribute.type == 'bundle': logging.info('Bundles Constraint Generating...') # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates - if selected_bundles != None and selected_bundles > 0: + if selected_bundles != None and len(selected_bundles) > 0: # make sure the total bundles used in generated form is limited between min-max set problem += lpSum([ bundles[bundle.id] for bundle in selected_bundles @@ -56,7 +56,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, int(constraint.maximum)) logging.info('Constraints Created...') - + # Behold our very own Elastic constraints! for tif_target in solver_run.objective_function.tif_targets: problem += lpSum([ @@ -101,7 +101,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, for item in selected_items ]) <= tcc_target.maximum( ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - + return problem except ValueError as error: logging.error(error) diff --git a/app/models/attribute.py b/app/models/attribute.py index 73be39a..171b5c2 100644 --- a/app/models/attribute.py +++ b/app/models/attribute.py @@ -1,7 +1,6 @@ from pydantic import BaseModel from typing import Optional - class Attribute(BaseModel): value: Optional[str] type: Optional[str] diff --git a/app/models/bundle_constraint.py b/app/models/bundle_constraint.py new file mode 100644 index 0000000..0b34dc8 --- /dev/null +++ b/app/models/bundle_constraint.py @@ -0,0 +1,20 @@ +import logging + +from random import randint + +from pulp import lpSum + +from models.constraint import Constraint +from models.problem import Problem + +class BundleConstraint(Constraint): + def build(self, problem_handler: Problem, _) -> None: + logging.info('Bundles Constraint Generating...') + + # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates + if problem_handler.bundles != None and len(problem_handler.bundles) > 0: + # make sure the total bundles used in generated form is limited between min-max set + problem_handler.problem += lpSum([ + problem_handler.solver_bundles_var[bundle.id] for bundle in problem_handler.bundles + ]) == randint(int(self.minimum), + int(self.maximum)), f'Allowing min - max bundles' diff --git a/app/models/constraint.py b/app/models/constraint.py index ba9286c..c60f80b 100644 --- a/app/models/constraint.py +++ b/app/models/constraint.py @@ -1,9 +1,12 @@ from pydantic import BaseModel +from helpers.common_helper import * from models.attribute import Attribute - class Constraint(BaseModel): reference_attribute: Attribute minimum: float maximum: float + + def __init__(self, **data) -> None: + super().__init__(**data) diff --git a/app/models/form.py b/app/models/form.py index 242acdc..140c840 100644 --- a/app/models/form.py +++ b/app/models/form.py @@ -1,14 +1,19 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + from pydantic import BaseModel from typing import List, TypeVar, Type from helpers import irt_helper -from models.solver_run import SolverRun from models.item import Item from models.target import Target from lib.irt.test_response_function import TestResponseFunction +if TYPE_CHECKING: + from models.solver_run import SolverRun + _T = TypeVar("_T") class Form(BaseModel): @@ -29,10 +34,10 @@ class Form(BaseModel): tcc_results=irt_helper.generate_tcc_results(items, solver_run), status=status, solver_variables=solver_variables) - + def has_item(self, item: Item) -> bool: for i in self.items: if item == i: return True - + return False diff --git a/app/models/irt_target_constraint.py b/app/models/irt_target_constraint.py new file mode 100644 index 0000000..c83a2a5 --- /dev/null +++ b/app/models/irt_target_constraint.py @@ -0,0 +1,51 @@ +from __future__ import annotations +from typing import TYPE_CHECKING, Optional + +from pulp import lpSum + +from models import Constraint, Problem, Attribute, Target, Item, Bundle + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class IrtTargetConstraint(Constraint): + reference_attribute: Optional[Attribute] + minimum: Optional[float] + maximum: Optional[float] + target: Target + target_type: str + + def build(self, problem_handler: Problem, solver_run: SolverRun): + problem_handler.problem += lpSum([ + self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + self.item_irt_function(item, solver_run.irt_model, self.target.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) >= self.target.minimum( + ), f'Min {self.target_type} theta({self.target.theta}) at target {self.target.value}' + + problem_handler.problem += lpSum([ + self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + self.item_irt_function(item, solver_run.irt_model, self.target.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) <= self.target.maximum( + ), f'Max {self.target_type} theta({self.target.theta}) at target {self.target.value}' + + def item_irt_function(self, item: Item, irt_model: str, theta: float) -> float: + if self.target_type == 'tcc': + return item.irf(irt_model, theta) + elif self.target_type == 'tif': + return item.iif(irt_model, theta) + + def bundle_irt_function(self, bundle: Bundle, irt_model: str, theta: float) -> float: + if self.target_type == 'tcc': + return bundle.trf(irt_model, theta) + elif self.target_type == 'tif': + return bundle.tif(irt_model, theta) \ No newline at end of file diff --git a/app/models/item.py b/app/models/item.py index 8869a6d..f084b78 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -10,16 +10,17 @@ class Item(BaseModel): id: int position: Optional[int] = None passage_id: Optional[int] = None + enemies: Optional[int] = None workflow_state: Optional[str] = None attributes: List[Attribute] = None b_param: float = 0.00 response: Optional[int] = None - def iif(self, solver_run, theta): - return ItemInformationFunction(solver_run.irt_model).calculate(b_param=self.b_param, theta=theta) + def iif(self, irt_model, theta): + return ItemInformationFunction(irt_model).calculate(b_param=self.b_param, theta=theta) - def irf(self, solver_run, theta): - return ItemResponseFunction(solver_run.irt_model).calculate(b_param=self.b_param, theta=theta) + def irf(self, irt_model, theta): + return ItemResponseFunction(irt_model).calculate(b_param=self.b_param, theta=theta) def get_attribute(self, ref_attribute: Attribute) -> Attribute or None: for attribute in self.attributes: @@ -42,7 +43,7 @@ class Item(BaseModel): total = 0 for target in solver_run.objective_function.tif_targets: - total += self.iif(solver_run, target.theta) + total += self.iif(solver_run.irt_model, target.theta) return total @@ -50,6 +51,6 @@ class Item(BaseModel): total = 0 for target in solver_run.objective_function.tif_targets: - total += self.irf(solver_run, target.theta) + total += self.irf(solver_run.irt_model, target.theta) return total diff --git a/app/models/metadata_constraint.py b/app/models/metadata_constraint.py new file mode 100644 index 0000000..d9221c9 --- /dev/null +++ b/app/models/metadata_constraint.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +import logging + +from pulp import lpSum + +from models.constraint import Constraint +from models.problem import Problem + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class MetadataConstraint(Constraint): + def build(self, problem_handler: Problem, solver_run: SolverRun) -> None: + logging.info('Metadata Constraint Generating...') + + problem_handler.problem += lpSum( + [ + len(bundle.items_with_attribute(self.reference_attribute)) * problem_handler.solver_bundles_var[bundle.id] for bundle in problem_handler.bundles + ] + + [ + item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items + ] + ) >= round(solver_run.total_form_items * (self.minimum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - min' + + problem_handler.problem += lpSum( + [ + len(bundle.items_with_attribute(self.reference_attribute)) * problem_handler.solver_bundles_var[bundle.id] for bundle in problem_handler.bundles + ] + + [ + item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items + ] + ) <= round(solver_run.total_form_items * (self.maximum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - max' diff --git a/app/models/problem.py b/app/models/problem.py index e03ab53..1fc2923 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -1,26 +1,31 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + from pydantic import BaseModel from typing import Any, List from pulp import LpProblem, LpVariable, lpSum -import logging, math +import logging -from helpers import solver_helper - -from models.solver_run import SolverRun from models.solution import Solution from models.item import Item from models.bundle import Bundle +from lib.errors.item_generation_error import ItemGenerationError + +if TYPE_CHECKING: + from models.solver_run import SolverRun + class Problem(BaseModel): items: List[Item] bundles: List[Bundle] problem: Any solver_items_var: Any = None solver_bundles_var: Any = None - + def __init__(self, **data) -> None: super().__init__(**data) - + # setup common Solver variables self.solver_items_var = LpVariable.dicts("Item", [item.id for item in self.items], @@ -32,7 +37,7 @@ class Problem(BaseModel): lowBound=0, upBound=1, cat='Binary') - + # objective function self.problem += lpSum([ bundle.count * self.solver_bundles_var[bundle.id] @@ -41,13 +46,13 @@ class Problem(BaseModel): self.solver_items_var[item.id] for item in self.items ]) - + def solve(self) -> LpProblem: self.problem.solve() return self.problem - + def generate(self, solution: Solution, solver_run: SolverRun): - + # Form Constraints self.problem += lpSum( [ @@ -58,7 +63,7 @@ class Problem(BaseModel): for item in self.items ] ) == solver_run.total_form_items, f'Total bundle form items for form' - + # each time a form is generated, we want to ensure # that it is unique to all other forms generated before it self.problem += lpSum( @@ -70,10 +75,64 @@ class Problem(BaseModel): for item in self.items ] ) <= solver_run.total_form_items - 1, f'Ensuring uniqueness for form' - + def generate_constraints(self, solver_run: SolverRun, current_drift: int): - self.problem = solver_helper.build_constraints( - solver_run, self.problem, self.solver_items_var, self.solver_bundles_var, self.items, self.bundles, current_drift) + logging.info('Creating Constraints...') + + try: + for constraint in solver_run.constraints: + constraint.build(self, solver_run) + + for tif_target in solver_run.objective_function.tif_targets: + self.problem += lpSum([ + bundle.tif(solver_run.irt_model, tif_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.iif(solver_run.irt_model, tif_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) >= tif_target.minimum( + ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' + self.problem += lpSum([ + bundle.tif(solver_run.irt_model, tif_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.iif(solver_run.irt_model, tif_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) <= tif_target.maximum( + ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' + + for tcc_target in solver_run.objective_function.tcc_targets: + self.problem += lpSum([ + bundle.trf(solver_run.irt_model, tcc_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.irf(solver_run.irt_model, tcc_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) >= tcc_target.minimum( + ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + self.problem += lpSum([ + bundle.trf(solver_run.irt_model, tcc_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.irf(solver_run.irt_model, tcc_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) <= tcc_target.maximum( + ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + + logging.info('Constraints Created...') + except ValueError as error: + logging.error(error) + raise ItemGenerationError( + "Bundle min and/or max larger than bundle amount provided", + error.args[0]) diff --git a/app/models/solution.py b/app/models/solution.py index 47fbd25..22f3620 100644 --- a/app/models/solution.py +++ b/app/models/solution.py @@ -4,11 +4,10 @@ from typing import List from models.form import Form from models.item import Item - class Solution(BaseModel): response_id: int forms: List[Form] - + def items_exist_in_forms(self, items: [Item]) -> bool: items_found = 0 @@ -16,7 +15,5 @@ class Solution(BaseModel): for form in self.forms: if form.has_item(item): items_found += 1 - + return items_found - - diff --git a/app/models/solver_run.py b/app/models/solver_run.py index 79eb9ed..439fb81 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -1,22 +1,24 @@ from pydantic import BaseModel -from typing import List, Literal, Optional +from typing import List, Literal, Optional, Union import logging import random from models.item import Item from models.constraint import Constraint +from models.metadata_constraint import MetadataConstraint +from models.bundle_constraint import BundleConstraint +# from models.irt_target_constraint import IrtTargetConstraint from models.irt_model import IRTModel from models.bundle import Bundle from models.objective_function import ObjectiveFunction from models.advanced_options import AdvancedOptions - class SolverRun(BaseModel): items: List[Item] = [] bundles: List[Bundle] = [] bundle_first_ordering: bool = True - constraints: List[Constraint] + constraints: List[Union[Constraint, MetadataConstraint, BundleConstraint]] irt_model: IRTModel objective_function: ObjectiveFunction total_form_items: int @@ -26,6 +28,28 @@ class SolverRun(BaseModel): advanced_options: Optional[AdvancedOptions] engine: str + def __init__(self, **data) -> None: + super().__init__(**data) + + # this is all a compensator for dynamically creating objects + # ideally we'd change the payload to determine what type it is + constraints: [Constraint|MetadataConstraint|BundleConstraint] = [] + + for constraint in self.constraints: + if constraint.reference_attribute.type == 'metadata': + constraints.append(MetadataConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) + elif constraint.reference_attribute.type == 'bundle': + constraints.append(BundleConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) + + # constraints for tif and tcc targets + # for target in self.objective_function.tif_targets: + # constraints.append(IrtTargetConstraint(target=target, target_type='tif')) + + # for target in self.objective_function.tcc_targets: + # constraints.append(IrtTargetConstraint(target=target, target_type='tcc')) + + self.constraints = constraints + def get_item(self, item_id: int) -> Item or None: for item in self.items: if item.id == item_id: diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index 6b3080a..bdf813f 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -80,11 +80,12 @@ class FormGenerationService(Base): drift_percent) # create problem - problem_handler = Problem(items = self.solver_run.items, bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) + problem_handler = Problem(items = self.solver_run.unbundled_items(), bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) problem_handler.generate(solution, self.solver_run) problem_handler.generate_constraints(self.solver_run, current_drift) - + problem = problem_handler.solve() + logging.info(problem) if LpStatus[problem.status] == 'Infeasible': logging.info( From 1dbf6723835227b30cbda03e127e5fc7222b43e9 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Fri, 10 Nov 2023 18:39:16 -0500 Subject: [PATCH 04/13] refactor create constraints for targets --- app/helpers/aws_helper.py | 1 - app/helpers/solver_helper.py | 133 ------------------------ app/models/bundle_constraint.py | 7 +- app/models/constraint.py | 8 +- app/models/irt_target_constraint.py | 51 --------- app/models/metadata_constraint.py | 7 +- app/models/objective_function.py | 13 ++- app/models/problem.py | 51 +-------- app/models/solver_run.py | 8 -- app/models/target.py | 9 ++ app/models/tcc_target.py | 31 ++++++ app/models/tif_target.py | 31 ++++++ app/services/form_generation_service.py | 2 +- 13 files changed, 92 insertions(+), 260 deletions(-) delete mode 100644 app/helpers/solver_helper.py delete mode 100644 app/models/irt_target_constraint.py create mode 100644 app/models/tcc_target.py create mode 100644 app/models/tif_target.py diff --git a/app/helpers/aws_helper.py b/app/helpers/aws_helper.py index 38eb4fe..c3f9a6b 100644 --- a/app/helpers/aws_helper.py +++ b/app/helpers/aws_helper.py @@ -37,6 +37,5 @@ def get_object_tags(key: str, bucket: str) -> list: tags = s3.get_object_tagging(Bucket=bucket, Key=key)['TagSet'] return tags - def file_stream_upload(buffer: io.BytesIO, name: str, bucket: str, action: str = None): return s3.upload_fileobj(buffer, bucket, name, ExtraArgs={'Tagging': f'action={action}'}) diff --git a/app/helpers/solver_helper.py b/app/helpers/solver_helper.py deleted file mode 100644 index 5ed124a..0000000 --- a/app/helpers/solver_helper.py +++ /dev/null @@ -1,133 +0,0 @@ -from pulp import lpSum, LpProblem -from random import randint, sample - -import logging - -from helpers.common_helper import * - -from models.bundle import Bundle -from models.solver_run import SolverRun -from models.item import Item - -from lib.errors.item_generation_error import ItemGenerationError - -# should probably be factored out into a bundle class method or a method in the solver run -def build_constraints(solver_run: SolverRun, problem: LpProblem, - items: list[Item], bundles: list[Bundle], selected_items: list[Item], selected_bundles: list[Bundle], current_drift: int) -> LpProblem: - logging.info('Creating Constraints...') - - try: - total_form_items = solver_run.total_form_items - constraints = solver_run.constraints - - for constraint in constraints: - attribute = constraint.reference_attribute - min = constraint.minimum - max = constraint.maximum - - if attribute.type == 'metadata': - logging.info('Metadata Constraint Generating...') - - problem += lpSum( - [ - len(bundle.items_with_attribute(attribute)) * bundles[bundle.id] for bundle in selected_bundles - ] + - [ - item.attribute_exists(attribute).real * items[item.id] for item in selected_items - ] - ) >= round(total_form_items * (min / 100)), f'{attribute.id} - {attribute.value} - min' - - problem += lpSum( - [ - len(bundle.items_with_attribute(attribute)) * bundles[bundle.id] for bundle in selected_bundles - ] + - [ - item.attribute_exists(attribute).real * items[item.id] for item in selected_items - ] - ) <= round(total_form_items * (max / 100)), f'{attribute.id} - {attribute.value} - max' - elif attribute.type == 'bundle': - logging.info('Bundles Constraint Generating...') - # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates - if selected_bundles != None and len(selected_bundles) > 0: - # make sure the total bundles used in generated form is limited between min-max set - problem += lpSum([ - bundles[bundle.id] for bundle in selected_bundles - ]) == randint(int(constraint.minimum), - int(constraint.maximum)) - - logging.info('Constraints Created...') - - # Behold our very own Elastic constraints! - for tif_target in solver_run.objective_function.tif_targets: - problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.iif(solver_run, tif_target.theta) * - items[item.id] - for item in selected_items - ]) >= tif_target.minimum( - ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.iif(solver_run, tif_target.theta) * - items[item.id] - for item in selected_items - ]) <= tif_target.maximum( - ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - - for tcc_target in solver_run.objective_function.tcc_targets: - problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.irf(solver_run, tcc_target.theta) * - items[item.id] - for item in selected_items - ]) >= tcc_target.minimum( - ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.irf(solver_run, tcc_target.theta) * - items[item.id] - for item in selected_items - ]) <= tcc_target.maximum( - ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - - return problem - except ValueError as error: - logging.error(error) - raise ItemGenerationError( - "Bundle min and/or max larger than bundle amount provided", - error.args[0]) - -# should probably be factored out into a bundle class method or a method in the solver run -def get_random_bundles(total_form_items: int, - bundles: list[Bundle], - min: int, - max: int, - found_bundles=False) -> list[Bundle]: - selected_bundles = None - total_bundle_items = 0 - total_bundles = randint(min, max) - logging.info(f'Selecting Bundles (total of {total_bundles})...') - - while found_bundles == False: - selected_bundles = sample(bundles, total_bundles) - total_bundle_items = sum(bundle.count for bundle in selected_bundles) - - if total_bundle_items <= total_form_items: - found_bundles = True - - if found_bundles == True: - return selected_bundles - else: - return get_random_bundles(total_form_items, total_bundles - 1, bundles) diff --git a/app/models/bundle_constraint.py b/app/models/bundle_constraint.py index 0b34dc8..5ef0afa 100644 --- a/app/models/bundle_constraint.py +++ b/app/models/bundle_constraint.py @@ -1,11 +1,6 @@ -import logging - from random import randint -from pulp import lpSum - -from models.constraint import Constraint -from models.problem import Problem +from models.constraint import * class BundleConstraint(Constraint): def build(self, problem_handler: Problem, _) -> None: diff --git a/app/models/constraint.py b/app/models/constraint.py index c60f80b..abb06a2 100644 --- a/app/models/constraint.py +++ b/app/models/constraint.py @@ -1,12 +1,14 @@ +import logging + +from pulp import lpSum from pydantic import BaseModel + from helpers.common_helper import * from models.attribute import Attribute +from models.problem import Problem class Constraint(BaseModel): reference_attribute: Attribute minimum: float maximum: float - - def __init__(self, **data) -> None: - super().__init__(**data) diff --git a/app/models/irt_target_constraint.py b/app/models/irt_target_constraint.py deleted file mode 100644 index c83a2a5..0000000 --- a/app/models/irt_target_constraint.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations -from typing import TYPE_CHECKING, Optional - -from pulp import lpSum - -from models import Constraint, Problem, Attribute, Target, Item, Bundle - -if TYPE_CHECKING: - from models.solver_run import SolverRun - -class IrtTargetConstraint(Constraint): - reference_attribute: Optional[Attribute] - minimum: Optional[float] - maximum: Optional[float] - target: Target - target_type: str - - def build(self, problem_handler: Problem, solver_run: SolverRun): - problem_handler.problem += lpSum([ - self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) - * problem_handler.solver_bundles_var[bundle.id] - for bundle in problem_handler.bundles - ] + [ - self.item_irt_function(item, solver_run.irt_model, self.target.theta) * - problem_handler.solver_items_var[item.id] - for item in problem_handler.items - ]) >= self.target.minimum( - ), f'Min {self.target_type} theta({self.target.theta}) at target {self.target.value}' - - problem_handler.problem += lpSum([ - self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) - * problem_handler.solver_bundles_var[bundle.id] - for bundle in problem_handler.bundles - ] + [ - self.item_irt_function(item, solver_run.irt_model, self.target.theta) * - problem_handler.solver_items_var[item.id] - for item in problem_handler.items - ]) <= self.target.maximum( - ), f'Max {self.target_type} theta({self.target.theta}) at target {self.target.value}' - - def item_irt_function(self, item: Item, irt_model: str, theta: float) -> float: - if self.target_type == 'tcc': - return item.irf(irt_model, theta) - elif self.target_type == 'tif': - return item.iif(irt_model, theta) - - def bundle_irt_function(self, bundle: Bundle, irt_model: str, theta: float) -> float: - if self.target_type == 'tcc': - return bundle.trf(irt_model, theta) - elif self.target_type == 'tif': - return bundle.tif(irt_model, theta) \ No newline at end of file diff --git a/app/models/metadata_constraint.py b/app/models/metadata_constraint.py index d9221c9..7284030 100644 --- a/app/models/metadata_constraint.py +++ b/app/models/metadata_constraint.py @@ -1,12 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -import logging - -from pulp import lpSum - -from models.constraint import Constraint -from models.problem import Problem +from models.constraint import * if TYPE_CHECKING: from models.solver_run import SolverRun diff --git a/app/models/objective_function.py b/app/models/objective_function.py index 9bd332f..3941434 100644 --- a/app/models/objective_function.py +++ b/app/models/objective_function.py @@ -1,15 +1,18 @@ +from __future__ import annotations + from pydantic import BaseModel from typing import Dict, List, AnyStr -from models.target import Target +from models.tif_target import TifTarget +from models.tcc_target import TccTarget class ObjectiveFunction(BaseModel): # minimizing tif/tcc target value is only option currently # as we add more we can build this out to be more dynamic # likely with models representing each objective function type - tif_targets: List[Target] - tcc_targets: List[Target] + tif_targets: List[TifTarget] + tcc_targets: List[TccTarget] target_variance_percentage: int = 10 objective: AnyStr = "minimize" weight: Dict = {'tif': 1, 'tcc': 1} @@ -18,7 +21,7 @@ class ObjectiveFunction(BaseModel): limit: float or bool, all: bool = False, amount: float = 0.1, - targets: list[Target] = []) -> bool: + targets: list[TifTarget|TccTarget] = []) -> bool: if all: for target in self.tif_targets: target.drift = round(target.drift + amount, 2) @@ -44,5 +47,5 @@ class ObjectiveFunction(BaseModel): return minimum_drift - def all_targets(self) -> list[Target]: + def all_targets(self) -> list[TifTarget|TccTarget]: return self.tif_targets + self.tcc_targets diff --git a/app/models/problem.py b/app/models/problem.py index 1fc2923..13f5ee0 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -51,9 +51,9 @@ class Problem(BaseModel): self.problem.solve() return self.problem - def generate(self, solution: Solution, solver_run: SolverRun): + def generate(self, solution: Solution, solver_run: SolverRun) -> None: - # Form Constraints + # Objective Function self.problem += lpSum( [ bundle.count * self.solver_bundles_var[bundle.id] @@ -76,56 +76,15 @@ class Problem(BaseModel): ] ) <= solver_run.total_form_items - 1, f'Ensuring uniqueness for form' - def generate_constraints(self, solver_run: SolverRun, current_drift: int): + def generate_constraints(self, solver_run: SolverRun, current_drift: int) -> None: logging.info('Creating Constraints...') try: for constraint in solver_run.constraints: constraint.build(self, solver_run) - - for tif_target in solver_run.objective_function.tif_targets: - self.problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.iif(solver_run.irt_model, tif_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) >= tif_target.minimum( - ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - self.problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.iif(solver_run.irt_model, tif_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) <= tif_target.maximum( - ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - for tcc_target in solver_run.objective_function.tcc_targets: - self.problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.irf(solver_run.irt_model, tcc_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) >= tcc_target.minimum( - ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - self.problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.irf(solver_run.irt_model, tcc_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) <= tcc_target.maximum( - ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + for target in solver_run.objective_function.all_targets(): + target.build_constraint(self, solver_run) logging.info('Constraints Created...') except ValueError as error: diff --git a/app/models/solver_run.py b/app/models/solver_run.py index 439fb81..c64897c 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -8,7 +8,6 @@ from models.item import Item from models.constraint import Constraint from models.metadata_constraint import MetadataConstraint from models.bundle_constraint import BundleConstraint -# from models.irt_target_constraint import IrtTargetConstraint from models.irt_model import IRTModel from models.bundle import Bundle from models.objective_function import ObjectiveFunction @@ -41,13 +40,6 @@ class SolverRun(BaseModel): elif constraint.reference_attribute.type == 'bundle': constraints.append(BundleConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) - # constraints for tif and tcc targets - # for target in self.objective_function.tif_targets: - # constraints.append(IrtTargetConstraint(target=target, target_type='tif')) - - # for target in self.objective_function.tcc_targets: - # constraints.append(IrtTargetConstraint(target=target, target_type='tcc')) - self.constraints = constraints def get_item(self, item_id: int) -> Item or None: diff --git a/app/models/target.py b/app/models/target.py index 1ad9fa4..bf9e1ea 100644 --- a/app/models/target.py +++ b/app/models/target.py @@ -1,6 +1,15 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + from pydantic import BaseModel from typing import Optional +from pulp import lpSum + +if TYPE_CHECKING: + from models.solver_run import SolverRun + from models.problem import Problem + class Target(BaseModel): theta: float value: float diff --git a/app/models/tcc_target.py b/app/models/tcc_target.py new file mode 100644 index 0000000..6c29181 --- /dev/null +++ b/app/models/tcc_target.py @@ -0,0 +1,31 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.target import * + +if TYPE_CHECKING: + from models.problem import Problem + +class TccTarget(Target): + def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + problem_handler.problem += lpSum([ + bundle.trf(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.irf(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) >= self.minimum( + ), f'Min TCC theta({self.theta}) at target {self.value} with a drift % of {self.drift}' + + problem_handler.problem += lpSum([ + bundle.trf(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.irf(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) <= self.maximum( + ), f'Max TCC theta({self.theta}) at target {self.value} with a drift % of {self.drift}' \ No newline at end of file diff --git a/app/models/tif_target.py b/app/models/tif_target.py new file mode 100644 index 0000000..fac10c2 --- /dev/null +++ b/app/models/tif_target.py @@ -0,0 +1,31 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.target import * + +if TYPE_CHECKING: + from models.problem import Problem + +class TifTarget(Target): + def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + problem_handler.problem += lpSum([ + bundle.tif(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.iif(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) >= self.minimum( + ), f'Min TIF theta({self.theta}) at target {self.value} with a drift % of {self.drift}' + + problem_handler.problem += lpSum([ + bundle.tif(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.iif(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) <= self.maximum( + ), f'Max TIF theta({self.theta}) at target {self.value} with a drift % of {self.drift}' \ No newline at end of file diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index bdf813f..a5bcaf8 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -3,7 +3,7 @@ import json, random, io, logging from pulp import LpProblem, LpVariable, LpMinimize, LpStatus, lpSum from lib.application_configs import ApplicationConfigs -from helpers import aws_helper, tar_helper, csv_helper, service_helper, solver_helper +from helpers import aws_helper, tar_helper, csv_helper, service_helper from lib.errors.item_generation_error import ItemGenerationError from models.solver_run import SolverRun From 07af0ac0ac7a8530fd888574c9f9c6933342ccc7 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Fri, 10 Nov 2023 19:12:25 -0500 Subject: [PATCH 05/13] cleanup --- app/models/problem.py | 2 +- app/models/tcc_target.py | 2 +- app/models/tif_target.py | 2 +- app/services/form_generation_service.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/problem.py b/app/models/problem.py index 13f5ee0..e9171cc 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -90,7 +90,7 @@ class Problem(BaseModel): except ValueError as error: logging.error(error) raise ItemGenerationError( - "Bundle min and/or max larger than bundle amount provided", + error.msg, error.args[0]) diff --git a/app/models/tcc_target.py b/app/models/tcc_target.py index 6c29181..8e78e91 100644 --- a/app/models/tcc_target.py +++ b/app/models/tcc_target.py @@ -28,4 +28,4 @@ class TccTarget(Target): problem_handler.solver_items_var[item.id] for item in problem_handler.items ]) <= self.maximum( - ), f'Max TCC theta({self.theta}) at target {self.value} with a drift % of {self.drift}' \ No newline at end of file + ), f'Max TCC theta({self.theta}) at target {self.value} with a drift % of {self.drift}' diff --git a/app/models/tif_target.py b/app/models/tif_target.py index fac10c2..04ed1c6 100644 --- a/app/models/tif_target.py +++ b/app/models/tif_target.py @@ -28,4 +28,4 @@ class TifTarget(Target): problem_handler.solver_items_var[item.id] for item in problem_handler.items ]) <= self.maximum( - ), f'Max TIF theta({self.theta}) at target {self.value} with a drift % of {self.drift}' \ No newline at end of file + ), f'Max TIF theta({self.theta}) at target {self.value} with a drift % of {self.drift}' diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index a5bcaf8..6f63b20 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -85,7 +85,6 @@ class FormGenerationService(Base): problem_handler.generate_constraints(self.solver_run, current_drift) problem = problem_handler.solve() - logging.info(problem) if LpStatus[problem.status] == 'Infeasible': logging.info( @@ -99,6 +98,7 @@ class FormGenerationService(Base): logging.info( f'No feasible solution found for Form {form_number}!' ) + logging.error(problem) self.add_form_to_solution(problem, solution) From f1fa519f31093b6285f0f3ca4ac337eec25cafc7 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Sun, 12 Nov 2023 18:32:48 -0500 Subject: [PATCH 06/13] refactor of model targets and constraints, addition of new constraint types and constraint construction process --- app/helpers/irt_helper.py | 7 +-- app/models/advanced_options.py | 1 + app/models/attribute.py | 4 +- .../{ => constraints}/bundle_constraint.py | 6 +-- .../constraints/form_uniqueness_constraint.py | 34 +++++++++++++ .../generic_constraint.py} | 5 +- .../{ => constraints}/metadata_constraint.py | 10 ++-- .../total_form_items_constraint.py | 33 +++++++++++++ app/models/form.py | 7 +-- app/models/objective_function.py | 16 +++++-- app/models/problem.py | 48 ++++--------------- app/models/solver_run.py | 43 +++++++++-------- app/models/{ => targets}/target.py | 0 app/models/{ => targets}/tcc_target.py | 4 +- app/models/{ => targets}/tif_target.py | 4 +- app/services/form_generation_service.py | 4 +- 16 files changed, 140 insertions(+), 86 deletions(-) rename app/models/{ => constraints}/bundle_constraint.py (80%) create mode 100644 app/models/constraints/form_uniqueness_constraint.py rename app/models/{constraint.py => constraints/generic_constraint.py} (74%) rename app/models/{ => constraints}/metadata_constraint.py (68%) create mode 100644 app/models/constraints/total_form_items_constraint.py rename app/models/{ => targets}/target.py (100%) rename app/models/{ => targets}/tcc_target.py (91%) rename app/models/{ => targets}/tif_target.py (91%) diff --git a/app/helpers/irt_helper.py b/app/helpers/irt_helper.py index 401e875..5a9821e 100644 --- a/app/helpers/irt_helper.py +++ b/app/helpers/irt_helper.py @@ -1,14 +1,15 @@ from lib.irt.test_response_function import TestResponseFunction from lib.irt.test_information_function import TestInformationFunction -from models.target import Target +from models.targets.tif_target import TifTarget +from models.targets.tcc_target import TccTarget def generate_tif_results(items, solver_run): targets = [] for target in solver_run.objective_function.tif_targets: tif = TestInformationFunction(solver_run.irt_model).calculate(items, theta=target.theta) - targets.append(Target(theta=target.theta, value=target.value, result=tif)) + targets.append(TifTarget(theta=target.theta, value=target.value, result=tif)) return targets @@ -17,6 +18,6 @@ def generate_tcc_results(items, solver_run): for target in solver_run.objective_function.tcc_targets: tcc = TestResponseFunction(solver_run.irt_model).calculate(items, theta=target.theta) - targets.append(Target(theta=target.theta, value=target.value, result=tcc)) + targets.append(TccTarget(theta=target.theta, value=target.value, result=tcc)) return targets diff --git a/app/models/advanced_options.py b/app/models/advanced_options.py index c9b7213..3ca1232 100644 --- a/app/models/advanced_options.py +++ b/app/models/advanced_options.py @@ -9,4 +9,5 @@ class AdvancedOptions(BaseModel): brand_bound_tolerance: Optional[float] = None max_forms: Optional[int] = None precision: Optional[float] = None + ensure_form_uniqueness: bool = True extra_param_range: Optional[List[Dict]] = None diff --git a/app/models/attribute.py b/app/models/attribute.py index 171b5c2..35254e0 100644 --- a/app/models/attribute.py +++ b/app/models/attribute.py @@ -1,7 +1,7 @@ from pydantic import BaseModel -from typing import Optional +from typing import Optional, Union class Attribute(BaseModel): - value: Optional[str] + value: Optional[Union[str,int]] type: Optional[str] id: str diff --git a/app/models/bundle_constraint.py b/app/models/constraints/bundle_constraint.py similarity index 80% rename from app/models/bundle_constraint.py rename to app/models/constraints/bundle_constraint.py index 5ef0afa..5809a35 100644 --- a/app/models/bundle_constraint.py +++ b/app/models/constraints/bundle_constraint.py @@ -1,9 +1,9 @@ from random import randint -from models.constraint import * +from models.constraints.generic_constraint import * -class BundleConstraint(Constraint): - def build(self, problem_handler: Problem, _) -> None: +class BundleConstraint(GenericConstraint): + def build(self, problem_handler: Problem, **kwargs) -> None: logging.info('Bundles Constraint Generating...') # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates diff --git a/app/models/constraints/form_uniqueness_constraint.py b/app/models/constraints/form_uniqueness_constraint.py new file mode 100644 index 0000000..fdbb1fb --- /dev/null +++ b/app/models/constraints/form_uniqueness_constraint.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.constraints.generic_constraint import * + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class FormUniquenessConstraint(GenericConstraint): + @classmethod + def create(cls: Type[_T], total_items: int) -> _T: + return cls( + minimum=0, + maximum=0, + reference_attribute=Attribute( + value=total_items, + type='form_uniqueness', + id='form_uniqueness' + ) + ) + + def build(self, problem_handler: Problem, **kwargs) -> None: + logging.info('Form Uniqueness Constraint Generating...') + + # form uniqueness constraint + problem_handler.problem += lpSum( + [ + kwargs['solution'].items_exist_in_forms(bundle.items) * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + kwargs['solution'].items_exist_in_forms([item]) * problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ] + ) <= self.reference_attribute.value, f'Ensuring uniqueness for form' diff --git a/app/models/constraint.py b/app/models/constraints/generic_constraint.py similarity index 74% rename from app/models/constraint.py rename to app/models/constraints/generic_constraint.py index abb06a2..a740b3b 100644 --- a/app/models/constraint.py +++ b/app/models/constraints/generic_constraint.py @@ -2,13 +2,16 @@ import logging from pulp import lpSum from pydantic import BaseModel +from typing import TypeVar, Type from helpers.common_helper import * from models.attribute import Attribute from models.problem import Problem -class Constraint(BaseModel): +_T = TypeVar("_T") + +class GenericConstraint(BaseModel): reference_attribute: Attribute minimum: float maximum: float diff --git a/app/models/metadata_constraint.py b/app/models/constraints/metadata_constraint.py similarity index 68% rename from app/models/metadata_constraint.py rename to app/models/constraints/metadata_constraint.py index 7284030..19fbb98 100644 --- a/app/models/metadata_constraint.py +++ b/app/models/constraints/metadata_constraint.py @@ -1,13 +1,13 @@ from __future__ import annotations from typing import TYPE_CHECKING -from models.constraint import * +from models.constraints.generic_constraint import * if TYPE_CHECKING: from models.solver_run import SolverRun -class MetadataConstraint(Constraint): - def build(self, problem_handler: Problem, solver_run: SolverRun) -> None: +class MetadataConstraint(GenericConstraint): + def build(self, problem_handler: Problem, **kwargs) -> None: logging.info('Metadata Constraint Generating...') problem_handler.problem += lpSum( @@ -17,7 +17,7 @@ class MetadataConstraint(Constraint): [ item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items ] - ) >= round(solver_run.total_form_items * (self.minimum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - min' + ) >= round(kwargs['solver_run'].total_form_items * (self.minimum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - min' problem_handler.problem += lpSum( [ @@ -26,4 +26,4 @@ class MetadataConstraint(Constraint): [ item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items ] - ) <= round(solver_run.total_form_items * (self.maximum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - max' + ) <= round(kwargs['solver_run'].total_form_items * (self.maximum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - max' diff --git a/app/models/constraints/total_form_items_constraint.py b/app/models/constraints/total_form_items_constraint.py new file mode 100644 index 0000000..5b3c64a --- /dev/null +++ b/app/models/constraints/total_form_items_constraint.py @@ -0,0 +1,33 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.constraints.generic_constraint import * + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class TotalFormItemsConstraint(GenericConstraint): + @classmethod + def create(cls: Type[_T], total_items: int) -> _T: + return cls( + minimum=0, + maximum=0, + reference_attribute=Attribute( + value=total_items, + type='form_uniqueness', + id='form_uniqueness' + ) + ) + + def build(self, problem_handler: Problem, **kwargs) -> None: + logging.info('Total Form Items Constraint Generating...') + + problem_handler.problem += lpSum( + [ + bundle.count * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + 1 * problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ] + ) == self.reference_attribute.value, f'Total bundle form items for form' diff --git a/app/models/form.py b/app/models/form.py index 140c840..6a0382e 100644 --- a/app/models/form.py +++ b/app/models/form.py @@ -7,7 +7,8 @@ from typing import List, TypeVar, Type from helpers import irt_helper from models.item import Item -from models.target import Target +from models.targets.tif_target import TifTarget +from models.targets.tcc_target import TccTarget from lib.irt.test_response_function import TestResponseFunction @@ -19,8 +20,8 @@ _T = TypeVar("_T") class Form(BaseModel): items: List[Item] cut_score: float - tif_results: List[Target] - tcc_results: List[Target] + tif_results: List[TifTarget] + tcc_results: List[TccTarget] status: str = 'Not Optimized' solver_variables: List[str] diff --git a/app/models/objective_function.py b/app/models/objective_function.py index 3941434..3a43680 100644 --- a/app/models/objective_function.py +++ b/app/models/objective_function.py @@ -2,10 +2,11 @@ from __future__ import annotations from pydantic import BaseModel from typing import Dict, List, AnyStr +from pulp import lpSum -from models.tif_target import TifTarget -from models.tcc_target import TccTarget - +from models.targets.tif_target import TifTarget +from models.targets.tcc_target import TccTarget +from models.problem import Problem class ObjectiveFunction(BaseModel): # minimizing tif/tcc target value is only option currently @@ -17,6 +18,15 @@ class ObjectiveFunction(BaseModel): objective: AnyStr = "minimize" weight: Dict = {'tif': 1, 'tcc': 1} + def for_problem(self, problem_handler: Problem) -> None: + problem_handler.problem += lpSum([ + bundle.count * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) + def increment_targets_drift(self, limit: float or bool, all: bool = False, diff --git a/app/models/problem.py b/app/models/problem.py index e9171cc..23e0cb4 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -38,53 +38,23 @@ class Problem(BaseModel): upBound=1, cat='Binary') - # objective function - self.problem += lpSum([ - bundle.count * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - self.solver_items_var[item.id] - for item in self.items - ]) - def solve(self) -> LpProblem: self.problem.solve() return self.problem def generate(self, solution: Solution, solver_run: SolverRun) -> None: - - # Objective Function - self.problem += lpSum( - [ - bundle.count * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - 1 * self.solver_items_var[item.id] - for item in self.items - ] - ) == solver_run.total_form_items, f'Total bundle form items for form' - - # each time a form is generated, we want to ensure - # that it is unique to all other forms generated before it - self.problem += lpSum( - [ - solution.items_exist_in_forms(bundle.items) * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - solution.items_exist_in_forms([item]) * self.solver_items_var[item.id] - for item in self.items - ] - ) <= solver_run.total_form_items - 1, f'Ensuring uniqueness for form' - - def generate_constraints(self, solver_run: SolverRun, current_drift: int) -> None: - logging.info('Creating Constraints...') - try: - for constraint in solver_run.constraints: - constraint.build(self, solver_run) + # creating problem objective function + solver_run.objective_function.for_problem(self) + logging.info('Creating Constraints...') + # generic constraints + for constraint in solver_run.constraints: + constraint.build(self, solver_run=solver_run, solution=solution) + + # irt target constraints for target in solver_run.objective_function.all_targets(): - target.build_constraint(self, solver_run) + target.constraints(self, solver_run) logging.info('Constraints Created...') except ValueError as error: diff --git a/app/models/solver_run.py b/app/models/solver_run.py index c64897c..a9a6d00 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -1,23 +1,32 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from pulp import lpSum from pydantic import BaseModel from typing import List, Literal, Optional, Union import logging -import random from models.item import Item -from models.constraint import Constraint -from models.metadata_constraint import MetadataConstraint -from models.bundle_constraint import BundleConstraint +from models.constraints.generic_constraint import GenericConstraint +from models.constraints.metadata_constraint import MetadataConstraint +from models.constraints.bundle_constraint import BundleConstraint +from models.constraints.form_uniqueness_constraint import FormUniquenessConstraint +from models.constraints.total_form_items_constraint import TotalFormItemsConstraint from models.irt_model import IRTModel from models.bundle import Bundle from models.objective_function import ObjectiveFunction from models.advanced_options import AdvancedOptions +if TYPE_CHECKING: + from models.solution import Solution + from models.problem import Problem + class SolverRun(BaseModel): items: List[Item] = [] bundles: List[Bundle] = [] bundle_first_ordering: bool = True - constraints: List[Union[Constraint, MetadataConstraint, BundleConstraint]] + constraints: List[Union[GenericConstraint, MetadataConstraint, BundleConstraint, FormUniquenessConstraint, TotalFormItemsConstraint]] irt_model: IRTModel objective_function: ObjectiveFunction total_form_items: int @@ -32,8 +41,16 @@ class SolverRun(BaseModel): # this is all a compensator for dynamically creating objects # ideally we'd change the payload to determine what type it is - constraints: [Constraint|MetadataConstraint|BundleConstraint] = [] + constraints: [GenericConstraint|MetadataConstraint|BundleConstraint|FormUniquenessConstraint|TotalFormItemsConstraint] = [] + # total form items + constraints.append(TotalFormItemsConstraint.create(self.total_form_items)) + + # ensure form uniqueness + if self.advanced_options.ensure_form_uniqueness: + constraints.append(FormUniquenessConstraint.create(self.total_form_items - 1)) + + # repackage to create appropriate constraint types for constraint in self.constraints: if constraint.reference_attribute.type == 'metadata': constraints.append(MetadataConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) @@ -130,17 +147,3 @@ class SolverRun(BaseModel): else: return self.items - def select_items_by_percent(self, percent: int) -> List[Item]: - items = self.unbundled_items() - total_items = len(items) - selected_items_amount = round(total_items - (total_items * - (percent / 100))) - - return random.sample(items, selected_items_amount) - - def select_bundles_by_percent(self, percent: int) -> List[Bundle]: - total_bundles = len(self.bundles) - selected_bundles_amount = round(total_bundles - (total_bundles * - (percent / 100))) - - return random.sample(self.bundles, selected_bundles_amount) diff --git a/app/models/target.py b/app/models/targets/target.py similarity index 100% rename from app/models/target.py rename to app/models/targets/target.py diff --git a/app/models/tcc_target.py b/app/models/targets/tcc_target.py similarity index 91% rename from app/models/tcc_target.py rename to app/models/targets/tcc_target.py index 8e78e91..f7b2368 100644 --- a/app/models/tcc_target.py +++ b/app/models/targets/tcc_target.py @@ -1,13 +1,13 @@ from __future__ import annotations from typing import TYPE_CHECKING -from models.target import * +from models.targets.target import * if TYPE_CHECKING: from models.problem import Problem class TccTarget(Target): - def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + def constraints(self, problem_handler: Problem, solver_run: SolverRun): problem_handler.problem += lpSum([ bundle.trf(solver_run.irt_model, self.theta) * problem_handler.solver_bundles_var[bundle.id] diff --git a/app/models/tif_target.py b/app/models/targets/tif_target.py similarity index 91% rename from app/models/tif_target.py rename to app/models/targets/tif_target.py index 04ed1c6..388091a 100644 --- a/app/models/tif_target.py +++ b/app/models/targets/tif_target.py @@ -1,13 +1,13 @@ from __future__ import annotations from typing import TYPE_CHECKING -from models.target import * +from models.targets.target import * if TYPE_CHECKING: from models.problem import Problem class TifTarget(Target): - def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + def constraints(self, problem_handler: Problem, solver_run: SolverRun): problem_handler.problem += lpSum([ bundle.tif(solver_run.irt_model, self.theta) * problem_handler.solver_bundles_var[bundle.id] diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index 6f63b20..082b10d 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -10,7 +10,7 @@ from models.solver_run import SolverRun from models.solution import Solution from models.problem import Problem from models.form import Form -from models.target import Target +from models.targets.target import Target from services.base import Base @@ -82,8 +82,6 @@ class FormGenerationService(Base): # create problem problem_handler = Problem(items = self.solver_run.unbundled_items(), bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) problem_handler.generate(solution, self.solver_run) - problem_handler.generate_constraints(self.solver_run, current_drift) - problem = problem_handler.solve() if LpStatus[problem.status] == 'Infeasible': From eb8138eaf9480323e31284b2c80e561b9b960c0d Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Sun, 12 Nov 2023 18:37:29 -0500 Subject: [PATCH 07/13] make sure to add all constraint types to return --- app/models/solver_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/solver_run.py b/app/models/solver_run.py index a9a6d00..77df041 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -69,7 +69,7 @@ class SolverRun(BaseModel): if bundle.id == bundle_id: return bundle - def get_constraint_by_type(self, type: str) -> Constraint or None: + def get_constraint_by_type(self, type: str) -> GenericConstraint|MetadataConstraint|BundleConstraint|FormUniquenessConstraint|TotalFormItemsConstraint or None: for constraint in self.constraints: if type == constraint.reference_attribute.type: return constraint @@ -130,7 +130,7 @@ class SolverRun(BaseModel): logging.info('Bundles Generated...') - def get_constraint(self, name: str) -> Constraint: + def get_constraint(self, name: str) -> GenericConstraint|MetadataConstraint|BundleConstraint|FormUniquenessConstraint|TotalFormItemsConstraint: return next((constraint for constraint in self.constraints if constraint.reference_attribute.id == name), None) From 8667bec8d505c9eb2b56413fefc5e2171b8745ce Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Mon, 13 Nov 2023 16:37:35 -0500 Subject: [PATCH 08/13] solve without enemies, initial impl. still requires factoring out --- app/helpers/irt_helper.py | 3 + app/models/bundle.py | 10 ++++ app/models/item.py | 2 +- app/models/problem.py | 73 +++++++++++++++++++++++-- app/models/solver_run.py | 1 + app/services/form_generation_service.py | 4 +- 6 files changed, 85 insertions(+), 8 deletions(-) diff --git a/app/helpers/irt_helper.py b/app/helpers/irt_helper.py index 5a9821e..19cba93 100644 --- a/app/helpers/irt_helper.py +++ b/app/helpers/irt_helper.py @@ -1,8 +1,11 @@ +from typing import List + from lib.irt.test_response_function import TestResponseFunction from lib.irt.test_information_function import TestInformationFunction from models.targets.tif_target import TifTarget from models.targets.tcc_target import TccTarget +from models.item import Item def generate_tif_results(items, solver_run): targets = [] diff --git a/app/models/bundle.py b/app/models/bundle.py index 475d710..bd778ff 100644 --- a/app/models/bundle.py +++ b/app/models/bundle.py @@ -14,6 +14,16 @@ class Bundle(BaseModel): items: List[Item] type: str + def find_items(self, requested_items_ids: [int]) -> [Item]: + found_items = [] + + for item in self.items: + if item.id in requested_items_ids: + found_items.append(item) + + return found_items + + def tif(self, irt_model: IRTModel, theta: float) -> float: return TestInformationFunction(irt_model).calculate(self.items, theta=theta) diff --git a/app/models/item.py b/app/models/item.py index f084b78..b82dd03 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -10,7 +10,7 @@ class Item(BaseModel): id: int position: Optional[int] = None passage_id: Optional[int] = None - enemies: Optional[int] = None + enemies: List[int] = [] workflow_state: Optional[str] = None attributes: List[Attribute] = None b_param: float = 0.00 diff --git a/app/models/problem.py b/app/models/problem.py index 23e0cb4..5ddd33a 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -3,10 +3,12 @@ from typing import TYPE_CHECKING from pydantic import BaseModel from typing import Any, List -from pulp import LpProblem, LpVariable, lpSum +from pulp import LpProblem, LpVariable, LpStatus, lpSum import logging +from helpers import service_helper, irt_helper + from models.solution import Solution from models.item import Item from models.bundle import Bundle @@ -38,8 +40,71 @@ class Problem(BaseModel): upBound=1, cat='Binary') - def solve(self) -> LpProblem: - self.problem.solve() + def solve(self, solver_run: SolverRun) -> LpProblem: + logging.info('solving problem...') + # if we allow enemies, go through the normal solving process + if solver_run.allow_enemies: + logging.info('enemes allowed, so just solving') + self.problem.solve() + # otherwise begin the process of filtering enemies + else: + self.problem.solve() + + # however, if the solve was infeasible, kick it back + # to the normal process + if LpStatus[self.problem.status] == 'Infeasible': + return self.problem + # otherwise continue + else: + # get items from solution + solved_items, _ = service_helper.solution_items(self.problem.variables(), solver_run) + + sacred_ids = [] + enemy_ids = [] + + # get all enemies + for item in solved_items: + # if it has enemies, check if it exists as part of the solved items + for enemy_id in item.enemies: + # if it does, it's a true enemy + if enemy_id in (item.id for item in solved_items): + enemy_ids.append(enemy_id) + # remove enemy from solved items, + # lest it has this sacred item added to enemies + solved_items = [i for i in solved_items if i.id != enemy_id] + + # the item is cleansed, now it's sacred + sacred_ids.append(item.id) + + if enemy_ids: + logging.info('enemies found, adding constraints...') + + # remove old enemy/sacred constraints + self.problem.constrants.pop('Exclude_enemy_items') + self.problem.constrants.pop('Include_sacred_items') + + # add constraint to not allow enemy items + self.problem += lpSum([ + len(bundle.find_items(enemy_ids)) * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + (item.id in enemy_ids) * self.solver_items_var[item.id] + for item in self.items + ]) == 0, 'Exclude enemy items' + + # add constraint to use sacred items + self.problem += lpSum([ + len(bundle.find_items(sacred_ids)) * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + (item.id in sacred_ids) * self.solver_items_var[item.id] + for item in self.items + ]) == len(sacred_ids), 'Include sacred items' + + # recursively solve until no enemies exist or infeasible + logging.info('recursively solving...') + self.solve(solver_run) + return self.problem def generate(self, solution: Solution, solver_run: SolverRun) -> None: @@ -63,5 +128,3 @@ class Problem(BaseModel): error.msg, error.args[0]) - - diff --git a/app/models/solver_run.py b/app/models/solver_run.py index 77df041..02653db 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -33,6 +33,7 @@ class SolverRun(BaseModel): total_forms: int = 1 theta_cut_score: float = 0.00 drift_style: Literal['constant', 'variable'] = 'constant' + allow_enemies: bool = False advanced_options: Optional[AdvancedOptions] engine: str diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index 082b10d..4b3a09a 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -1,6 +1,6 @@ import json, random, io, logging -from pulp import LpProblem, LpVariable, LpMinimize, LpStatus, lpSum +from pulp import LpProblem, LpMinimize, LpStatus from lib.application_configs import ApplicationConfigs from helpers import aws_helper, tar_helper, csv_helper, service_helper @@ -82,7 +82,7 @@ class FormGenerationService(Base): # create problem problem_handler = Problem(items = self.solver_run.unbundled_items(), bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) problem_handler.generate(solution, self.solver_run) - problem = problem_handler.solve() + problem = problem_handler.solve(self.solver_run) if LpStatus[problem.status] == 'Infeasible': logging.info( From 4e8c8416b674afe2774cbc254801652218df5d60 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Mon, 13 Nov 2023 19:08:01 -0500 Subject: [PATCH 09/13] typos, factoring out functionality --- app/helpers/problem_helper.py | 23 +++++++++++++++++++++++ app/models/problem.py | 25 ++++++------------------- 2 files changed, 29 insertions(+), 19 deletions(-) create mode 100644 app/helpers/problem_helper.py diff --git a/app/helpers/problem_helper.py b/app/helpers/problem_helper.py new file mode 100644 index 0000000..27f5629 --- /dev/null +++ b/app/helpers/problem_helper.py @@ -0,0 +1,23 @@ +from typing import Tuple + +from models.item import Item + +def sanctify(solved_items: [Item]) -> Tuple[list]: + sacred_ids = [] + enemy_ids = [] + + # get all enemies + for item in solved_items: + # if it has enemies, check if it exists as part of the solved items + for enemy_id in item.enemies: + # if it does, it's a true enemy + if enemy_id in (item.id for item in solved_items): + enemy_ids.append(enemy_id) + # remove enemy from solved items, + # lest it has this sacred item added to enemies + solved_items = [i for i in solved_items if i.id != enemy_id] + + # the item is cleansed, now it's sacred + sacred_ids.append(item.id) + + return sacred_ids, enemy_ids \ No newline at end of file diff --git a/app/models/problem.py b/app/models/problem.py index 5ddd33a..8976c64 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -7,7 +7,9 @@ from pulp import LpProblem, LpVariable, LpStatus, lpSum import logging -from helpers import service_helper, irt_helper +from helpers.problem_helper import * + +from helpers import service_helper from models.solution import Solution from models.item import Item @@ -59,29 +61,14 @@ class Problem(BaseModel): # get items from solution solved_items, _ = service_helper.solution_items(self.problem.variables(), solver_run) - sacred_ids = [] - enemy_ids = [] - - # get all enemies - for item in solved_items: - # if it has enemies, check if it exists as part of the solved items - for enemy_id in item.enemies: - # if it does, it's a true enemy - if enemy_id in (item.id for item in solved_items): - enemy_ids.append(enemy_id) - # remove enemy from solved items, - # lest it has this sacred item added to enemies - solved_items = [i for i in solved_items if i.id != enemy_id] - - # the item is cleansed, now it's sacred - sacred_ids.append(item.id) + sacred_ids, enemy_ids = sanctify(solved_items) if enemy_ids: logging.info('enemies found, adding constraints...') # remove old enemy/sacred constraints - self.problem.constrants.pop('Exclude_enemy_items') - self.problem.constrants.pop('Include_sacred_items') + self.problem.constraints.pop('Exclude_enemy_items') + self.problem.constraints.pop('Include_sacred_items') # add constraint to not allow enemy items self.problem += lpSum([ From c15345867d9b7d100affcd0a269d6dd5f7241350 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Tue, 14 Nov 2023 16:14:23 -0500 Subject: [PATCH 10/13] but when removing enemy constraints --- app/helpers/problem_helper.py | 20 +++++++++----------- app/models/item.py | 9 ++++++++- app/models/problem.py | 5 ++--- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/app/helpers/problem_helper.py b/app/helpers/problem_helper.py index 27f5629..84a6c85 100644 --- a/app/helpers/problem_helper.py +++ b/app/helpers/problem_helper.py @@ -3,21 +3,19 @@ from typing import Tuple from models.item import Item def sanctify(solved_items: [Item]) -> Tuple[list]: - sacred_ids = [] enemy_ids = [] # get all enemies for item in solved_items: - # if it has enemies, check if it exists as part of the solved items - for enemy_id in item.enemies: - # if it does, it's a true enemy - if enemy_id in (item.id for item in solved_items): - enemy_ids.append(enemy_id) - # remove enemy from solved items, - # lest it has this sacred item added to enemies - solved_items = [i for i in solved_items if i.id != enemy_id] + # if the item is already an an enemy + # then it's enemy is sacred + if item.id not in enemy_ids: + # if it has enemies, check if it exists as part of the solved items + for enemy_id in item.enemies: + # if it does, it's a true enemy + if enemy_id in (i.id for i in solved_items): + enemy_ids.append(enemy_id) - # the item is cleansed, now it's sacred - sacred_ids.append(item.id) + sacred_ids = [i.id for i in solved_items if i.id not in enemy_ids] return sacred_ids, enemy_ids \ No newline at end of file diff --git a/app/models/item.py b/app/models/item.py index b82dd03..2f34ed1 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, validator from typing import List, Optional from models.attribute import Attribute @@ -16,6 +16,13 @@ class Item(BaseModel): b_param: float = 0.00 response: Optional[int] = None + @validator("enemies", pre=True) + def set_enemies(cls, v) -> List[id]: + if v == "": + return [] + enemies = list(filter(None, [int(enemy) for enemy in v.split(",")])) + return enemies + def iif(self, irt_model, theta): return ItemInformationFunction(irt_model).calculate(b_param=self.b_param, theta=theta) diff --git a/app/models/problem.py b/app/models/problem.py index 8976c64..fac4dec 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -8,7 +8,6 @@ from pulp import LpProblem, LpVariable, LpStatus, lpSum import logging from helpers.problem_helper import * - from helpers import service_helper from models.solution import Solution @@ -67,8 +66,8 @@ class Problem(BaseModel): logging.info('enemies found, adding constraints...') # remove old enemy/sacred constraints - self.problem.constraints.pop('Exclude_enemy_items') - self.problem.constraints.pop('Include_sacred_items') + if 'Exclude_enemy_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Exclude_enemy_items') + if 'Include_sacred_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Include_sacred_items') # add constraint to not allow enemy items self.problem += lpSum([ From eef86369ab87ef7fc566d4f147e6988b28f2acb5 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Wed, 15 Nov 2023 11:14:35 -0500 Subject: [PATCH 11/13] handle n enemy cleansing runs --- app/models/problem.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/app/models/problem.py b/app/models/problem.py index fac4dec..879e10c 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -41,7 +41,7 @@ class Problem(BaseModel): upBound=1, cat='Binary') - def solve(self, solver_run: SolverRun) -> LpProblem: + def solve(self, solver_run: SolverRun, enemy_ids: List[int] = []) -> LpProblem: logging.info('solving problem...') # if we allow enemies, go through the normal solving process if solver_run.allow_enemies: @@ -60,11 +60,17 @@ class Problem(BaseModel): # get items from solution solved_items, _ = service_helper.solution_items(self.problem.variables(), solver_run) - sacred_ids, enemy_ids = sanctify(solved_items) + # sacred items will remain the same between solve attempts + # but new enemies will be appended + sacred_ids, new_enemy_ids = sanctify(solved_items) - if enemy_ids: + # the current solve run found new enemies + if new_enemy_ids: logging.info('enemies found, adding constraints...') + # append the new enemies to the enemies_id list + enemy_ids = list(set(enemy_ids+new_enemy_ids)) + # remove old enemy/sacred constraints if 'Exclude_enemy_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Exclude_enemy_items') if 'Include_sacred_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Include_sacred_items') From 5b4387a04b6e0836adbb5d691cc79db117a95f5f Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Wed, 15 Nov 2023 11:35:56 -0500 Subject: [PATCH 12/13] consolidate constraint types into single type var --- app/models/problem.py | 2 +- app/models/solver_run.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/app/models/problem.py b/app/models/problem.py index 879e10c..5c6f75d 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -60,7 +60,7 @@ class Problem(BaseModel): # get items from solution solved_items, _ = service_helper.solution_items(self.problem.variables(), solver_run) - # sacred items will remain the same between solve attempts + # sacred items will remain the same (with new items added each run) between solve attempts # but new enemies will be appended sacred_ids, new_enemy_ids = sanctify(solved_items) diff --git a/app/models/solver_run.py b/app/models/solver_run.py index 02653db..ff320a5 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -1,9 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING - +from typing import TYPE_CHECKING, List, Literal, Optional, Union, TypeVar from pulp import lpSum from pydantic import BaseModel -from typing import List, Literal, Optional, Union + import logging @@ -22,11 +21,13 @@ if TYPE_CHECKING: from models.solution import Solution from models.problem import Problem +ConstraintType = TypeVar('ConstraintType', bound=GenericConstraint) + class SolverRun(BaseModel): items: List[Item] = [] bundles: List[Bundle] = [] bundle_first_ordering: bool = True - constraints: List[Union[GenericConstraint, MetadataConstraint, BundleConstraint, FormUniquenessConstraint, TotalFormItemsConstraint]] + constraints: List[ConstraintType] irt_model: IRTModel objective_function: ObjectiveFunction total_form_items: int @@ -42,7 +43,7 @@ class SolverRun(BaseModel): # this is all a compensator for dynamically creating objects # ideally we'd change the payload to determine what type it is - constraints: [GenericConstraint|MetadataConstraint|BundleConstraint|FormUniquenessConstraint|TotalFormItemsConstraint] = [] + constraints: [ConstraintType] = [] # total form items constraints.append(TotalFormItemsConstraint.create(self.total_form_items)) @@ -70,7 +71,7 @@ class SolverRun(BaseModel): if bundle.id == bundle_id: return bundle - def get_constraint_by_type(self, type: str) -> GenericConstraint|MetadataConstraint|BundleConstraint|FormUniquenessConstraint|TotalFormItemsConstraint or None: + def get_constraint_by_type(self, type: str) -> ConstraintType or None: for constraint in self.constraints: if type == constraint.reference_attribute.type: return constraint @@ -131,7 +132,7 @@ class SolverRun(BaseModel): logging.info('Bundles Generated...') - def get_constraint(self, name: str) -> GenericConstraint|MetadataConstraint|BundleConstraint|FormUniquenessConstraint|TotalFormItemsConstraint: + def get_constraint(self, name: str) -> ConstraintType: return next((constraint for constraint in self.constraints if constraint.reference_attribute.id == name), None) From 6e320d7cbc0752aa29384718b6944dc53b78c055 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Thu, 16 Nov 2023 12:52:37 -0500 Subject: [PATCH 13/13] enemy management v2.0 --- app/models/attribute.py | 2 +- app/models/bundle.py | 10 ++ .../constraints/enemy_pair_constraint.py | 31 +++++++ app/models/item.py | 15 ++- app/models/problem.py | 93 ++++++++++--------- app/models/solver_run.py | 34 +++++-- app/services/form_generation_service.py | 1 + 7 files changed, 131 insertions(+), 55 deletions(-) create mode 100644 app/models/constraints/enemy_pair_constraint.py diff --git a/app/models/attribute.py b/app/models/attribute.py index 35254e0..ee01651 100644 --- a/app/models/attribute.py +++ b/app/models/attribute.py @@ -2,6 +2,6 @@ from pydantic import BaseModel from typing import Optional, Union class Attribute(BaseModel): - value: Optional[Union[str,int]] + value: Optional[Union[str,int,list]] type: Optional[str] id: str diff --git a/app/models/bundle.py b/app/models/bundle.py index bd778ff..8b5ded1 100644 --- a/app/models/bundle.py +++ b/app/models/bundle.py @@ -59,3 +59,13 @@ class Bundle(BaseModel): def ordered_items(self) -> List[Item]: return sorted(self.items, key=lambda item: item.position) + + # are there enemys in the bundle? + def enemy_pair_count(self, pair: List[Item]) -> int: + pair_count = 0 + + for item in self.items: + if pair in item.enemy_pairs(): + pair_count += 1 + + return pair_count diff --git a/app/models/constraints/enemy_pair_constraint.py b/app/models/constraints/enemy_pair_constraint.py new file mode 100644 index 0000000..ee4a312 --- /dev/null +++ b/app/models/constraints/enemy_pair_constraint.py @@ -0,0 +1,31 @@ +from __future__ import annotations +from typing import List + +from models.constraints.generic_constraint import * + +class EnemyPairConstraint(GenericConstraint): + @classmethod + def create(cls: Type[_T], pair: List[int]) -> _T: + return cls( + minimum=0, + maximum=0, + reference_attribute=Attribute( + value=pair, + type='enemy_pair', + id='enemy_pair' + ) + ) + + def build(self, problem_handler: Problem, **_) -> None: + logging.info('Enemy Pair Constraint Generating...') + + pair = self.reference_attribute.value + problem_handler.problem += lpSum( + [ + bundle.enemy_pair_count(pair) * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + (pair in item.enemy_pairs()).real * problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ] + ) <= 1, f'Enemy Pair constraint for pair: {pair}' diff --git a/app/models/item.py b/app/models/item.py index 2f34ed1..2c1ca66 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -1,5 +1,6 @@ +import logging from pydantic import BaseModel, validator -from typing import List, Optional +from typing import List, Optional, Tuple from models.attribute import Attribute @@ -61,3 +62,15 @@ class Item(BaseModel): total += self.irf(solver_run.irt_model, target.theta) return total + + def enemy_pairs(self, sort: bool = True) -> List[List[int]]: + pairs = [] + + for enemy_id in self.enemies: + pair = [self.id, enemy_id] + + if sort: pair.sort() + + pairs.append(pair) + + return pairs diff --git a/app/models/problem.py b/app/models/problem.py index 5c6f75d..f08ed73 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -43,59 +43,64 @@ class Problem(BaseModel): def solve(self, solver_run: SolverRun, enemy_ids: List[int] = []) -> LpProblem: logging.info('solving problem...') + self.problem.solve() + + # NOTICE: Legacy enemies implementation + # leaving this in, just in case the current impl fails to function + # and we need an immediate solution # if we allow enemies, go through the normal solving process - if solver_run.allow_enemies: - logging.info('enemes allowed, so just solving') - self.problem.solve() - # otherwise begin the process of filtering enemies - else: - self.problem.solve() + # if solver_run.allow_enemies: + # logging.info('enemes allowed, so just solving') + # self.problem.solve() + # # otherwise begin the process of filtering enemies + # else: + # self.problem.solve() - # however, if the solve was infeasible, kick it back - # to the normal process - if LpStatus[self.problem.status] == 'Infeasible': - return self.problem - # otherwise continue - else: - # get items from solution - solved_items, _ = service_helper.solution_items(self.problem.variables(), solver_run) + # # however, if the solve was infeasible, kick it back + # # to the normal process + # if LpStatus[self.problem.status] == 'Infeasible': + # return self.problem + # # otherwise continue + # else: + # # get items from solution + # solved_items, _ = service_helper.solution_items(self.problem.variables(), solver_run) - # sacred items will remain the same (with new items added each run) between solve attempts - # but new enemies will be appended - sacred_ids, new_enemy_ids = sanctify(solved_items) + # # sacred items will remain the same (with new items added each run) between solve attempts + # # but new enemies will be appended + # sacred_ids, new_enemy_ids = sanctify(solved_items) - # the current solve run found new enemies - if new_enemy_ids: - logging.info('enemies found, adding constraints...') + # # the current solve run found new enemies + # if new_enemy_ids: + # logging.info('enemies found, adding constraints...') - # append the new enemies to the enemies_id list - enemy_ids = list(set(enemy_ids+new_enemy_ids)) + # # append the new enemies to the enemies_id list + # enemy_ids = list(set(enemy_ids+new_enemy_ids)) - # remove old enemy/sacred constraints - if 'Exclude_enemy_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Exclude_enemy_items') - if 'Include_sacred_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Include_sacred_items') + # # remove old enemy/sacred constraints + # if 'Exclude_enemy_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Exclude_enemy_items') + # if 'Include_sacred_items' in self.problem.constraints.keys(): self.problem.constraints.pop('Include_sacred_items') - # add constraint to not allow enemy items - self.problem += lpSum([ - len(bundle.find_items(enemy_ids)) * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - (item.id in enemy_ids) * self.solver_items_var[item.id] - for item in self.items - ]) == 0, 'Exclude enemy items' + # # add constraint to not allow enemy items + # self.problem += lpSum([ + # len(bundle.find_items(enemy_ids)) * self.solver_bundles_var[bundle.id] + # for bundle in self.bundles + # ] + [ + # (item.id in enemy_ids) * self.solver_items_var[item.id] + # for item in self.items + # ]) == 0, 'Exclude enemy items' - # add constraint to use sacred items - self.problem += lpSum([ - len(bundle.find_items(sacred_ids)) * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - (item.id in sacred_ids) * self.solver_items_var[item.id] - for item in self.items - ]) == len(sacred_ids), 'Include sacred items' + # # add constraint to use sacred items + # self.problem += lpSum([ + # len(bundle.find_items(sacred_ids)) * self.solver_bundles_var[bundle.id] + # for bundle in self.bundles + # ] + [ + # (item.id in sacred_ids) * self.solver_items_var[item.id] + # for item in self.items + # ]) == len(sacred_ids), 'Include sacred items' - # recursively solve until no enemies exist or infeasible - logging.info('recursively solving...') - self.solve(solver_run) + # # recursively solve until no enemies exist or infeasible + # logging.info('recursively solving...') + # self.solve(solver_run) return self.problem diff --git a/app/models/solver_run.py b/app/models/solver_run.py index ff320a5..2d486dd 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -1,9 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Literal, Optional, Union, TypeVar +from typing import TYPE_CHECKING, List, Literal, Optional, Tuple, Union, TypeVar from pulp import lpSum from pydantic import BaseModel - +import itertools import logging from models.item import Item @@ -12,6 +12,7 @@ from models.constraints.metadata_constraint import MetadataConstraint from models.constraints.bundle_constraint import BundleConstraint from models.constraints.form_uniqueness_constraint import FormUniquenessConstraint from models.constraints.total_form_items_constraint import TotalFormItemsConstraint +from models.constraints.enemy_pair_constraint import EnemyPairConstraint from models.irt_model import IRTModel from models.bundle import Bundle from models.objective_function import ObjectiveFunction @@ -45,13 +46,6 @@ class SolverRun(BaseModel): # ideally we'd change the payload to determine what type it is constraints: [ConstraintType] = [] - # total form items - constraints.append(TotalFormItemsConstraint.create(self.total_form_items)) - - # ensure form uniqueness - if self.advanced_options.ensure_form_uniqueness: - constraints.append(FormUniquenessConstraint.create(self.total_form_items - 1)) - # repackage to create appropriate constraint types for constraint in self.constraints: if constraint.reference_attribute.type == 'metadata': @@ -80,6 +74,18 @@ class SolverRun(BaseModel): self.items = [item for item in self.items if item not in items] return True + def generate_constraints(self) -> None: + # total form items + self.constraints.append(TotalFormItemsConstraint.create(self.total_form_items)) + + # ensure form uniqueness + if self.advanced_options.ensure_form_uniqueness: + self.constraints.append(FormUniquenessConstraint.create(self.total_form_items - 1)) + + # enemies constraints + for pair in self.enemy_pairs(): + self.constraints.append(EnemyPairConstraint.create(pair)) + def generate_bundles(self): logging.info('Generating Bundles...') # confirms bundle constraints exists @@ -149,3 +155,13 @@ class SolverRun(BaseModel): else: return self.items + def enemy_pairs(self) -> List[List[int]]: + pairs = [] + + for item in self.items: + # add enemy pairs for item to pairs + pairs += item.enemy_pairs() + + # remove duplicates + pairs.sort() + return list(k for k,_ in itertools.groupby(pairs)) diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index 4b3a09a..2f14f2f 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -21,6 +21,7 @@ class FormGenerationService(Base): try: self.solver_run = self.create_solver_run_from_attributes() self.solver_run.generate_bundles() + self.solver_run.generate_constraints() self.solution = self.generate_solution() self.result = self.stream_to_s3_bucket() except ItemGenerationError as error: