From 6d3639a0c10152d0e280408c9306124807b98594 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Fri, 10 Nov 2023 15:21:16 -0500 Subject: [PATCH] moved constraints to objects --- app/helpers/solver_helper.py | 6 +- app/models/attribute.py | 1 - app/models/bundle_constraint.py | 20 ++++++ app/models/constraint.py | 5 +- app/models/form.py | 11 +++- app/models/irt_target_constraint.py | 51 +++++++++++++++ app/models/item.py | 13 ++-- app/models/metadata_constraint.py | 34 ++++++++++ app/models/problem.py | 87 +++++++++++++++++++++---- app/models/solution.py | 7 +- app/models/solver_run.py | 30 ++++++++- app/services/form_generation_service.py | 5 +- 12 files changed, 232 insertions(+), 38 deletions(-) create mode 100644 app/models/bundle_constraint.py create mode 100644 app/models/irt_target_constraint.py create mode 100644 app/models/metadata_constraint.py diff --git a/app/helpers/solver_helper.py b/app/helpers/solver_helper.py index ac2f6a5..5ed124a 100644 --- a/app/helpers/solver_helper.py +++ b/app/helpers/solver_helper.py @@ -48,7 +48,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, elif attribute.type == 'bundle': logging.info('Bundles Constraint Generating...') # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates - if selected_bundles != None and selected_bundles > 0: + if selected_bundles != None and len(selected_bundles) > 0: # make sure the total bundles used in generated form is limited between min-max set problem += lpSum([ bundles[bundle.id] for bundle in selected_bundles @@ -56,7 +56,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, int(constraint.maximum)) logging.info('Constraints Created...') - + # Behold our very own Elastic constraints! for tif_target in solver_run.objective_function.tif_targets: problem += lpSum([ @@ -101,7 +101,7 @@ def build_constraints(solver_run: SolverRun, problem: LpProblem, for item in selected_items ]) <= tcc_target.maximum( ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - + return problem except ValueError as error: logging.error(error) diff --git a/app/models/attribute.py b/app/models/attribute.py index 73be39a..171b5c2 100644 --- a/app/models/attribute.py +++ b/app/models/attribute.py @@ -1,7 +1,6 @@ from pydantic import BaseModel from typing import Optional - class Attribute(BaseModel): value: Optional[str] type: Optional[str] diff --git a/app/models/bundle_constraint.py b/app/models/bundle_constraint.py new file mode 100644 index 0000000..0b34dc8 --- /dev/null +++ b/app/models/bundle_constraint.py @@ -0,0 +1,20 @@ +import logging + +from random import randint + +from pulp import lpSum + +from models.constraint import Constraint +from models.problem import Problem + +class BundleConstraint(Constraint): + def build(self, problem_handler: Problem, _) -> None: + logging.info('Bundles Constraint Generating...') + + # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates + if problem_handler.bundles != None and len(problem_handler.bundles) > 0: + # make sure the total bundles used in generated form is limited between min-max set + problem_handler.problem += lpSum([ + problem_handler.solver_bundles_var[bundle.id] for bundle in problem_handler.bundles + ]) == randint(int(self.minimum), + int(self.maximum)), f'Allowing min - max bundles' diff --git a/app/models/constraint.py b/app/models/constraint.py index ba9286c..c60f80b 100644 --- a/app/models/constraint.py +++ b/app/models/constraint.py @@ -1,9 +1,12 @@ from pydantic import BaseModel +from helpers.common_helper import * from models.attribute import Attribute - class Constraint(BaseModel): reference_attribute: Attribute minimum: float maximum: float + + def __init__(self, **data) -> None: + super().__init__(**data) diff --git a/app/models/form.py b/app/models/form.py index 242acdc..140c840 100644 --- a/app/models/form.py +++ b/app/models/form.py @@ -1,14 +1,19 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + from pydantic import BaseModel from typing import List, TypeVar, Type from helpers import irt_helper -from models.solver_run import SolverRun from models.item import Item from models.target import Target from lib.irt.test_response_function import TestResponseFunction +if TYPE_CHECKING: + from models.solver_run import SolverRun + _T = TypeVar("_T") class Form(BaseModel): @@ -29,10 +34,10 @@ class Form(BaseModel): tcc_results=irt_helper.generate_tcc_results(items, solver_run), status=status, solver_variables=solver_variables) - + def has_item(self, item: Item) -> bool: for i in self.items: if item == i: return True - + return False diff --git a/app/models/irt_target_constraint.py b/app/models/irt_target_constraint.py new file mode 100644 index 0000000..c83a2a5 --- /dev/null +++ b/app/models/irt_target_constraint.py @@ -0,0 +1,51 @@ +from __future__ import annotations +from typing import TYPE_CHECKING, Optional + +from pulp import lpSum + +from models import Constraint, Problem, Attribute, Target, Item, Bundle + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class IrtTargetConstraint(Constraint): + reference_attribute: Optional[Attribute] + minimum: Optional[float] + maximum: Optional[float] + target: Target + target_type: str + + def build(self, problem_handler: Problem, solver_run: SolverRun): + problem_handler.problem += lpSum([ + self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + self.item_irt_function(item, solver_run.irt_model, self.target.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) >= self.target.minimum( + ), f'Min {self.target_type} theta({self.target.theta}) at target {self.target.value}' + + problem_handler.problem += lpSum([ + self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + self.item_irt_function(item, solver_run.irt_model, self.target.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) <= self.target.maximum( + ), f'Max {self.target_type} theta({self.target.theta}) at target {self.target.value}' + + def item_irt_function(self, item: Item, irt_model: str, theta: float) -> float: + if self.target_type == 'tcc': + return item.irf(irt_model, theta) + elif self.target_type == 'tif': + return item.iif(irt_model, theta) + + def bundle_irt_function(self, bundle: Bundle, irt_model: str, theta: float) -> float: + if self.target_type == 'tcc': + return bundle.trf(irt_model, theta) + elif self.target_type == 'tif': + return bundle.tif(irt_model, theta) \ No newline at end of file diff --git a/app/models/item.py b/app/models/item.py index 8869a6d..f084b78 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -10,16 +10,17 @@ class Item(BaseModel): id: int position: Optional[int] = None passage_id: Optional[int] = None + enemies: Optional[int] = None workflow_state: Optional[str] = None attributes: List[Attribute] = None b_param: float = 0.00 response: Optional[int] = None - def iif(self, solver_run, theta): - return ItemInformationFunction(solver_run.irt_model).calculate(b_param=self.b_param, theta=theta) + def iif(self, irt_model, theta): + return ItemInformationFunction(irt_model).calculate(b_param=self.b_param, theta=theta) - def irf(self, solver_run, theta): - return ItemResponseFunction(solver_run.irt_model).calculate(b_param=self.b_param, theta=theta) + def irf(self, irt_model, theta): + return ItemResponseFunction(irt_model).calculate(b_param=self.b_param, theta=theta) def get_attribute(self, ref_attribute: Attribute) -> Attribute or None: for attribute in self.attributes: @@ -42,7 +43,7 @@ class Item(BaseModel): total = 0 for target in solver_run.objective_function.tif_targets: - total += self.iif(solver_run, target.theta) + total += self.iif(solver_run.irt_model, target.theta) return total @@ -50,6 +51,6 @@ class Item(BaseModel): total = 0 for target in solver_run.objective_function.tif_targets: - total += self.irf(solver_run, target.theta) + total += self.irf(solver_run.irt_model, target.theta) return total diff --git a/app/models/metadata_constraint.py b/app/models/metadata_constraint.py new file mode 100644 index 0000000..d9221c9 --- /dev/null +++ b/app/models/metadata_constraint.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +import logging + +from pulp import lpSum + +from models.constraint import Constraint +from models.problem import Problem + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class MetadataConstraint(Constraint): + def build(self, problem_handler: Problem, solver_run: SolverRun) -> None: + logging.info('Metadata Constraint Generating...') + + problem_handler.problem += lpSum( + [ + len(bundle.items_with_attribute(self.reference_attribute)) * problem_handler.solver_bundles_var[bundle.id] for bundle in problem_handler.bundles + ] + + [ + item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items + ] + ) >= round(solver_run.total_form_items * (self.minimum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - min' + + problem_handler.problem += lpSum( + [ + len(bundle.items_with_attribute(self.reference_attribute)) * problem_handler.solver_bundles_var[bundle.id] for bundle in problem_handler.bundles + ] + + [ + item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items + ] + ) <= round(solver_run.total_form_items * (self.maximum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - max' diff --git a/app/models/problem.py b/app/models/problem.py index e03ab53..1fc2923 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -1,26 +1,31 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + from pydantic import BaseModel from typing import Any, List from pulp import LpProblem, LpVariable, lpSum -import logging, math +import logging -from helpers import solver_helper - -from models.solver_run import SolverRun from models.solution import Solution from models.item import Item from models.bundle import Bundle +from lib.errors.item_generation_error import ItemGenerationError + +if TYPE_CHECKING: + from models.solver_run import SolverRun + class Problem(BaseModel): items: List[Item] bundles: List[Bundle] problem: Any solver_items_var: Any = None solver_bundles_var: Any = None - + def __init__(self, **data) -> None: super().__init__(**data) - + # setup common Solver variables self.solver_items_var = LpVariable.dicts("Item", [item.id for item in self.items], @@ -32,7 +37,7 @@ class Problem(BaseModel): lowBound=0, upBound=1, cat='Binary') - + # objective function self.problem += lpSum([ bundle.count * self.solver_bundles_var[bundle.id] @@ -41,13 +46,13 @@ class Problem(BaseModel): self.solver_items_var[item.id] for item in self.items ]) - + def solve(self) -> LpProblem: self.problem.solve() return self.problem - + def generate(self, solution: Solution, solver_run: SolverRun): - + # Form Constraints self.problem += lpSum( [ @@ -58,7 +63,7 @@ class Problem(BaseModel): for item in self.items ] ) == solver_run.total_form_items, f'Total bundle form items for form' - + # each time a form is generated, we want to ensure # that it is unique to all other forms generated before it self.problem += lpSum( @@ -70,10 +75,64 @@ class Problem(BaseModel): for item in self.items ] ) <= solver_run.total_form_items - 1, f'Ensuring uniqueness for form' - + def generate_constraints(self, solver_run: SolverRun, current_drift: int): - self.problem = solver_helper.build_constraints( - solver_run, self.problem, self.solver_items_var, self.solver_bundles_var, self.items, self.bundles, current_drift) + logging.info('Creating Constraints...') + + try: + for constraint in solver_run.constraints: + constraint.build(self, solver_run) + + for tif_target in solver_run.objective_function.tif_targets: + self.problem += lpSum([ + bundle.tif(solver_run.irt_model, tif_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.iif(solver_run.irt_model, tif_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) >= tif_target.minimum( + ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' + self.problem += lpSum([ + bundle.tif(solver_run.irt_model, tif_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.iif(solver_run.irt_model, tif_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) <= tif_target.maximum( + ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' + + for tcc_target in solver_run.objective_function.tcc_targets: + self.problem += lpSum([ + bundle.trf(solver_run.irt_model, tcc_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.irf(solver_run.irt_model, tcc_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) >= tcc_target.minimum( + ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + self.problem += lpSum([ + bundle.trf(solver_run.irt_model, tcc_target.theta) + * self.solver_bundles_var[bundle.id] + for bundle in self.bundles + ] + [ + item.irf(solver_run.irt_model, tcc_target.theta) * + self.solver_items_var[item.id] + for item in self.items + ]) <= tcc_target.maximum( + ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + + logging.info('Constraints Created...') + except ValueError as error: + logging.error(error) + raise ItemGenerationError( + "Bundle min and/or max larger than bundle amount provided", + error.args[0]) diff --git a/app/models/solution.py b/app/models/solution.py index 47fbd25..22f3620 100644 --- a/app/models/solution.py +++ b/app/models/solution.py @@ -4,11 +4,10 @@ from typing import List from models.form import Form from models.item import Item - class Solution(BaseModel): response_id: int forms: List[Form] - + def items_exist_in_forms(self, items: [Item]) -> bool: items_found = 0 @@ -16,7 +15,5 @@ class Solution(BaseModel): for form in self.forms: if form.has_item(item): items_found += 1 - + return items_found - - diff --git a/app/models/solver_run.py b/app/models/solver_run.py index 79eb9ed..439fb81 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -1,22 +1,24 @@ from pydantic import BaseModel -from typing import List, Literal, Optional +from typing import List, Literal, Optional, Union import logging import random from models.item import Item from models.constraint import Constraint +from models.metadata_constraint import MetadataConstraint +from models.bundle_constraint import BundleConstraint +# from models.irt_target_constraint import IrtTargetConstraint from models.irt_model import IRTModel from models.bundle import Bundle from models.objective_function import ObjectiveFunction from models.advanced_options import AdvancedOptions - class SolverRun(BaseModel): items: List[Item] = [] bundles: List[Bundle] = [] bundle_first_ordering: bool = True - constraints: List[Constraint] + constraints: List[Union[Constraint, MetadataConstraint, BundleConstraint]] irt_model: IRTModel objective_function: ObjectiveFunction total_form_items: int @@ -26,6 +28,28 @@ class SolverRun(BaseModel): advanced_options: Optional[AdvancedOptions] engine: str + def __init__(self, **data) -> None: + super().__init__(**data) + + # this is all a compensator for dynamically creating objects + # ideally we'd change the payload to determine what type it is + constraints: [Constraint|MetadataConstraint|BundleConstraint] = [] + + for constraint in self.constraints: + if constraint.reference_attribute.type == 'metadata': + constraints.append(MetadataConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) + elif constraint.reference_attribute.type == 'bundle': + constraints.append(BundleConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) + + # constraints for tif and tcc targets + # for target in self.objective_function.tif_targets: + # constraints.append(IrtTargetConstraint(target=target, target_type='tif')) + + # for target in self.objective_function.tcc_targets: + # constraints.append(IrtTargetConstraint(target=target, target_type='tcc')) + + self.constraints = constraints + def get_item(self, item_id: int) -> Item or None: for item in self.items: if item.id == item_id: diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index 6b3080a..bdf813f 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -80,11 +80,12 @@ class FormGenerationService(Base): drift_percent) # create problem - problem_handler = Problem(items = self.solver_run.items, bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) + problem_handler = Problem(items = self.solver_run.unbundled_items(), bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) problem_handler.generate(solution, self.solver_run) problem_handler.generate_constraints(self.solver_run, current_drift) - + problem = problem_handler.solve() + logging.info(problem) if LpStatus[problem.status] == 'Infeasible': logging.info(