From f1fa519f31093b6285f0f3ca4ac337eec25cafc7 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Sun, 12 Nov 2023 18:32:48 -0500 Subject: [PATCH] refactor of model targets and constraints, addition of new constraint types and constraint construction process --- app/helpers/irt_helper.py | 7 +-- app/models/advanced_options.py | 1 + app/models/attribute.py | 4 +- .../{ => constraints}/bundle_constraint.py | 6 +-- .../constraints/form_uniqueness_constraint.py | 34 +++++++++++++ .../generic_constraint.py} | 5 +- .../{ => constraints}/metadata_constraint.py | 10 ++-- .../total_form_items_constraint.py | 33 +++++++++++++ app/models/form.py | 7 +-- app/models/objective_function.py | 16 +++++-- app/models/problem.py | 48 ++++--------------- app/models/solver_run.py | 43 +++++++++-------- app/models/{ => targets}/target.py | 0 app/models/{ => targets}/tcc_target.py | 4 +- app/models/{ => targets}/tif_target.py | 4 +- app/services/form_generation_service.py | 4 +- 16 files changed, 140 insertions(+), 86 deletions(-) rename app/models/{ => constraints}/bundle_constraint.py (80%) create mode 100644 app/models/constraints/form_uniqueness_constraint.py rename app/models/{constraint.py => constraints/generic_constraint.py} (74%) rename app/models/{ => constraints}/metadata_constraint.py (68%) create mode 100644 app/models/constraints/total_form_items_constraint.py rename app/models/{ => targets}/target.py (100%) rename app/models/{ => targets}/tcc_target.py (91%) rename app/models/{ => targets}/tif_target.py (91%) diff --git a/app/helpers/irt_helper.py b/app/helpers/irt_helper.py index 401e875..5a9821e 100644 --- a/app/helpers/irt_helper.py +++ b/app/helpers/irt_helper.py @@ -1,14 +1,15 @@ from lib.irt.test_response_function import TestResponseFunction from lib.irt.test_information_function import TestInformationFunction -from models.target import Target +from models.targets.tif_target import TifTarget +from models.targets.tcc_target import TccTarget def generate_tif_results(items, solver_run): targets = [] for target in solver_run.objective_function.tif_targets: tif = TestInformationFunction(solver_run.irt_model).calculate(items, theta=target.theta) - targets.append(Target(theta=target.theta, value=target.value, result=tif)) + targets.append(TifTarget(theta=target.theta, value=target.value, result=tif)) return targets @@ -17,6 +18,6 @@ def generate_tcc_results(items, solver_run): for target in solver_run.objective_function.tcc_targets: tcc = TestResponseFunction(solver_run.irt_model).calculate(items, theta=target.theta) - targets.append(Target(theta=target.theta, value=target.value, result=tcc)) + targets.append(TccTarget(theta=target.theta, value=target.value, result=tcc)) return targets diff --git a/app/models/advanced_options.py b/app/models/advanced_options.py index c9b7213..3ca1232 100644 --- a/app/models/advanced_options.py +++ b/app/models/advanced_options.py @@ -9,4 +9,5 @@ class AdvancedOptions(BaseModel): brand_bound_tolerance: Optional[float] = None max_forms: Optional[int] = None precision: Optional[float] = None + ensure_form_uniqueness: bool = True extra_param_range: Optional[List[Dict]] = None diff --git a/app/models/attribute.py b/app/models/attribute.py index 171b5c2..35254e0 100644 --- a/app/models/attribute.py +++ b/app/models/attribute.py @@ -1,7 +1,7 @@ from pydantic import BaseModel -from typing import Optional +from typing import Optional, Union class Attribute(BaseModel): - value: Optional[str] + value: Optional[Union[str,int]] type: Optional[str] id: str diff --git a/app/models/bundle_constraint.py b/app/models/constraints/bundle_constraint.py similarity index 80% rename from app/models/bundle_constraint.py rename to app/models/constraints/bundle_constraint.py index 5ef0afa..5809a35 100644 --- a/app/models/bundle_constraint.py +++ b/app/models/constraints/bundle_constraint.py @@ -1,9 +1,9 @@ from random import randint -from models.constraint import * +from models.constraints.generic_constraint import * -class BundleConstraint(Constraint): - def build(self, problem_handler: Problem, _) -> None: +class BundleConstraint(GenericConstraint): + def build(self, problem_handler: Problem, **kwargs) -> None: logging.info('Bundles Constraint Generating...') # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates diff --git a/app/models/constraints/form_uniqueness_constraint.py b/app/models/constraints/form_uniqueness_constraint.py new file mode 100644 index 0000000..fdbb1fb --- /dev/null +++ b/app/models/constraints/form_uniqueness_constraint.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.constraints.generic_constraint import * + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class FormUniquenessConstraint(GenericConstraint): + @classmethod + def create(cls: Type[_T], total_items: int) -> _T: + return cls( + minimum=0, + maximum=0, + reference_attribute=Attribute( + value=total_items, + type='form_uniqueness', + id='form_uniqueness' + ) + ) + + def build(self, problem_handler: Problem, **kwargs) -> None: + logging.info('Form Uniqueness Constraint Generating...') + + # form uniqueness constraint + problem_handler.problem += lpSum( + [ + kwargs['solution'].items_exist_in_forms(bundle.items) * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + kwargs['solution'].items_exist_in_forms([item]) * problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ] + ) <= self.reference_attribute.value, f'Ensuring uniqueness for form' diff --git a/app/models/constraint.py b/app/models/constraints/generic_constraint.py similarity index 74% rename from app/models/constraint.py rename to app/models/constraints/generic_constraint.py index abb06a2..a740b3b 100644 --- a/app/models/constraint.py +++ b/app/models/constraints/generic_constraint.py @@ -2,13 +2,16 @@ import logging from pulp import lpSum from pydantic import BaseModel +from typing import TypeVar, Type from helpers.common_helper import * from models.attribute import Attribute from models.problem import Problem -class Constraint(BaseModel): +_T = TypeVar("_T") + +class GenericConstraint(BaseModel): reference_attribute: Attribute minimum: float maximum: float diff --git a/app/models/metadata_constraint.py b/app/models/constraints/metadata_constraint.py similarity index 68% rename from app/models/metadata_constraint.py rename to app/models/constraints/metadata_constraint.py index 7284030..19fbb98 100644 --- a/app/models/metadata_constraint.py +++ b/app/models/constraints/metadata_constraint.py @@ -1,13 +1,13 @@ from __future__ import annotations from typing import TYPE_CHECKING -from models.constraint import * +from models.constraints.generic_constraint import * if TYPE_CHECKING: from models.solver_run import SolverRun -class MetadataConstraint(Constraint): - def build(self, problem_handler: Problem, solver_run: SolverRun) -> None: +class MetadataConstraint(GenericConstraint): + def build(self, problem_handler: Problem, **kwargs) -> None: logging.info('Metadata Constraint Generating...') problem_handler.problem += lpSum( @@ -17,7 +17,7 @@ class MetadataConstraint(Constraint): [ item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items ] - ) >= round(solver_run.total_form_items * (self.minimum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - min' + ) >= round(kwargs['solver_run'].total_form_items * (self.minimum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - min' problem_handler.problem += lpSum( [ @@ -26,4 +26,4 @@ class MetadataConstraint(Constraint): [ item.attribute_exists(self.reference_attribute).real * problem_handler.solver_items_var[item.id] for item in problem_handler.items ] - ) <= round(solver_run.total_form_items * (self.maximum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - max' + ) <= round(kwargs['solver_run'].total_form_items * (self.maximum / 100)), f'{self.reference_attribute.id} - {self.reference_attribute.value} - max' diff --git a/app/models/constraints/total_form_items_constraint.py b/app/models/constraints/total_form_items_constraint.py new file mode 100644 index 0000000..5b3c64a --- /dev/null +++ b/app/models/constraints/total_form_items_constraint.py @@ -0,0 +1,33 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.constraints.generic_constraint import * + +if TYPE_CHECKING: + from models.solver_run import SolverRun + +class TotalFormItemsConstraint(GenericConstraint): + @classmethod + def create(cls: Type[_T], total_items: int) -> _T: + return cls( + minimum=0, + maximum=0, + reference_attribute=Attribute( + value=total_items, + type='form_uniqueness', + id='form_uniqueness' + ) + ) + + def build(self, problem_handler: Problem, **kwargs) -> None: + logging.info('Total Form Items Constraint Generating...') + + problem_handler.problem += lpSum( + [ + bundle.count * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + 1 * problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ] + ) == self.reference_attribute.value, f'Total bundle form items for form' diff --git a/app/models/form.py b/app/models/form.py index 140c840..6a0382e 100644 --- a/app/models/form.py +++ b/app/models/form.py @@ -7,7 +7,8 @@ from typing import List, TypeVar, Type from helpers import irt_helper from models.item import Item -from models.target import Target +from models.targets.tif_target import TifTarget +from models.targets.tcc_target import TccTarget from lib.irt.test_response_function import TestResponseFunction @@ -19,8 +20,8 @@ _T = TypeVar("_T") class Form(BaseModel): items: List[Item] cut_score: float - tif_results: List[Target] - tcc_results: List[Target] + tif_results: List[TifTarget] + tcc_results: List[TccTarget] status: str = 'Not Optimized' solver_variables: List[str] diff --git a/app/models/objective_function.py b/app/models/objective_function.py index 3941434..3a43680 100644 --- a/app/models/objective_function.py +++ b/app/models/objective_function.py @@ -2,10 +2,11 @@ from __future__ import annotations from pydantic import BaseModel from typing import Dict, List, AnyStr +from pulp import lpSum -from models.tif_target import TifTarget -from models.tcc_target import TccTarget - +from models.targets.tif_target import TifTarget +from models.targets.tcc_target import TccTarget +from models.problem import Problem class ObjectiveFunction(BaseModel): # minimizing tif/tcc target value is only option currently @@ -17,6 +18,15 @@ class ObjectiveFunction(BaseModel): objective: AnyStr = "minimize" weight: Dict = {'tif': 1, 'tcc': 1} + def for_problem(self, problem_handler: Problem) -> None: + problem_handler.problem += lpSum([ + bundle.count * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) + def increment_targets_drift(self, limit: float or bool, all: bool = False, diff --git a/app/models/problem.py b/app/models/problem.py index e9171cc..23e0cb4 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -38,53 +38,23 @@ class Problem(BaseModel): upBound=1, cat='Binary') - # objective function - self.problem += lpSum([ - bundle.count * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - self.solver_items_var[item.id] - for item in self.items - ]) - def solve(self) -> LpProblem: self.problem.solve() return self.problem def generate(self, solution: Solution, solver_run: SolverRun) -> None: - - # Objective Function - self.problem += lpSum( - [ - bundle.count * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - 1 * self.solver_items_var[item.id] - for item in self.items - ] - ) == solver_run.total_form_items, f'Total bundle form items for form' - - # each time a form is generated, we want to ensure - # that it is unique to all other forms generated before it - self.problem += lpSum( - [ - solution.items_exist_in_forms(bundle.items) * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - solution.items_exist_in_forms([item]) * self.solver_items_var[item.id] - for item in self.items - ] - ) <= solver_run.total_form_items - 1, f'Ensuring uniqueness for form' - - def generate_constraints(self, solver_run: SolverRun, current_drift: int) -> None: - logging.info('Creating Constraints...') - try: - for constraint in solver_run.constraints: - constraint.build(self, solver_run) + # creating problem objective function + solver_run.objective_function.for_problem(self) + logging.info('Creating Constraints...') + # generic constraints + for constraint in solver_run.constraints: + constraint.build(self, solver_run=solver_run, solution=solution) + + # irt target constraints for target in solver_run.objective_function.all_targets(): - target.build_constraint(self, solver_run) + target.constraints(self, solver_run) logging.info('Constraints Created...') except ValueError as error: diff --git a/app/models/solver_run.py b/app/models/solver_run.py index c64897c..a9a6d00 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -1,23 +1,32 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from pulp import lpSum from pydantic import BaseModel from typing import List, Literal, Optional, Union import logging -import random from models.item import Item -from models.constraint import Constraint -from models.metadata_constraint import MetadataConstraint -from models.bundle_constraint import BundleConstraint +from models.constraints.generic_constraint import GenericConstraint +from models.constraints.metadata_constraint import MetadataConstraint +from models.constraints.bundle_constraint import BundleConstraint +from models.constraints.form_uniqueness_constraint import FormUniquenessConstraint +from models.constraints.total_form_items_constraint import TotalFormItemsConstraint from models.irt_model import IRTModel from models.bundle import Bundle from models.objective_function import ObjectiveFunction from models.advanced_options import AdvancedOptions +if TYPE_CHECKING: + from models.solution import Solution + from models.problem import Problem + class SolverRun(BaseModel): items: List[Item] = [] bundles: List[Bundle] = [] bundle_first_ordering: bool = True - constraints: List[Union[Constraint, MetadataConstraint, BundleConstraint]] + constraints: List[Union[GenericConstraint, MetadataConstraint, BundleConstraint, FormUniquenessConstraint, TotalFormItemsConstraint]] irt_model: IRTModel objective_function: ObjectiveFunction total_form_items: int @@ -32,8 +41,16 @@ class SolverRun(BaseModel): # this is all a compensator for dynamically creating objects # ideally we'd change the payload to determine what type it is - constraints: [Constraint|MetadataConstraint|BundleConstraint] = [] + constraints: [GenericConstraint|MetadataConstraint|BundleConstraint|FormUniquenessConstraint|TotalFormItemsConstraint] = [] + # total form items + constraints.append(TotalFormItemsConstraint.create(self.total_form_items)) + + # ensure form uniqueness + if self.advanced_options.ensure_form_uniqueness: + constraints.append(FormUniquenessConstraint.create(self.total_form_items - 1)) + + # repackage to create appropriate constraint types for constraint in self.constraints: if constraint.reference_attribute.type == 'metadata': constraints.append(MetadataConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) @@ -130,17 +147,3 @@ class SolverRun(BaseModel): else: return self.items - def select_items_by_percent(self, percent: int) -> List[Item]: - items = self.unbundled_items() - total_items = len(items) - selected_items_amount = round(total_items - (total_items * - (percent / 100))) - - return random.sample(items, selected_items_amount) - - def select_bundles_by_percent(self, percent: int) -> List[Bundle]: - total_bundles = len(self.bundles) - selected_bundles_amount = round(total_bundles - (total_bundles * - (percent / 100))) - - return random.sample(self.bundles, selected_bundles_amount) diff --git a/app/models/target.py b/app/models/targets/target.py similarity index 100% rename from app/models/target.py rename to app/models/targets/target.py diff --git a/app/models/tcc_target.py b/app/models/targets/tcc_target.py similarity index 91% rename from app/models/tcc_target.py rename to app/models/targets/tcc_target.py index 8e78e91..f7b2368 100644 --- a/app/models/tcc_target.py +++ b/app/models/targets/tcc_target.py @@ -1,13 +1,13 @@ from __future__ import annotations from typing import TYPE_CHECKING -from models.target import * +from models.targets.target import * if TYPE_CHECKING: from models.problem import Problem class TccTarget(Target): - def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + def constraints(self, problem_handler: Problem, solver_run: SolverRun): problem_handler.problem += lpSum([ bundle.trf(solver_run.irt_model, self.theta) * problem_handler.solver_bundles_var[bundle.id] diff --git a/app/models/tif_target.py b/app/models/targets/tif_target.py similarity index 91% rename from app/models/tif_target.py rename to app/models/targets/tif_target.py index 04ed1c6..388091a 100644 --- a/app/models/tif_target.py +++ b/app/models/targets/tif_target.py @@ -1,13 +1,13 @@ from __future__ import annotations from typing import TYPE_CHECKING -from models.target import * +from models.targets.target import * if TYPE_CHECKING: from models.problem import Problem class TifTarget(Target): - def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + def constraints(self, problem_handler: Problem, solver_run: SolverRun): problem_handler.problem += lpSum([ bundle.tif(solver_run.irt_model, self.theta) * problem_handler.solver_bundles_var[bundle.id] diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index 6f63b20..082b10d 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -10,7 +10,7 @@ from models.solver_run import SolverRun from models.solution import Solution from models.problem import Problem from models.form import Form -from models.target import Target +from models.targets.target import Target from services.base import Base @@ -82,8 +82,6 @@ class FormGenerationService(Base): # create problem problem_handler = Problem(items = self.solver_run.unbundled_items(), bundles = self.solver_run.bundles, problem = LpProblem('ata-form-generate', LpMinimize)) problem_handler.generate(solution, self.solver_run) - problem_handler.generate_constraints(self.solver_run, current_drift) - problem = problem_handler.solve() if LpStatus[problem.status] == 'Infeasible':