From 1dbf6723835227b30cbda03e127e5fc7222b43e9 Mon Sep 17 00:00:00 2001 From: Joshua Burman Date: Fri, 10 Nov 2023 18:39:16 -0500 Subject: [PATCH] refactor create constraints for targets --- app/helpers/aws_helper.py | 1 - app/helpers/solver_helper.py | 133 ------------------------ app/models/bundle_constraint.py | 7 +- app/models/constraint.py | 8 +- app/models/irt_target_constraint.py | 51 --------- app/models/metadata_constraint.py | 7 +- app/models/objective_function.py | 13 ++- app/models/problem.py | 51 +-------- app/models/solver_run.py | 8 -- app/models/target.py | 9 ++ app/models/tcc_target.py | 31 ++++++ app/models/tif_target.py | 31 ++++++ app/services/form_generation_service.py | 2 +- 13 files changed, 92 insertions(+), 260 deletions(-) delete mode 100644 app/helpers/solver_helper.py delete mode 100644 app/models/irt_target_constraint.py create mode 100644 app/models/tcc_target.py create mode 100644 app/models/tif_target.py diff --git a/app/helpers/aws_helper.py b/app/helpers/aws_helper.py index 38eb4fe..c3f9a6b 100644 --- a/app/helpers/aws_helper.py +++ b/app/helpers/aws_helper.py @@ -37,6 +37,5 @@ def get_object_tags(key: str, bucket: str) -> list: tags = s3.get_object_tagging(Bucket=bucket, Key=key)['TagSet'] return tags - def file_stream_upload(buffer: io.BytesIO, name: str, bucket: str, action: str = None): return s3.upload_fileobj(buffer, bucket, name, ExtraArgs={'Tagging': f'action={action}'}) diff --git a/app/helpers/solver_helper.py b/app/helpers/solver_helper.py deleted file mode 100644 index 5ed124a..0000000 --- a/app/helpers/solver_helper.py +++ /dev/null @@ -1,133 +0,0 @@ -from pulp import lpSum, LpProblem -from random import randint, sample - -import logging - -from helpers.common_helper import * - -from models.bundle import Bundle -from models.solver_run import SolverRun -from models.item import Item - -from lib.errors.item_generation_error import ItemGenerationError - -# should probably be factored out into a bundle class method or a method in the solver run -def build_constraints(solver_run: SolverRun, problem: LpProblem, - items: list[Item], bundles: list[Bundle], selected_items: list[Item], selected_bundles: list[Bundle], current_drift: int) -> LpProblem: - logging.info('Creating Constraints...') - - try: - total_form_items = solver_run.total_form_items - constraints = solver_run.constraints - - for constraint in constraints: - attribute = constraint.reference_attribute - min = constraint.minimum - max = constraint.maximum - - if attribute.type == 'metadata': - logging.info('Metadata Constraint Generating...') - - problem += lpSum( - [ - len(bundle.items_with_attribute(attribute)) * bundles[bundle.id] for bundle in selected_bundles - ] + - [ - item.attribute_exists(attribute).real * items[item.id] for item in selected_items - ] - ) >= round(total_form_items * (min / 100)), f'{attribute.id} - {attribute.value} - min' - - problem += lpSum( - [ - len(bundle.items_with_attribute(attribute)) * bundles[bundle.id] for bundle in selected_bundles - ] + - [ - item.attribute_exists(attribute).real * items[item.id] for item in selected_items - ] - ) <= round(total_form_items * (max / 100)), f'{attribute.id} - {attribute.value} - max' - elif attribute.type == 'bundle': - logging.info('Bundles Constraint Generating...') - # TODO: account for many different bundle types, since the id condition in L33 could yield duplicates - if selected_bundles != None and len(selected_bundles) > 0: - # make sure the total bundles used in generated form is limited between min-max set - problem += lpSum([ - bundles[bundle.id] for bundle in selected_bundles - ]) == randint(int(constraint.minimum), - int(constraint.maximum)) - - logging.info('Constraints Created...') - - # Behold our very own Elastic constraints! - for tif_target in solver_run.objective_function.tif_targets: - problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.iif(solver_run, tif_target.theta) * - items[item.id] - for item in selected_items - ]) >= tif_target.minimum( - ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.iif(solver_run, tif_target.theta) * - items[item.id] - for item in selected_items - ]) <= tif_target.maximum( - ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - - for tcc_target in solver_run.objective_function.tcc_targets: - problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.irf(solver_run, tcc_target.theta) * - items[item.id] - for item in selected_items - ]) >= tcc_target.minimum( - ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * bundles[bundle.id] - for bundle in selected_bundles - ] + [ - item.irf(solver_run, tcc_target.theta) * - items[item.id] - for item in selected_items - ]) <= tcc_target.maximum( - ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - - return problem - except ValueError as error: - logging.error(error) - raise ItemGenerationError( - "Bundle min and/or max larger than bundle amount provided", - error.args[0]) - -# should probably be factored out into a bundle class method or a method in the solver run -def get_random_bundles(total_form_items: int, - bundles: list[Bundle], - min: int, - max: int, - found_bundles=False) -> list[Bundle]: - selected_bundles = None - total_bundle_items = 0 - total_bundles = randint(min, max) - logging.info(f'Selecting Bundles (total of {total_bundles})...') - - while found_bundles == False: - selected_bundles = sample(bundles, total_bundles) - total_bundle_items = sum(bundle.count for bundle in selected_bundles) - - if total_bundle_items <= total_form_items: - found_bundles = True - - if found_bundles == True: - return selected_bundles - else: - return get_random_bundles(total_form_items, total_bundles - 1, bundles) diff --git a/app/models/bundle_constraint.py b/app/models/bundle_constraint.py index 0b34dc8..5ef0afa 100644 --- a/app/models/bundle_constraint.py +++ b/app/models/bundle_constraint.py @@ -1,11 +1,6 @@ -import logging - from random import randint -from pulp import lpSum - -from models.constraint import Constraint -from models.problem import Problem +from models.constraint import * class BundleConstraint(Constraint): def build(self, problem_handler: Problem, _) -> None: diff --git a/app/models/constraint.py b/app/models/constraint.py index c60f80b..abb06a2 100644 --- a/app/models/constraint.py +++ b/app/models/constraint.py @@ -1,12 +1,14 @@ +import logging + +from pulp import lpSum from pydantic import BaseModel + from helpers.common_helper import * from models.attribute import Attribute +from models.problem import Problem class Constraint(BaseModel): reference_attribute: Attribute minimum: float maximum: float - - def __init__(self, **data) -> None: - super().__init__(**data) diff --git a/app/models/irt_target_constraint.py b/app/models/irt_target_constraint.py deleted file mode 100644 index c83a2a5..0000000 --- a/app/models/irt_target_constraint.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations -from typing import TYPE_CHECKING, Optional - -from pulp import lpSum - -from models import Constraint, Problem, Attribute, Target, Item, Bundle - -if TYPE_CHECKING: - from models.solver_run import SolverRun - -class IrtTargetConstraint(Constraint): - reference_attribute: Optional[Attribute] - minimum: Optional[float] - maximum: Optional[float] - target: Target - target_type: str - - def build(self, problem_handler: Problem, solver_run: SolverRun): - problem_handler.problem += lpSum([ - self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) - * problem_handler.solver_bundles_var[bundle.id] - for bundle in problem_handler.bundles - ] + [ - self.item_irt_function(item, solver_run.irt_model, self.target.theta) * - problem_handler.solver_items_var[item.id] - for item in problem_handler.items - ]) >= self.target.minimum( - ), f'Min {self.target_type} theta({self.target.theta}) at target {self.target.value}' - - problem_handler.problem += lpSum([ - self.bundle_irt_function(bundle, solver_run.irt_model, self.target.theta) - * problem_handler.solver_bundles_var[bundle.id] - for bundle in problem_handler.bundles - ] + [ - self.item_irt_function(item, solver_run.irt_model, self.target.theta) * - problem_handler.solver_items_var[item.id] - for item in problem_handler.items - ]) <= self.target.maximum( - ), f'Max {self.target_type} theta({self.target.theta}) at target {self.target.value}' - - def item_irt_function(self, item: Item, irt_model: str, theta: float) -> float: - if self.target_type == 'tcc': - return item.irf(irt_model, theta) - elif self.target_type == 'tif': - return item.iif(irt_model, theta) - - def bundle_irt_function(self, bundle: Bundle, irt_model: str, theta: float) -> float: - if self.target_type == 'tcc': - return bundle.trf(irt_model, theta) - elif self.target_type == 'tif': - return bundle.tif(irt_model, theta) \ No newline at end of file diff --git a/app/models/metadata_constraint.py b/app/models/metadata_constraint.py index d9221c9..7284030 100644 --- a/app/models/metadata_constraint.py +++ b/app/models/metadata_constraint.py @@ -1,12 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -import logging - -from pulp import lpSum - -from models.constraint import Constraint -from models.problem import Problem +from models.constraint import * if TYPE_CHECKING: from models.solver_run import SolverRun diff --git a/app/models/objective_function.py b/app/models/objective_function.py index 9bd332f..3941434 100644 --- a/app/models/objective_function.py +++ b/app/models/objective_function.py @@ -1,15 +1,18 @@ +from __future__ import annotations + from pydantic import BaseModel from typing import Dict, List, AnyStr -from models.target import Target +from models.tif_target import TifTarget +from models.tcc_target import TccTarget class ObjectiveFunction(BaseModel): # minimizing tif/tcc target value is only option currently # as we add more we can build this out to be more dynamic # likely with models representing each objective function type - tif_targets: List[Target] - tcc_targets: List[Target] + tif_targets: List[TifTarget] + tcc_targets: List[TccTarget] target_variance_percentage: int = 10 objective: AnyStr = "minimize" weight: Dict = {'tif': 1, 'tcc': 1} @@ -18,7 +21,7 @@ class ObjectiveFunction(BaseModel): limit: float or bool, all: bool = False, amount: float = 0.1, - targets: list[Target] = []) -> bool: + targets: list[TifTarget|TccTarget] = []) -> bool: if all: for target in self.tif_targets: target.drift = round(target.drift + amount, 2) @@ -44,5 +47,5 @@ class ObjectiveFunction(BaseModel): return minimum_drift - def all_targets(self) -> list[Target]: + def all_targets(self) -> list[TifTarget|TccTarget]: return self.tif_targets + self.tcc_targets diff --git a/app/models/problem.py b/app/models/problem.py index 1fc2923..13f5ee0 100644 --- a/app/models/problem.py +++ b/app/models/problem.py @@ -51,9 +51,9 @@ class Problem(BaseModel): self.problem.solve() return self.problem - def generate(self, solution: Solution, solver_run: SolverRun): + def generate(self, solution: Solution, solver_run: SolverRun) -> None: - # Form Constraints + # Objective Function self.problem += lpSum( [ bundle.count * self.solver_bundles_var[bundle.id] @@ -76,56 +76,15 @@ class Problem(BaseModel): ] ) <= solver_run.total_form_items - 1, f'Ensuring uniqueness for form' - def generate_constraints(self, solver_run: SolverRun, current_drift: int): + def generate_constraints(self, solver_run: SolverRun, current_drift: int) -> None: logging.info('Creating Constraints...') try: for constraint in solver_run.constraints: constraint.build(self, solver_run) - - for tif_target in solver_run.objective_function.tif_targets: - self.problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.iif(solver_run.irt_model, tif_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) >= tif_target.minimum( - ), f'Min TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - self.problem += lpSum([ - bundle.tif(solver_run.irt_model, tif_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.iif(solver_run.irt_model, tif_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) <= tif_target.maximum( - ), f'Max TIF theta({tif_target.theta}) at target {tif_target.value} drift at {current_drift}%' - for tcc_target in solver_run.objective_function.tcc_targets: - self.problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.irf(solver_run.irt_model, tcc_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) >= tcc_target.minimum( - ), f'Min TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' - self.problem += lpSum([ - bundle.trf(solver_run.irt_model, tcc_target.theta) - * self.solver_bundles_var[bundle.id] - for bundle in self.bundles - ] + [ - item.irf(solver_run.irt_model, tcc_target.theta) * - self.solver_items_var[item.id] - for item in self.items - ]) <= tcc_target.maximum( - ), f'Max TCC theta({tcc_target.theta}) at target {tcc_target.value} drift at {current_drift}%' + for target in solver_run.objective_function.all_targets(): + target.build_constraint(self, solver_run) logging.info('Constraints Created...') except ValueError as error: diff --git a/app/models/solver_run.py b/app/models/solver_run.py index 439fb81..c64897c 100644 --- a/app/models/solver_run.py +++ b/app/models/solver_run.py @@ -8,7 +8,6 @@ from models.item import Item from models.constraint import Constraint from models.metadata_constraint import MetadataConstraint from models.bundle_constraint import BundleConstraint -# from models.irt_target_constraint import IrtTargetConstraint from models.irt_model import IRTModel from models.bundle import Bundle from models.objective_function import ObjectiveFunction @@ -41,13 +40,6 @@ class SolverRun(BaseModel): elif constraint.reference_attribute.type == 'bundle': constraints.append(BundleConstraint(reference_attribute=constraint.reference_attribute, minimum=constraint.minimum, maximum=constraint.maximum)) - # constraints for tif and tcc targets - # for target in self.objective_function.tif_targets: - # constraints.append(IrtTargetConstraint(target=target, target_type='tif')) - - # for target in self.objective_function.tcc_targets: - # constraints.append(IrtTargetConstraint(target=target, target_type='tcc')) - self.constraints = constraints def get_item(self, item_id: int) -> Item or None: diff --git a/app/models/target.py b/app/models/target.py index 1ad9fa4..bf9e1ea 100644 --- a/app/models/target.py +++ b/app/models/target.py @@ -1,6 +1,15 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + from pydantic import BaseModel from typing import Optional +from pulp import lpSum + +if TYPE_CHECKING: + from models.solver_run import SolverRun + from models.problem import Problem + class Target(BaseModel): theta: float value: float diff --git a/app/models/tcc_target.py b/app/models/tcc_target.py new file mode 100644 index 0000000..6c29181 --- /dev/null +++ b/app/models/tcc_target.py @@ -0,0 +1,31 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.target import * + +if TYPE_CHECKING: + from models.problem import Problem + +class TccTarget(Target): + def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + problem_handler.problem += lpSum([ + bundle.trf(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.irf(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) >= self.minimum( + ), f'Min TCC theta({self.theta}) at target {self.value} with a drift % of {self.drift}' + + problem_handler.problem += lpSum([ + bundle.trf(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.irf(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) <= self.maximum( + ), f'Max TCC theta({self.theta}) at target {self.value} with a drift % of {self.drift}' \ No newline at end of file diff --git a/app/models/tif_target.py b/app/models/tif_target.py new file mode 100644 index 0000000..fac10c2 --- /dev/null +++ b/app/models/tif_target.py @@ -0,0 +1,31 @@ +from __future__ import annotations +from typing import TYPE_CHECKING + +from models.target import * + +if TYPE_CHECKING: + from models.problem import Problem + +class TifTarget(Target): + def build_constraint(self, problem_handler: Problem, solver_run: SolverRun): + problem_handler.problem += lpSum([ + bundle.tif(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.iif(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) >= self.minimum( + ), f'Min TIF theta({self.theta}) at target {self.value} with a drift % of {self.drift}' + + problem_handler.problem += lpSum([ + bundle.tif(solver_run.irt_model, self.theta) + * problem_handler.solver_bundles_var[bundle.id] + for bundle in problem_handler.bundles + ] + [ + item.iif(solver_run.irt_model, self.theta) * + problem_handler.solver_items_var[item.id] + for item in problem_handler.items + ]) <= self.maximum( + ), f'Max TIF theta({self.theta}) at target {self.value} with a drift % of {self.drift}' \ No newline at end of file diff --git a/app/services/form_generation_service.py b/app/services/form_generation_service.py index bdf813f..a5bcaf8 100644 --- a/app/services/form_generation_service.py +++ b/app/services/form_generation_service.py @@ -3,7 +3,7 @@ import json, random, io, logging from pulp import LpProblem, LpVariable, LpMinimize, LpStatus, lpSum from lib.application_configs import ApplicationConfigs -from helpers import aws_helper, tar_helper, csv_helper, service_helper, solver_helper +from helpers import aws_helper, tar_helper, csv_helper, service_helper from lib.errors.item_generation_error import ItemGenerationError from models.solver_run import SolverRun