the big format
This commit is contained in:
Joshua Burman
@ -1,4 +1,5 @@
|
||||
class Base:
|
||||
def __init__(self, source, ingest_type='message'):
|
||||
self.ingest_type = ingest_type
|
||||
self.source = source
|
||||
|
||||
def __init__(self, source, ingest_type='message'):
|
||||
self.ingest_type = ingest_type
|
||||
self.source = source
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import os, json, random, io, logging
|
||||
|
||||
from pulp import LpProblem, LpVariable, LpMinimize, LpStatus, lpSum
|
||||
from pulp import LpProblem, LpVariable, LpMinimize, LpMaximize, LpStatus, lpSum
|
||||
|
||||
from helpers import aws_helper, tar_helper, csv_helper, service_helper, solver_helper
|
||||
from lib.errors.item_generation_error import ItemGenerationError
|
||||
@ -12,134 +12,233 @@ from models.item import Item
|
||||
|
||||
from services.base import Base
|
||||
|
||||
|
||||
class LoftService(Base):
|
||||
def process(self):
|
||||
try:
|
||||
self.solver_run = self.create_solver_run_from_attributes()
|
||||
self.solver_run.generate_bundles()
|
||||
self.solution = self.generate_solution()
|
||||
self.result = self.stream_to_s3_bucket()
|
||||
except ItemGenerationError as error:
|
||||
self.result = self.stream_to_s3_bucket(error)
|
||||
except TypeError as error:
|
||||
logging.error(error)
|
||||
self.result = self.stream_to_s3_bucket(ItemGenerationError("Provided params causing error in calculation results"))
|
||||
|
||||
def create_solver_run_from_attributes(self) -> SolverRun:
|
||||
logging.info('Retrieving attributes from message...')
|
||||
# get s3 object
|
||||
self.key = aws_helper.get_key_from_message(self.source)
|
||||
s3_object = aws_helper.get_object(self.key, aws_helper.get_bucket_from_message(self.source))
|
||||
def process(self):
|
||||
try:
|
||||
self.solver_run = self.create_solver_run_from_attributes()
|
||||
self.solver_run.generate_bundles()
|
||||
self.solution = self.generate_solution()
|
||||
# self.solution = self.generate_test_solution()
|
||||
self.result = self.stream_to_s3_bucket()
|
||||
except ItemGenerationError as error:
|
||||
self.result = self.stream_to_s3_bucket(error)
|
||||
except TypeError as error:
|
||||
logging.error(error)
|
||||
self.result = self.stream_to_s3_bucket(
|
||||
ItemGenerationError(
|
||||
"Provided params causing error in calculation results"))
|
||||
|
||||
# convert to tar
|
||||
self.tar = tar_helper.raw_to_tar(s3_object)
|
||||
def generate_test_solution(self) -> Solution:
|
||||
solution = Solution(response_id=random.randint(100, 5000), forms=[])
|
||||
|
||||
# get attributes file and convert to dict
|
||||
attributes = json.loads(tar_helper.extract_file_from_tar(self.tar , 'solver_run_attributes.json').read())
|
||||
problem = LpProblem("ata-form-generate-with-bundles", LpMinimize)
|
||||
|
||||
# create solver run
|
||||
solver_run = SolverRun.parse_obj(attributes)
|
||||
bundles = LpVariable.dicts(
|
||||
"Bundle", [bundle.id for bundle in self.solver_run.bundles],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
items = LpVariable.dicts("Item",
|
||||
[item.id for item in self.solver_run.items],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
|
||||
# get items file and convert to dict
|
||||
items_csv = tar_helper.extract_file_from_tar(self.tar , 'items.csv')
|
||||
items_csv_reader = csv_helper.file_stream_reader(items_csv)
|
||||
problem += lpSum(
|
||||
[bundles[bundle.id] for bundle in self.solver_run.bundles])
|
||||
# problem += lpSum([items[item.id] for item in self.solver_run.items])
|
||||
|
||||
# add items to solver run
|
||||
for item in service_helper.items_csv_to_dict(items_csv_reader, solver_run):
|
||||
solver_run.items.append(Item.parse_obj(item))
|
||||
# problem += lpSum([bundles[bundle.id] for bundle in self.solver_run.bundles]) <= 3, 'max total bundles used'
|
||||
# problem += lpSum([bundles[bundle.id] for bundle in self.solver_run.bundles]) >= 1, 'min total bundles used'
|
||||
problem += lpSum(
|
||||
[bundles[bundle.id] for bundle in self.solver_run.bundles]) == 3
|
||||
|
||||
logging.info('Processed Attributes...')
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.count * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] +
|
||||
[1 * items[item.id] for item in self.solver_run.unbundled_items()]
|
||||
) == self.solver_run.total_form_items, 'Total bundle form items for form'
|
||||
|
||||
return solver_run
|
||||
problem.solve()
|
||||
|
||||
def generate_solution(self) -> Solution:
|
||||
logging.info('Generating Solution...')
|
||||
# for v in problem.variables():
|
||||
# print(f'{v.name}: {v.varValue}')
|
||||
|
||||
# unsolved solution
|
||||
solution = Solution(
|
||||
response_id=random.randint(100, 5000),
|
||||
forms=[]
|
||||
)
|
||||
# add return items and create as a form
|
||||
form_items = service_helper.solution_items(problem.variables(),
|
||||
self.solver_run)
|
||||
|
||||
# counter for number of forms
|
||||
f = 0
|
||||
# add form to solution
|
||||
solution.forms.append(
|
||||
Form.create(form_items, self.solver_run, LpStatus[problem.status]))
|
||||
logging.info('Form generated and added to solution...')
|
||||
|
||||
# iterate for number of forms that require creation
|
||||
# currently creates distinc forms with no item overlap
|
||||
while f < self.solver_run.total_forms:
|
||||
# setup vars
|
||||
items = LpVariable.dicts(
|
||||
"Item", [item.id for item in self.solver_run.items], lowBound=1, upBound=1, cat='Binary')
|
||||
# bundles = LpVariable.dicts(
|
||||
# "Bundle", [bundle.id for bundle in self.solver_run.bundles], lowBound=1, upBound=1, cat='Binary')
|
||||
return solution
|
||||
|
||||
problem_objection_functions = []
|
||||
def create_solver_run_from_attributes(self) -> SolverRun:
|
||||
logging.info('Retrieving attributes from message...')
|
||||
# get s3 object
|
||||
self.key = aws_helper.get_key_from_message(self.source)
|
||||
s3_object = aws_helper.get_object(
|
||||
self.key, aws_helper.get_bucket_from_message(self.source))
|
||||
|
||||
# create problem
|
||||
problem = LpProblem("ata-form-generate", LpMinimize)
|
||||
# convert to tar
|
||||
self.tar = tar_helper.raw_to_tar(s3_object)
|
||||
|
||||
# dummy objective function, because it just makes things easier™
|
||||
# problem += lpSum([items[item.id]
|
||||
# for item in self.solver_run.items])
|
||||
# get attributes file and convert to dict
|
||||
attributes = json.loads(
|
||||
tar_helper.extract_file_from_tar(
|
||||
self.tar, 'solver_run_attributes.json').read())
|
||||
|
||||
# constraints
|
||||
problem += lpSum([items[item.id]
|
||||
for item in self.solver_run.items]) == self.solver_run.total_form_items, 'Total form items'
|
||||
# create solver run
|
||||
solver_run = SolverRun.parse_obj(attributes)
|
||||
|
||||
# dynamic constraints
|
||||
problem = solver_helper.build_constraints(self.solver_run, problem, items)
|
||||
# get items file and convert to dict
|
||||
items_csv = tar_helper.extract_file_from_tar(self.tar, 'items.csv')
|
||||
items_csv_reader = csv_helper.file_stream_reader(items_csv)
|
||||
|
||||
# multi-objective constraints
|
||||
logging.info('Creating TIF and TCC constraints')
|
||||
for target in self.solver_run.objective_function.tif_targets:
|
||||
tif = lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items])
|
||||
problem_objection_functions.append(tif)
|
||||
problem += lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) >= target.value - 8, f'max tif theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) <= target.value + 8, f'min tif theta ({target.theta}) target value {target.value}'
|
||||
# add items to solver run
|
||||
for item in service_helper.items_csv_to_dict(items_csv_reader,
|
||||
solver_run):
|
||||
solver_run.items.append(Item.parse_obj(item))
|
||||
|
||||
for target in self.solver_run.objective_function.tcc_targets:
|
||||
tcc = lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items])
|
||||
problem_objection_functions.append(tcc)
|
||||
problem += lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) >= target.value - 20, f'max tcc theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
for item in self.solver_run.items]) <= target.value + 20, f'min tcc theta ({target.theta}) target value {target.value}'
|
||||
logging.info('Processed Attributes...')
|
||||
|
||||
# solve problem
|
||||
logging.info('Solving...')
|
||||
# problem.solve()
|
||||
problem.sequentialSolve(problem_objection_functions)
|
||||
logging.info('Solved...generating form and adding to solution')
|
||||
return solver_run
|
||||
|
||||
# add return items and create as a form
|
||||
form_items = service_helper.solution_items(problem.variables(), self.solver_run)
|
||||
def generate_solution(self) -> Solution:
|
||||
logging.info('Generating Solution...')
|
||||
|
||||
# add form to solution
|
||||
solution.forms.append(Form.create(form_items, self.solver_run, LpStatus[problem.status]))
|
||||
logging.info('Form generated and added to solution...')
|
||||
# unsolved solution
|
||||
solution = Solution(response_id=random.randint(100, 5000), forms=[])
|
||||
|
||||
# successfull form, increment
|
||||
f += 1
|
||||
# counter for number of forms
|
||||
f = 0
|
||||
|
||||
logging.info('Solution Generated.')
|
||||
return solution
|
||||
# iterate for number of forms that require creation
|
||||
# currently creates distinc forms with no item overlap
|
||||
while f < self.solver_run.total_forms:
|
||||
# setup vars
|
||||
items = LpVariable.dicts(
|
||||
"Item", [item.id for item in self.solver_run.items],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
bundles = LpVariable.dicts(
|
||||
"Bundle", [bundle.id for bundle in self.solver_run.bundles],
|
||||
lowBound=1,
|
||||
upBound=1,
|
||||
cat='Binary')
|
||||
|
||||
def stream_to_s3_bucket(self, error = None):
|
||||
self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv'
|
||||
solution_file = None
|
||||
# setup writer buffer and write processed forms to file
|
||||
buffer = io.StringIO()
|
||||
# problem_objection_functions = []
|
||||
|
||||
if error:
|
||||
logging.info('Streaming %s error response to s3 bucket - %s', self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.error_to_file(buffer, error)
|
||||
else:
|
||||
logging.info('Streaming %s to s3 bucket - %s', self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.solution_to_file(buffer, self.solver_run.total_form_items, self.solution.forms)
|
||||
# create problem
|
||||
problem = LpProblem("ata-form-generate", LpMinimize)
|
||||
|
||||
# upload generated file to s3 and return result
|
||||
return aws_helper.file_stream_upload(solution_file, self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
# dummy objective function, because it just makes things easier™
|
||||
problem += lpSum(
|
||||
[items[item.id] for item in self.solver_run.items])
|
||||
|
||||
# constraints
|
||||
# problem += lpSum([items[item.id]
|
||||
# for item in self.solver_run.items]) == self.solver_run.total_form_items, 'Total form items'
|
||||
problem += lpSum(
|
||||
[
|
||||
bundle.count * bundles[bundle.id]
|
||||
for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
1 * items[item.id]
|
||||
for item in self.solver_run.unbundled_items()
|
||||
]
|
||||
) == self.solver_run.total_form_items, 'Total bundle form items for form'
|
||||
|
||||
# dynamic constraints
|
||||
problem = solver_helper.build_constraints(self.solver_run, problem,
|
||||
items, bundles)
|
||||
|
||||
# multi-objective constraints
|
||||
logging.info('Creating TIF and TCC constraints')
|
||||
for target in self.solver_run.objective_function.tif_targets:
|
||||
|
||||
# tif = lpSum([item.iif(self.solver_run, target.theta)*items[item.id]
|
||||
# for item in self.solver_run.items])
|
||||
# problem_objection_functions.append(tif)
|
||||
problem += lpSum([
|
||||
bundle.tif(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.iif(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) >= target.value - 8, f'max tif theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([
|
||||
bundle.tif(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.iif(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) <= target.value + 8, f'min tif theta ({target.theta}) target value {target.value}'
|
||||
|
||||
for target in self.solver_run.objective_function.tcc_targets:
|
||||
# tcc = lpSum([item.irf(self.solver_run, target.theta)*items[item.id]
|
||||
# for item in self.solver_run.items])
|
||||
# problem_objection_functions.append(tcc)
|
||||
problem += lpSum([
|
||||
bundle.trf(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.irf(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) >= target.value - 20, f'max tcc theta ({target.theta}) target value {target.value}'
|
||||
problem += lpSum([
|
||||
bundle.trf(self.solver_run.irt_model, target.theta) *
|
||||
bundles[bundle.id] for bundle in self.solver_run.bundles
|
||||
] + [
|
||||
item.irf(self.solver_run, target.theta) * items[item.id]
|
||||
for item in self.solver_run.items
|
||||
]) <= target.value + 20, f'min tcc theta ({target.theta}) target value {target.value}'
|
||||
|
||||
# solve problem
|
||||
logging.info('Solving...')
|
||||
problem.solve()
|
||||
# problem.sequentialSolve(problem_objection_functions)
|
||||
logging.info('Solved...generating form and adding to solution')
|
||||
|
||||
# add return items and create as a form
|
||||
form_items = service_helper.solution_items(problem.variables(),
|
||||
self.solver_run)
|
||||
|
||||
# add form to solution
|
||||
solution.forms.append(
|
||||
Form.create(form_items, self.solver_run,
|
||||
LpStatus[problem.status]))
|
||||
logging.info('Form generated and added to solution...')
|
||||
|
||||
# successfull form, increment
|
||||
f += 1
|
||||
|
||||
logging.info('Solution Generated.')
|
||||
return solution
|
||||
|
||||
def stream_to_s3_bucket(self, error=None):
|
||||
self.file_name = f'{service_helper.key_to_uuid(self.key)}.csv'
|
||||
solution_file = None
|
||||
# setup writer buffer and write processed forms to file
|
||||
buffer = io.StringIO()
|
||||
|
||||
if error:
|
||||
logging.info('Streaming %s error response to s3 bucket - %s',
|
||||
self.file_name, os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.error_to_file(buffer, error)
|
||||
else:
|
||||
logging.info('Streaming %s to s3 bucket - %s', self.file_name,
|
||||
os.environ['S3_PROCESSED_BUCKET'])
|
||||
solution_file = service_helper.solution_to_file(
|
||||
buffer, self.solver_run.total_form_items, self.solution.forms)
|
||||
|
||||
# upload generated file to s3 and return result
|
||||
return aws_helper.file_stream_upload(solution_file, self.file_name,
|
||||
os.environ['S3_PROCESSED_BUCKET'])
|
||||
|
||||
Reference in New Issue
Block a user