diff --git a/pyomo/contrib/observer/__init__.py b/pyomo/contrib/observer/__init__.py new file mode 100644 index 00000000000..6eb9ea8b81d --- /dev/null +++ b/pyomo/contrib/observer/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py new file mode 100644 index 00000000000..fced330fdbf --- /dev/null +++ b/pyomo/contrib/observer/component_collector.py @@ -0,0 +1,118 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NegationExpression, + PowExpression, + MaxExpression, + MinExpression, + ProductExpression, + MonomialTermExpression, + DivisionExpression, + SumExpression, + Expr_ifExpression, + UnaryFunctionExpression, + AbsExpression, +) +from pyomo.core.expr.relational_expr import ( + RangedExpression, + InequalityExpression, + EqualityExpression, +) +from pyomo.core.base.var import VarData, ScalarVar +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.expression import ExpressionData, ScalarExpression +from pyomo.repn.util import ExitNodeDispatcher +from pyomo.common.numeric_types import native_numeric_types +from pyomo.common.collections import ComponentSet + + +def handle_var(node, collector): + collector.variables.add(node) + return None + + +def handle_param(node, collector): + collector.params.add(node) + return None + + +def handle_named_expression(node, collector): + collector.named_expressions.add(node) + return None + + +def handle_external_function(node, collector): + collector.external_functions.add(node) + return None + + +def handle_skip(node, collector): + return None + + +collector_handlers = ExitNodeDispatcher() +collector_handlers[VarData] = handle_var +collector_handlers[ParamData] = handle_param +collector_handlers[ExpressionData] = handle_named_expression +collector_handlers[ScalarExpression] = handle_named_expression +collector_handlers[ExternalFunctionExpression] = handle_external_function +collector_handlers[NegationExpression] = handle_skip +collector_handlers[PowExpression] = handle_skip +collector_handlers[MaxExpression] = handle_skip +collector_handlers[MinExpression] = handle_skip +collector_handlers[ProductExpression] = handle_skip +collector_handlers[MonomialTermExpression] = handle_skip +collector_handlers[DivisionExpression] = handle_skip +collector_handlers[SumExpression] = handle_skip +collector_handlers[Expr_ifExpression] = handle_skip +collector_handlers[UnaryFunctionExpression] = handle_skip +collector_handlers[AbsExpression] = handle_skip +collector_handlers[RangedExpression] = handle_skip +collector_handlers[InequalityExpression] = handle_skip +collector_handlers[EqualityExpression] = handle_skip + + +class _ComponentFromExprCollector(StreamBasedExpressionVisitor): + def __init__(self, **kwds): + self.named_expressions = ComponentSet() + self.variables = ComponentSet() + self.params = ComponentSet() + self.external_functions = ComponentSet() + super().__init__(**kwds) + + def exitNode(self, node, data): + if type(node) in native_numeric_types: + # we need this here to handle numpy + # (we can't put numpy in the dispatcher?) + return None + return collector_handlers[node.__class__](node, self) + + def beforeChild(self, node, child, child_idx): + if child in self.named_expressions: + return False, None + return True, None + + +_visitor = _ComponentFromExprCollector() + + +def collect_components_from_expr(expr): + _visitor.__init__() + _visitor.walk_expression(expr) + return ( + _visitor.named_expressions, + _visitor.variables, + _visitor.params, + _visitor.external_functions, + ) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py new file mode 100644 index 00000000000..8fa6b597cd2 --- /dev/null +++ b/pyomo/contrib/observer/model_observer.py @@ -0,0 +1,1294 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# __________________________________________________________________________ + +from __future__ import annotations +import abc +from typing import ( + List, + Sequence, + Optional, + Mapping, + MutableMapping, + MutableSet, + Tuple, + Collection, + Union, +) + +from pyomo.common.enums import ObjectiveSense +from pyomo.common.config import ConfigDict, ConfigValue, document_configdict +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.var import VarData +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.expression import ExpressionData +from pyomo.core.base.objective import ObjectiveData, Objective +from pyomo.core.base.block import BlockData, Block +from pyomo.core.base.suffix import Suffix +from pyomo.core.base.component import ActiveComponent +from pyomo.core.expr.numeric_expr import NumericValue +from pyomo.core.expr.relational_expr import RelationalExpression +from pyomo.common.collections import ( + ComponentMap, + ComponentSet, + OrderedSet, + DefaultComponentMap, +) +from pyomo.common.gc_manager import PauseGC +from pyomo.common.timing import HierarchicalTimer +from pyomo.contrib.solver.common.util import get_objective +from pyomo.contrib.observer.component_collector import collect_components_from_expr +from pyomo.common.numeric_types import native_numeric_types +import warnings +import enum +from collections import defaultdict + + +# The ModelChangeDetector is meant to be used to automatically identify changes +# in a Pyomo model or block. Here is a list of changes that will be detected. +# Note that inactive components (e.g., constraints) are treated as "removed". +# - new constraints that have been added to the model +# - constraints that have been removed from the model +# - new variables that have been detected in new or modified constraints/objectives +# - old variables that are no longer used in any constraints/objectives +# - new parameters that have been detected in new or modified constraints/objectives +# - old parameters that are no longer used in any constraints/objectives +# - new objectives that have been added to the model +# - objectives that have been removed from the model +# - modified constraint expressions (relies on expressions being immutable) +# - modified objective expressions (relies on expressions being immutable) +# - modified objective sense +# - changes to variable bounds, domains, "fixed" flags, and values for fixed variables +# - changes to named expressions (relies on expressions being immutable) +# - changes to parameter values + + +_param_types = {ParamData, ScalarParam} + + +@document_configdict() +class AutoUpdateConfig(ConfigDict): + """ + Control which parts of the model are automatically checked and/or updated upon re-solve + """ + + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + if doc is None: + doc = 'Configuration options to detect changes in model between solves' + super().__init__( + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + + #: automatically detect new/removed constraints on subsequent solves + self.check_for_new_or_removed_constraints: bool = self.declare( + 'check_for_new_or_removed_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old constraints will not be automatically detected on + subsequent solves. Use False only when manually updating the change + detector with cd.add_constraints() and cd.remove_constraints() or + when you are certain constraints are not being added to/removed from the + model.""", + ), + ) + #: automatically detect new/removed objectives on subsequent solves + self.check_for_new_or_removed_objectives: bool = self.declare( + 'check_for_new_or_removed_objectives', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old objectives will not be automatically detected on + subsequent solves. Use False only when manually updating the solver + with opt.add_objectives() and opt.remove_objectives() or when you + are certain objectives are not being added to/removed from the + model.""", + ), + ) + #: automatically detect changes to constraints on subsequent solves + self.update_constraints: bool = self.declare( + 'update_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing constraints will not be automatically + detected on subsequent solves. This includes changes to the lower, + body, and upper attributes of constraints. Use False only when + manually updating the solver with opt.remove_constraints() and + opt.add_constraints() or when you are certain constraints are not + being modified.""", + ), + ) + #: automatically detect changes to variables on subsequent solves + self.update_vars: bool = self.declare( + 'update_vars', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing variables will not be automatically + detected on subsequent solves. This includes changes to the lb, ub, + domain, and fixed attributes of variables. Use False only when + manually updating the observer with opt.update_variables() or when + you are certain variables are not being modified.""", + ), + ) + #: automatically detect changes to parameters on subsequent solves + self.update_parameters: bool = self.declare( + 'update_parameters', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to parameter values and fixed variable values will + not be automatically detected on subsequent solves. Use False only + when manually updating the observer with + opt.update_parameters_and_fixed_variables() or when you are certain + parameters are not being modified.""", + ), + ) + #: automatically detect changes to named expressions on subsequent solves + self.update_named_expressions: bool = self.declare( + 'update_named_expressions', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to Expressions will not be automatically detected on + subsequent solves. Use False only when manually updating the solver + with opt.remove_constraints() and opt.add_constraints() or when you + are certain Expressions are not being modified.""", + ), + ) + #: automatically detect changes to objectives on subsequent solves + self.update_objectives: bool = self.declare( + 'update_objectives', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to objectives will not be automatically detected on + subsequent solves. This includes the expr and sense attributes of + objectives. Use False only when manually updating the solver with + opt.set_objective() or when you are certain objectives are not being + modified.""", + ), + ) + + +class Reason(enum.Flag): + no_change = 0 + bounds = 1 + fixed = 2 + domain = 4 + value = 8 + added = 16 + removed = 32 + expr = 64 + sense = 128 + sos_items = 256 + + +class Observer(abc.ABC): + @abc.abstractmethod + def _update_variables(self, variables: Mapping[VarData, Reason]): + """ + This method gets called by the ModelChangeDetector when there are + any modifications to the set of "active" variables in the model being + observed. By "active" variables, we mean variables + that are used within an active component such as a constraint or + an objective. Changes include new variables being added to the model, + variables being removed from the model, or changes to variables + already in the model + + Parameters + ---------- + variables: Mapping[VarData, Reason] + The variables and what changed about them + """ + pass + + @abc.abstractmethod + def _update_parameters(self, params: Mapping[ParamData, Reason]): + """ + This method gets called by the ModelChangeDetector when there are any + modifications to the set of "active" parameters in the model being + observed. By "active" parameters, we mean parameters that are used within + an active component such as a constraint or an objective. Changes include + parameters being added to the model, parameters being removed from the model, + or changes to parameters already in the model + + Parameters + ---------- + params: Mapping[ParamData, Reason] + The parameters and what changed about them + """ + pass + + @abc.abstractmethod + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + """ + This method gets called by the ModelChangeDetector when there are any + modifications to the set of active constraints in the model being observed. + Changes include constraints being added to the model, constraints being + removed from the model, or changes to constraints already in the model. + + Parameters + ---------- + cons: Mapping[ConstraintData, Reason] + The constraints and what changed about them + """ + pass + + @abc.abstractmethod + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + """ + This method gets called by the ModelChangeDetector when there are any + modifications to the set of active SOS constraints in the model being + observed. Changes include constraints being added to the model, constraints + being removed from the model, or changes to constraints already in the model. + + Parameters + ---------- + cons: Mapping[SOSConstraintData, Reason] + The SOS constraints and what changed about them + """ + pass + + @abc.abstractmethod + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + """ + This method gets called by the ModelChangeDetector when there are any + modifications to the set of active objectives in the model being observed. + Changes include objectives being added to the model, objectives being + removed from the model, or changes to objectives already in the model. + + Parameters + ---------- + objs: Mapping[ObjectiveData, Reason] + The objectives and what changed about them + """ + pass + + +def _default_reason(): + return Reason.no_change + + +class _Updates: + def __init__(self, observers: Collection[Observer]) -> None: + self.vars_to_update = DefaultComponentMap(_default_reason) + self.params_to_update = DefaultComponentMap(_default_reason) + self.cons_to_update = defaultdict(_default_reason) + self.sos_to_update = defaultdict(_default_reason) + self.objs_to_update = DefaultComponentMap(_default_reason) + self.observers = observers + + def run(self): + # split up new, removed, and modified variables + new_vars = ComponentMap( + (k, v) for k, v in self.vars_to_update.items() if v & Reason.added + ) + other_vars = ComponentMap( + (k, v) for k, v in self.vars_to_update.items() if not (v & Reason.added) + ) + + new_params = ComponentMap( + (k, v) for k, v in self.params_to_update.items() if v & Reason.added + ) + other_params = ComponentMap( + (k, v) for k, v in self.params_to_update.items() if not (v & Reason.added) + ) + + for obs in self.observers: + if new_vars: + obs._update_variables(new_vars) + if new_params: + obs._update_parameters(new_params) + if self.cons_to_update: + obs._update_constraints(self.cons_to_update) + if self.sos_to_update: + obs._update_sos_constraints(self.sos_to_update) + if self.objs_to_update: + obs._update_objectives(self.objs_to_update) + if other_vars: + obs._update_variables(other_vars) + if other_params: + obs._update_parameters(other_params) + + self.clear() + + def clear(self): + self.vars_to_update.clear() + self.params_to_update.clear() + self.cons_to_update.clear() + self.sos_to_update.clear() + self.objs_to_update.clear() + + +""" +There are three stages: +- identification of differences between the model and the internal data structures of the Change Detector +- synchronization of the model with the internal data structures of the ChangeDetector +- notification of the observers + +The first two really happen at the same time + +Update order when notifying the observers: + - add new variables + - add new constraints + - add new objectives + - remove old constraints + - remove old objectives + - remove old variables + - update modified constraints + - update modified objectives + - update modified variables +""" + + +class ModelChangeDetector: + """ + This class "watches" a pyomo model and notifies the observers when any + changes to the model are made (but only when ModelChangeDetector.update + is called). An example use case is for the persistent solver interfaces. + + The ModelChangeDetector considers the model to be defined by its set of + active components and any components used by those active components. For + example, the observers will not be notified of the addition of a variable + if that variable is not used in any constraints. + + The Observer/ModelChangeDetector are most useful when a small number + of changes are being made relative to the size of the model. For example, + the persistent solver interfaces can be very efficient when repeatedly + solving the same model but with different values for mutable parameters. + + If you know that certain changes will not be made to the model, the + config can be modified to improve performance. For example, if you + know that no constraints will be added to or removed from the model, + then ``check_for_new_or_removed_constraints`` can be set to ``False``, + which will save some time when ``update`` is called. + + Here are some usage examples: + + >>> import pyomo.environ as pyo + >>> from typing import Mapping + >>> from pyomo.contrib.observer.model_observer import ( + ... AutoUpdateConfig, + ... Observer, + ... ModelChangeDetector, + ... Reason, + ... ) + >>> from pyomo.core.base import ( + ... VarData, + ... ParamData, + ... ConstraintData, + ... SOSConstraintData, + ... ObjectiveData, + ... ) + >>> class PrintObserver(Observer): + ... def _update_variables(self, vars: Mapping[VarData, Reason]): + ... for v, r in vars.items(): + ... print(f'{v}: {r.name}') + ... def _update_parameters(self, params: Mapping[ParamData, Reason]): + ... for p, r in params.items(): + ... print(f'{p}: {r.name}') + ... def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + ... for c, r in cons.items(): + ... print(f'{c}: {r.name}') + ... def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + ... for c, r in cons.items(): + ... print(f'{c}: {r.name}') + ... def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + ... for o, r in objs.items(): + ... print(f'{o}: {r.name}') + >>> m = pyo.ConcreteModel() + >>> obs = PrintObserver() + >>> detector = ModelChangeDetector(m, [obs]) + >>> m.x = pyo.Var() + >>> m.y = pyo.Var() + >>> detector.update() # no output because the variables are not used + >>> m.obj = pyo.Objective(expr=m.x**2 + m.y**2) + >>> detector.update() + x: added + y: added + obj: added + >>> del m.obj + >>> detector.update() + obj: removed + x: removed + y: removed + >>> m.px = pyo.Param(mutable=True, initialize=1) + >>> m.py = pyo.Param(mutable=True, initialize=1) + >>> m.obj = pyo.Objective(expr=m.px*m.x + m.py*m.y) + >>> detector.update() + x: added + y: added + px: added + py: added + obj: added + >>> m.px.value = 2 + >>> detector.update() + px: value + >>> detector.config.check_for_new_or_removed_constraints = False + >>> detector.config.check_for_new_or_removed_objectives = False + >>> detector.config.update_constraints = False + >>> detector.config.update_vars = False + >>> detector.config.update_parameters = True + >>> detector.config.update_named_expressions = False + >>> detector.config.update_objectives = False + >>> for i in range(10): + ... m.py.value = i + ... detector.update() # this will be faster because it is only checking for changes to parameters + py: value + py: value + py: value + py: value + py: value + py: value + py: value + py: value + py: value + py: value + >>> m.c = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) + >>> detector.update() # no output because we did not check for new constraints + >>> detector.config.check_for_new_or_removed_constraints = True + >>> detector.update() + c: added + + """ + + def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): + """ + Parameters + ---------- + model: BlockData + The model for which changes should be detected + observers: Sequence[Observer] + The objects to notify when changes are made to the model + """ + self._known_active_ctypes = {Constraint, SOSConstraint, Objective, Block} + self._observers: List[Observer] = list(observers) + + self._active_constraints: MutableMapping[ + ConstraintData, Union[RelationalExpression, None] + ] = {} + + self._active_sos = {} + + # maps var to (lb, ub, fixed, domain, value) + self._vars: MutableMapping[VarData, Tuple] = ComponentMap() + + # maps param to value + self._params: MutableMapping[ParamData, float] = ComponentMap() + + self._objectives: MutableMapping[ + ObjectiveData, Tuple[Union[NumericValue, float, int, None], ObjectiveSense] + ] = ComponentMap() # maps objective to (expression, sense) + + # maps constraints/objectives to list of tuples (named_expr, named_expr.expr) + self._named_expressions: MutableMapping[ + ConstraintData, + List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]], + ] = {} + self._obj_named_expressions: MutableMapping[ + ObjectiveData, + List[Tuple[ExpressionData, Union[NumericValue, float, int, None]]], + ] = ComponentMap() + + self._external_functions = ComponentMap() + + self._referenced_variables: MutableMapping[ + VarData, + Tuple[ + MutableSet[ConstraintData], + MutableSet[SOSConstraintData], + MutableSet[ObjectiveData], + ], + ] = ComponentMap() + + self._referenced_params: MutableMapping[ + ParamData, + Tuple[ + MutableSet[ConstraintData], + MutableSet[SOSConstraintData], + MutableSet[ObjectiveData], + MutableSet[VarData], + ], + ] = ComponentMap() + + self._vars_referenced_by_con: MutableMapping[ + Union[ConstraintData, SOSConstraintData], MutableSet[VarData] + ] = {} + self._vars_referenced_by_obj: MutableMapping[ + ObjectiveData, MutableSet[VarData] + ] = ComponentMap() + self._params_referenced_by_con: MutableMapping[ + Union[ConstraintData, SOSConstraintData], MutableSet[ParamData] + ] = {} + # for when parameters show up in variable bounds + self._params_referenced_by_var: MutableMapping[ + VarData, MutableSet[ParamData] + ] = ComponentMap() + self._params_referenced_by_obj: MutableMapping[ + ObjectiveData, MutableSet[ParamData] + ] = ComponentMap() + + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) + + self._updates = _Updates(self._observers) + + self._model: BlockData = model + self._set_instance() + + def _add_variables(self, variables: Collection[VarData]): + for v in variables: + if v in self._referenced_variables: + raise ValueError(f'Variable {v.name} has already been added') + self._updates.vars_to_update[v] |= Reason.added + self._referenced_variables[v] = (OrderedSet(), OrderedSet(), ComponentSet()) + self._vars[v] = (v._lb, v._ub, v.fixed, v.domain.get_interval(), v.value) + ref_params = ComponentSet() + for bnd in (v._lb, v._ub): + if bnd is None or type(bnd) in native_numeric_types: + continue + (named_exprs, _vars, parameters, external_functions) = ( + collect_components_from_expr(bnd) + ) + if _vars: + raise NotImplementedError( + 'ModelChangeDetector does not support variables in the bounds of other variables' + ) + if named_exprs: + raise NotImplementedError( + 'ModelChangeDetector does not support Expressions in the bounds of other variables' + ) + if external_functions: + raise NotImplementedError( + 'ModelChangeDetector does not support external functions in the bounds of other variables' + ) + ref_params.update(parameters) + self._params_referenced_by_var[v] = ref_params + if ref_params: + self._check_for_new_params(ref_params) + for p in ref_params: + self._referenced_params[p][3].add(v) + + def _add_parameters(self, params: Collection[ParamData]): + for p in params: + if p in self._referenced_params: + raise ValueError(f'Parameter {p.name} has already been added') + self._updates.params_to_update[p] |= Reason.added + self._referenced_params[p] = ( + OrderedSet(), + OrderedSet(), + ComponentSet(), + ComponentSet(), + ) + self._params[p] = p.value + + def _check_for_new_vars(self, variables: Collection[VarData]): + new_vars = ComponentSet( + v for v in variables if v not in self._referenced_variables + ) + self._add_variables(new_vars) + + def _check_to_remove_vars(self, variables: Collection[VarData]): + vars_to_remove = ComponentSet() + for v in variables: + if not any(self._referenced_variables[v]): + vars_to_remove.add(v) + self._remove_variables(vars_to_remove) + + def _check_for_new_params(self, params: Collection[ParamData]): + new_params = ComponentSet(p for p in params if p not in self._referenced_params) + self._add_parameters(new_params) + + def _check_to_remove_params(self, params: Collection[ParamData]): + params_to_remove = ComponentSet() + for p in params: + if not any(self._referenced_params[p]): + params_to_remove.add(p) + self._remove_parameters(params_to_remove) + + def _add_constraints(self, cons: Collection[ConstraintData]): + for con in cons: + if con in self._active_constraints: + raise ValueError(f'Constraint {con.name} has already been added') + self._updates.cons_to_update[con] |= Reason.added + self._active_constraints[con] = con.expr + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(con.expr) + ) + self._check_for_new_vars(variables) + self._check_for_new_params(parameters) + if named_exprs: + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + if external_functions: + self._external_functions[con] = external_functions + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + for v in variables: + self._referenced_variables[v][0].add(con) + for p in parameters: + self._referenced_params[p][0].add(con) + + def add_constraints(self, cons: Collection[ConstraintData]): + self._add_constraints(cons) + self._updates.run() + + def _add_sos_constraints(self, cons: Collection[SOSConstraintData]): + for con in cons: + if con in self._active_sos: + raise ValueError(f'Constraint {con.name} has already been added') + self._updates.sos_to_update[con] |= Reason.added + sos_items = list(con.get_items()) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) + variables = ComponentSet() + params = ComponentSet() + for v, p in sos_items: + variables.add(v) + if type(p) in native_numeric_types: + continue + if p.is_parameter_type(): + params.add(p) + self._check_for_new_vars(variables) + self._check_for_new_params(params) + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = params + for v in variables: + self._referenced_variables[v][1].add(con) + for p in params: + self._referenced_params[p][1].add(con) + + def add_sos_constraints(self, cons: Collection[SOSConstraintData]): + self._add_sos_constraints(cons) + self._updates.run() + + def _add_objectives(self, objs: Collection[ObjectiveData]): + for obj in objs: + self._updates.objs_to_update[obj] |= Reason.added + self._objectives[obj] = (obj.expr, obj.sense) + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(obj.expr) + ) + self._check_for_new_vars(variables) + self._check_for_new_params(parameters) + if named_exprs: + self._obj_named_expressions[obj] = [(e, e.expr) for e in named_exprs] + if external_functions: + self._external_functions[obj] = external_functions + self._vars_referenced_by_obj[obj] = variables + self._params_referenced_by_obj[obj] = parameters + for v in variables: + self._referenced_variables[v][2].add(obj) + for p in parameters: + self._referenced_params[p][2].add(obj) + + def add_objectives(self, objs: Collection[ObjectiveData]): + self._add_objectives(objs) + self._updates.run() + + def _remove_objectives(self, objs: Collection[ObjectiveData]): + for obj in objs: + if obj not in self._objectives: + raise ValueError( + f'cannot remove objective {obj.name} - it was not added' + ) + self._updates.objs_to_update[obj] |= Reason.removed + for v in self._vars_referenced_by_obj[obj]: + self._referenced_variables[v][2].remove(obj) + for p in self._params_referenced_by_obj[obj]: + self._referenced_params[p][2].remove(obj) + self._check_to_remove_vars(self._vars_referenced_by_obj[obj]) + self._check_to_remove_params(self._params_referenced_by_obj[obj]) + del self._objectives[obj] + self._obj_named_expressions.pop(obj, None) + self._external_functions.pop(obj, None) + self._vars_referenced_by_obj.pop(obj) + self._params_referenced_by_obj.pop(obj) + + def remove_objectives(self, objs: Collection[ObjectiveData]): + self._remove_objectives(objs) + self._updates.run() + + def _check_for_unknown_active_components(self): + for ctype in self._model.collect_ctypes(active=True, descend_into=True): + if not issubclass(ctype, ActiveComponent): + continue + if ctype in self._known_active_ctypes: + continue + if ctype is Suffix: + warnings.warn('ModelChangeDetector does not detect changes to suffixes') + continue + raise NotImplementedError( + f'ModelChangeDetector does not know how to ' + f'handle components with ctype {ctype}' + ) + + def _set_instance(self): + + with PauseGC() as pgc: + self._check_for_unknown_active_components() + + self.add_constraints( + list( + self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + ) + ) + self.add_sos_constraints( + list( + self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) + ) + self.add_objectives( + list( + self._model.component_data_objects( + Objective, descend_into=True, active=True + ) + ) + ) + + def _remove_constraints(self, cons: Collection[ConstraintData]): + for con in cons: + if con not in self._active_constraints: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + self._updates.cons_to_update[con] |= Reason.removed + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[v][0].remove(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[p][0].remove(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + self._check_to_remove_params(self._params_referenced_by_con[con]) + self._active_constraints.pop(con) + self._named_expressions.pop(con, None) + self._external_functions.pop(con, None) + self._vars_referenced_by_con.pop(con) + self._params_referenced_by_con.pop(con) + + def remove_constraints(self, cons: Collection[ConstraintData]): + self._remove_constraints(cons) + self._updates.run() + + def _remove_sos_constraints(self, cons: Collection[SOSConstraintData]): + for con in cons: + if con not in self._active_sos: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + self._updates.sos_to_update[con] |= Reason.removed + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[v][1].remove(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[p][1].remove(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + self._check_to_remove_params(self._params_referenced_by_con[con]) + self._active_sos.pop(con) + self._vars_referenced_by_con.pop(con) + self._params_referenced_by_con.pop(con) + + def remove_sos_constraints(self, cons: Collection[SOSConstraintData]): + self._remove_sos_constraints(cons) + self._updates.run() + + def _remove_variables(self, variables: Collection[VarData]): + for v in variables: + if v not in self._referenced_variables: + raise ValueError( + f'Cannot remove variable {v.name} - it has not been added' + ) + self._updates.vars_to_update[v] |= Reason.removed + for p in self._params_referenced_by_var[v]: + self._referenced_params[p][3].remove(v) + self._check_to_remove_params(self._params_referenced_by_var[v]) + self._params_referenced_by_var.pop(v) + if any(self._referenced_variables[v]): + raise ValueError( + f'Cannot remove variable {v.name} - it is still being used by constraints/objectives' + ) + self._referenced_variables.pop(v) + self._vars.pop(v) + + def _remove_parameters(self, params: Collection[ParamData]): + for p in params: + if p not in self._referenced_params: + raise ValueError( + f'Cannot remove parameter {p.name} - it has not been added' + ) + self._updates.params_to_update[p] |= Reason.removed + if any(self._referenced_params[p]): + raise ValueError( + f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' + ) + self._referenced_params.pop(p) + self._params.pop(p) + + def _update_var_bounds(self, v: VarData): + ref_params = ComponentSet() + for bnd in (v._lb, v._ub): + if bnd is None or type(bnd) in native_numeric_types: + continue + (named_exprs, _vars, parameters, external_functions) = ( + collect_components_from_expr(bnd) + ) + if _vars: + raise NotImplementedError( + 'ModelChangeDetector does not support variables in the bounds of other variables' + ) + if named_exprs: + raise NotImplementedError( + 'ModelChangeDetector does not support Expressions in the bounds of other variables' + ) + if external_functions: + raise NotImplementedError( + 'ModelChangeDetector does not support external functions in the bounds of other variables' + ) + ref_params.update(parameters) + + _ref_params = self._params_referenced_by_var[v] + new_params = ref_params - _ref_params + old_params = _ref_params - ref_params + + self._params_referenced_by_var[v] = ref_params + + if new_params: + self._check_for_new_params(new_params) + + for p in new_params: + self._referenced_params[p][3].add(v) + for p in old_params: + self._referenced_params[p][3].remove(v) + + if old_params: + self._check_to_remove_params(old_params) + + def _update_variables(self, variables: Optional[Collection[VarData]] = None): + if variables is None: + variables = self._vars + for v in variables: + _lb, _ub, _fixed, _domain_interval, _value = self._vars[v] + lb, ub, fixed, domain_interval, value = ( + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) + reason = Reason.no_change + if _fixed != fixed: + reason |= Reason.fixed + elif (_fixed or fixed) and (value != _value): + reason |= Reason.value + if lb is not _lb or ub is not _ub: + reason |= Reason.bounds + if _domain_interval != domain_interval: + reason |= Reason.domain + if reason: + self._updates.vars_to_update[v] |= reason + self._vars[v] = (lb, ub, fixed, domain_interval, value) + if reason & Reason.bounds: + self._update_var_bounds(v) + + def update_variables(self, variables: Optional[Collection[VarData]] = None): + self._update_variables(variables) + self._updates.run() + + def _update_parameters(self, params: Optional[Collection[ParamData]] = None): + if params is None: + params = self._params + for p in params: + _val = self._params[p] + val = p.value + reason = Reason.no_change + if _val != val: + reason |= Reason.value + if reason: + self._updates.params_to_update[p] |= reason + self._params[p] = val + + def update_parameters(self, params: Optional[Collection[ParamData]]): + self._update_parameters(params) + self._updates.run() + + def _update_con(self, con: ConstraintData): + self._active_constraints[con] = con.expr + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(con.expr) + ) + if named_exprs: + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + else: + self._named_expressions.pop(con, None) + if external_functions: + self._external_functions[con] = external_functions + else: + self._external_functions.pop(con, None) + + _variables = self._vars_referenced_by_con[con] + _parameters = self._params_referenced_by_con[con] + new_vars = variables - _variables + old_vars = _variables - variables + new_params = parameters - _parameters + old_params = _parameters - parameters + + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + + if new_vars: + self._check_for_new_vars(new_vars) + if new_params: + self._check_for_new_params(new_params) + + for v in new_vars: + self._referenced_variables[v][0].add(con) + for v in old_vars: + self._referenced_variables[v][0].remove(con) + for p in new_params: + self._referenced_params[p][0].add(con) + for p in old_params: + self._referenced_params[p][0].remove(con) + + if old_vars: + self._check_to_remove_vars(old_vars) + if old_params: + self._check_to_remove_params(old_params) + + def _update_constraints(self, cons: Optional[Collection[ConstraintData]] = None): + if cons is None: + cons = self._active_constraints + for c in cons: + reason = Reason.no_change + if c.expr is not self._active_constraints[c]: + reason |= Reason.expr + if reason: + self._updates.cons_to_update[c] |= reason + self._update_con(c) + + def update_constraints(self, cons: Optional[Collection[ConstraintData]] = None): + self._update_constraints(cons) + self._updates.run() + + def _update_sos_con(self, con: SOSConstraintData): + sos_items = list(con.get_items()) + self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + variables = ComponentSet() + parameters = ComponentSet() + for v, p in sos_items: + variables.add(v) + if type(p) in native_numeric_types: + continue + if p.is_parameter_type(): + parameters.add(p) + + _variables = self._vars_referenced_by_con[con] + _parameters = self._params_referenced_by_con[con] + new_vars = variables - _variables + old_vars = _variables - variables + new_params = parameters - _parameters + old_params = _parameters - parameters + + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + + if new_vars: + self._check_for_new_vars(new_vars) + if new_params: + self._check_for_new_params(new_params) + + for v in new_vars: + self._referenced_variables[v][1].add(con) + for v in old_vars: + self._referenced_variables[v][1].remove(con) + for p in new_params: + self._referenced_params[p][1].add(con) + for p in old_params: + self._referenced_params[p][1].remove(con) + + if old_vars: + self._check_to_remove_vars(old_vars) + if old_params: + self._check_to_remove_params(old_params) + + def _update_sos_constraints( + self, cons: Optional[Collection[SOSConstraintData]] = None + ): + if cons is None: + cons = self._active_sos + for c in cons: + reason = Reason.no_change + _vlist, _plist = self._active_sos[c] + sos_items = list(c.get_items()) + vlist = [i[0] for i in sos_items] + plist = [i[1] for i in sos_items] + needs_update = False + if len(_vlist) != len(vlist) or len(_plist) != len(plist): + needs_update = True + else: + for v1, v2 in zip(_vlist, vlist): + if v1 is not v2: + needs_update = True + break + for p1, p2 in zip(_plist, plist): + if p1 is not p2: + needs_update = True + break + if needs_update: + reason |= Reason.sos_items + self._updates.sos_to_update[c] |= reason + self._update_sos_con(c) + + def update_sos_constraints( + self, cons: Optional[Collection[SOSConstraintData]] = None + ): + self._update_sos_constraints(cons) + self._updates.run() + + def _update_obj_expr(self, obj: ObjectiveData): + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(obj.expr) + ) + if named_exprs: + self._obj_named_expressions[obj] = [(e, e.expr) for e in named_exprs] + else: + self._obj_named_expressions.pop(obj, None) + if external_functions: + self._external_functions[obj] = external_functions + else: + self._external_functions.pop(obj, None) + + _variables = self._vars_referenced_by_obj[obj] + _parameters = self._params_referenced_by_obj[obj] + new_vars = variables - _variables + old_vars = _variables - variables + new_params = parameters - _parameters + old_params = _parameters - parameters + + self._vars_referenced_by_obj[obj] = variables + self._params_referenced_by_obj[obj] = parameters + + if new_vars: + self._check_for_new_vars(new_vars) + if new_params: + self._check_for_new_params(new_params) + + for v in new_vars: + self._referenced_variables[v][2].add(obj) + for v in old_vars: + self._referenced_variables[v][2].remove(obj) + for p in new_params: + self._referenced_params[p][2].add(obj) + for p in old_params: + self._referenced_params[p][2].remove(obj) + + if old_vars: + self._check_to_remove_vars(old_vars) + if old_params: + self._check_to_remove_params(old_params) + + def _update_objectives(self, objs: Optional[Collection[ObjectiveData]] = None): + if objs is None: + objs = self._objectives + for obj in objs: + reason = Reason.no_change + _expr, _sense = self._objectives[obj] + if _expr is not obj.expr: + reason |= Reason.expr + if _sense != obj.sense: + reason |= Reason.sense + if reason: + self._updates.objs_to_update[obj] |= reason + self._objectives[obj] = (obj.expr, obj.sense) + if reason & Reason.expr: + self._update_obj_expr(obj) + + def update_objectives(self, objs: Optional[Collection[ObjectiveData]] = None): + self._update_objectives(objs) + self._updates.run() + + def _check_for_new_or_removed_sos(self): + new_sos = [] + old_sos = [] + current_sos_set = OrderedSet( + self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) + for c in current_sos_set: + if c not in self._active_sos: + new_sos.append(c) + for c in self._active_sos: + if c not in current_sos_set: + old_sos.append(c) + return new_sos, old_sos + + def _check_for_new_or_removed_constraints(self): + new_cons = [] + old_cons = [] + current_cons_set = OrderedSet( + self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + ) + for c in current_cons_set: + if c not in self._active_constraints: + new_cons.append(c) + for c in self._active_constraints: + if c not in current_cons_set: + old_cons.append(c) + return new_cons, old_cons + + def _check_for_named_expression_changes(self): + for con, ne_list in self._named_expressions.items(): + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + self._updates.cons_to_update[con] |= Reason.expr + self._update_con(con) + break + for obj, ne_list in self._obj_named_expressions.items(): + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + self._updates.objs_to_update[obj] |= Reason.expr + self._update_obj_expr(obj) + break + + def _check_for_new_or_removed_objectives(self): + new_objs = [] + old_objs = [] + current_objs_set = ComponentSet( + self._model.component_data_objects( + Objective, descend_into=True, active=True + ) + ) + for obj in current_objs_set: + if obj not in self._objectives: + new_objs.append(obj) + for obj in self._objectives.keys(): + if obj not in current_objs_set: + old_objs.append(obj) + return new_objs, old_objs + + def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): + """ + Check for changes to the model and notify the observers. + + Parameters + ---------- + timer: Optional[HierarchicalTimer] + The timer to use for tracking how much time is spent detecting + different kinds of changes + """ + + """ + When possible, it is better to add new constraints before removing old + constraints. This prevents unnecessarily removing and adding variables. + If a constraint is removed, any variables that are used only by that + constraint will be removed. If there is a new constraint that uses + the same variable, then we don't actually need to remove the variable. + This is hard to avoid when we are modifying a constraint or changing + the objective. When the objective changes, we remove the old one + first just because most things don't handle multiple objectives. + + We check for changes to constraints/objectives before variables/parameters + so that we don't waste time updating a variable/parameter that is going to + get removed. + """ + if timer is None: + timer = HierarchicalTimer() + config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) + + with PauseGC() as pgc: + self._check_for_unknown_active_components() + + if config.check_for_new_or_removed_constraints: + new_cons, old_cons = self._check_for_new_or_removed_constraints() + new_sos, old_sos = self._check_for_new_or_removed_sos() + else: + new_cons = [] + old_cons = [] + new_sos = [] + old_sos = [] + + if config.check_for_new_or_removed_objectives: + new_objs, old_objs = self._check_for_new_or_removed_objectives() + else: + new_objs = [] + old_objs = [] + + if new_cons: + self._add_constraints(new_cons) + if new_sos: + self._add_sos_constraints(new_sos) + if new_objs: + self._add_objectives(new_objs) + + if old_cons: + self._remove_constraints(old_cons) + if old_sos: + self._remove_sos_constraints(old_sos) + if old_objs: + self._remove_objectives(old_objs) + + if config.update_constraints: + self._update_constraints() + self._update_sos_constraints() + if config.update_objectives: + self._update_objectives() + + if config.update_named_expressions: + self._check_for_named_expression_changes() + + if config.update_vars: + self._update_variables() + + if config.update_parameters: + self._update_parameters() + + self._updates.run() + + def get_variables_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[p][3]) + + def get_constraints_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[p][0]) + + def get_constraints_impacted_by_var(self, v: VarData): + return list(self._referenced_variables[v][0]) + + def get_objectives_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[p][2]) + + def get_objectives_impacted_by_var(self, v: VarData): + return list(self._referenced_variables[v][2]) diff --git a/pyomo/contrib/observer/tests/__init__.py b/pyomo/contrib/observer/tests/__init__.py new file mode 100644 index 00000000000..6eb9ea8b81d --- /dev/null +++ b/pyomo/contrib/observer/tests/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py new file mode 100644 index 00000000000..b342bed9e4f --- /dev/null +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -0,0 +1,545 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import logging +from typing import List, Mapping + +import pyomo.environ as pyo +from pyomo.core.base.constraint import ConstraintData +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.base.param import ParamData +from pyomo.core.base.sos import SOSConstraintData +from pyomo.core.base.var import VarData +from pyomo.common import unittest +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, + Reason, +) +from pyomo.common.collections import DefaultComponentMap, ComponentMap +from pyomo.common.errors import PyomoException + + +logger = logging.getLogger(__name__) + + +def make_count_dict(): + d = {i: 0 for i in Reason} + return d + + +class ObserverChecker(Observer): + def __init__(self): + super().__init__() + self.counts = DefaultComponentMap(make_count_dict) + """ + counts is a mapping from component (e.g., variable) to another + mapping from Reason to an int that indicates the number of times + the corresponding method has been called + """ + + def check(self, expected): + unittest.assertStructuredAlmostEqual( + first=expected, second=self.counts, places=7 + ) + + def pprint(self): + for k, d in self.counts.items(): + print(f'{k}:') + for a, v in d.items(): + print(f' {a}: {v}') + + def _update_variables(self, variables: Mapping[VarData, Reason]): + for v, reason in variables.items(): + self.counts[v][reason] += 1 + + def _update_parameters(self, params: Mapping[ParamData, Reason]): + for p, reason in params.items(): + self.counts[p][reason] += 1 + + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + for c, reason in cons.items(): + self.counts[c][reason] += 1 + + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + for c, reason in cons.items(): + self.counts[c][reason] += 1 + + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + for obj, reason in objs.items(): + self.counts[obj][reason] += 1 + + +class TestChangeDetector(unittest.TestCase): + def test_objective(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) + detector.update() + expected[m.obj][Reason.added] += 1 + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + obs.check(expected) + + m.y.setlb(0) + detector.update() + expected[m.y][Reason.bounds] += 1 + obs.check(expected) + + m.x.fix(2) + detector.update() + expected[m.x][Reason.fixed] += 1 + obs.check(expected) + + m.x.unfix() + detector.update() + expected[m.x][Reason.fixed] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update() + expected[m.p][Reason.value] += 1 + obs.check(expected) + + m.obj.expr = m.x**2 + m.y**2 + detector.update() + expected[m.obj][Reason.expr] += 1 + expected[m.p][Reason.removed] += 1 + obs.check(expected) + + expected[m.obj][Reason.removed] += 1 + del m.obj + m.obj2 = pyo.Objective(expr=m.p * m.x) + detector.update() + # remember, m.obj is a different object now + expected[m.obj2][Reason.added] += 1 + expected[m.y][Reason.removed] += 1 + expected[m.p][Reason.added] += 1 + obs.check(expected) + + def test_constraints(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= (m.x - m.p) ** 2) + detector.update() + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 + obs.check(expected) + + m.x.fix(1) + detector.update() + expected[m.x][Reason.fixed] += 1 + obs.check(expected) + + m.z = pyo.Var() + m.c1.set_value(m.y == 2 * m.z) + detector.update() + expected[m.z][Reason.added] += 1 + expected[m.c1][Reason.expr] += 1 + expected[m.p][Reason.removed] += 1 + expected[m.x][Reason.removed] += 1 + obs.check(expected) + + expected[m.c1][Reason.removed] += 1 + del m.c1 + detector.update() + expected[m.z][Reason.removed] += 1 + obs.check(expected) + + def test_sos(self): + m = pyo.ConcreteModel() + m.a = pyo.Set(initialize=[1, 2, 3], ordered=True) + m.x = pyo.Var(m.a, within=pyo.Binary) + m.y = pyo.Var(within=pyo.Binary) + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.SOSConstraint(var=m.x, sos=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + expected[m.obj][Reason.added] += 1 + for i in m.a: + expected[m.x[i]][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + obs.check(expected) + + detector.update() + obs.check(expected) + + m.c1.set_items([m.x[2], m.x[1], m.x[3]], [1, 2, 3]) + detector.update() + expected[m.c1][Reason.sos_items] += 1 + obs.check(expected) + + m.c1.set_items([m.x[2], m.x[1]], [1, 2]) + detector.update() + expected[m.c1][Reason.sos_items] += 1 + expected[m.x[3]][Reason.removed] += 1 + obs.check(expected) + + m.c1.set_items([m.x[2], m.x[1], m.x[3]], [1, 2, 3]) + detector.update() + expected[m.c1][Reason.sos_items] += 1 + expected[m.x[3]][Reason.added] += 1 + obs.check(expected) + + for i in m.a: + expected[m.x[i]][Reason.removed] += 1 + expected[m.c1][Reason.removed] += 1 + del m.c1 + detector.update() + obs.check(expected) + + def test_vars_and_params_elsewhere(self): + m1 = pyo.ConcreteModel() + m1.x = pyo.Var() + m1.y = pyo.Var() + m1.p = pyo.Param(mutable=True, initialize=1) + + m2 = pyo.ConcreteModel() + + obs = ObserverChecker() + detector = ModelChangeDetector(m2, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m2.obj = pyo.Objective(expr=m1.y) + m2.c1 = pyo.Constraint(expr=m1.y >= (m1.x - m1.p) ** 2) + detector.update() + expected[m1.x][Reason.added] += 1 + expected[m1.y][Reason.added] += 1 + expected[m1.p][Reason.added] += 1 + expected[m2.c1][Reason.added] += 1 + expected[m2.obj][Reason.added] += 1 + obs.check(expected) + + m1.x.fix(1) + detector.update() + expected[m1.x][Reason.fixed] += 1 + obs.check(expected) + + def test_named_expression(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.e = pyo.Expression(expr=m.x - m.p) + m.c1 = pyo.Constraint(expr=m.y >= m.e) + detector.update() + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 + obs.check(expected) + + # now modify the named expression and make sure the + # constraint gets removed and added + m.e.expr = (m.x - m.p) ** 2 + detector.update() + expected[m.c1][Reason.expr] += 1 + obs.check(expected) + + def test_update_config(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(initialize=1, mutable=True) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = False + detector.config.check_for_new_or_removed_objectives = False + detector.config.update_constraints = False + detector.config.update_objectives = False + detector.config.update_vars = False + detector.config.update_parameters = False + detector.config.update_named_expressions = False + + m.e = pyo.Expression(expr=pyo.exp(m.x)) + m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) + m.c1 = pyo.Constraint(expr=m.y >= m.e + m.p) + + detector.update() + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = True + detector.update() + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + obs.check(expected) + + detector.config.check_for_new_or_removed_objectives = True + detector.update() + expected[m.obj][Reason.added] += 1 + obs.check(expected) + + m.x.setlb(0) + detector.update() + obs.check(expected) + + detector.config.update_vars = True + detector.update() + expected[m.x][Reason.bounds] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update() + obs.check(expected) + + detector.config.update_parameters = True + detector.update() + expected[m.p][Reason.value] += 1 + obs.check(expected) + + m.e.expr += 1 + detector.update() + obs.check(expected) + + detector.config.update_named_expressions = True + detector.update() + expected[m.c1][Reason.expr] += 1 + obs.check(expected) + + m.obj.expr += 1 + detector.update() + obs.check(expected) + + detector.config.update_objectives = True + detector.update() + expected[m.obj][Reason.expr] += 1 + obs.check(expected) + + m.c1 = m.y >= m.e + detector.update() + obs.check(expected) + + detector.config.update_constraints = True + detector.update() + expected[m.c1][Reason.expr] += 1 + obs.check(expected) + + def test_param_in_bounds(self): + m = pyo.ConcreteModel() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.y.setlb(m.p - 1) + detector.update() + expected[m.y][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update() + expected[m.p][Reason.value] += 1 + obs.check(expected) + + m.p2 = pyo.Param(mutable=True, initialize=1) + m.y.setub(m.p2 + 1) + detector.update() + expected[m.p2][Reason.added] += 1 + expected[m.y][Reason.bounds] += 1 + obs.check(expected) + + def test_incidence(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.p1 = pyo.Param(mutable=True, initialize=1) + m.p2 = pyo.Param(mutable=True, initialize=1) + m.x.setlb(m.p1) + + m.e1 = pyo.Expression(expr=m.x + m.p1) + m.e2 = pyo.Expression(expr=(m.e1**2)) + m.obj = pyo.Objective(expr=m.e2 + m.y**2) + m.c1 = pyo.Constraint(expr=m.z + m.p2 == 0) + m.c2 = pyo.Constraint(expr=m.x + m.p2 == 0) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + expected[m.x][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + expected[m.z][Reason.added] += 1 + expected[m.p1][Reason.added] += 1 + expected[m.p2][Reason.added] += 1 + expected[m.obj][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + expected[m.c2][Reason.added] += 1 + obs.check(expected) + + self.assertEqual(detector.get_variables_impacted_by_param(m.p1), [m.x]) + self.assertEqual(detector.get_variables_impacted_by_param(m.p2), []) + self.assertEqual(detector.get_constraints_impacted_by_param(m.p1), []) + self.assertEqual(detector.get_constraints_impacted_by_param(m.p2), [m.c1, m.c2]) + self.assertEqual(detector.get_constraints_impacted_by_var(m.x), [m.c2]) + self.assertEqual(detector.get_constraints_impacted_by_var(m.y), []) + self.assertEqual(detector.get_constraints_impacted_by_var(m.z), [m.c1]) + self.assertEqual(detector.get_objectives_impacted_by_param(m.p1), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_param(m.p2), []) + self.assertEqual(detector.get_objectives_impacted_by_var(m.x), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.y), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.z), []) + + m.e1.expr += m.z + detector.update() + expected[m.obj][Reason.expr] += 1 + obs.check(expected) + + self.assertEqual(detector.get_objectives_impacted_by_param(m.p1), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_param(m.p2), []) + self.assertEqual(detector.get_objectives_impacted_by_var(m.x), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.y), [m.obj]) + self.assertEqual(detector.get_objectives_impacted_by_var(m.z), [m.obj]) + + def test_manual_updates(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector(m, [obs]) + + expected = DefaultComponentMap(make_count_dict) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= (m.x - m.p) ** 2) + m.c2 = pyo.Constraint(expr=m.x + m.y == 0) + + detector.add_objectives([m.obj]) + expected[m.obj][Reason.added] += 1 + expected[m.y][Reason.added] += 1 + obs.check(expected) + + detector.add_constraints([m.c1]) + expected[m.x][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + expected[m.c1][Reason.added] += 1 + obs.check(expected) + + detector.add_constraints([m.c2]) + expected[m.c2][Reason.added] += 1 + obs.check(expected) + + detector.remove_constraints([m.c1]) + expected[m.c1][Reason.removed] += 1 + expected[m.p][Reason.removed] += 1 + obs.check(expected) + + detector.add_constraints([m.c1]) + expected[m.c1][Reason.added] += 1 + expected[m.p][Reason.added] += 1 + obs.check(expected) + + detector.remove_objectives([m.obj]) + expected[m.obj][Reason.removed] += 1 + obs.check(expected) + + detector.add_objectives([m.obj]) + expected[m.obj][Reason.added] += 1 + obs.check(expected) + + m.x.setlb(0) + detector.update_variables([m.x, m.y]) + expected[m.x][Reason.bounds] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update_parameters([m.p]) + expected[m.p][Reason.value] += 1 + obs.check(expected) + + m.c1.set_value(m.y >= m.x**2) + detector.update_constraints([m.c1, m.c2]) + expected[m.p][Reason.removed] += 1 + expected[m.c1][Reason.expr] += 1 + obs.check(expected) + + m.obj.expr += m.x + detector.update_objectives([m.obj]) + expected[m.obj][Reason.expr] += 1 + obs.check(expected) + + def test_mutable_parameters_in_sos(self): + """ + There is logic in the ModelChangeDetector to handle + mutable parameters in SOS constraints. However, we cannot + currently test it because of #3769. For now, we will + just make sure that an error is raised when attempting to + use a mutable parameter in an SOS constraint. If #3769 is + resolved, we will just need to update this test to make + sure the ModelChangeDetector does the right thing. + """ + m = pyo.ConcreteModel() + m.a = pyo.Set(initialize=[1, 2, 3]) + m.x = pyo.Var(m.a) + m.p = pyo.Param(m.a, mutable=True) + m.p[1].value = 1 + m.p[2].value = 2 + m.p[3].value = 3 + + with self.assertRaisesRegex( + PyomoException, 'Cannot convert non-constant Pyomo expression .* to bool.*' + ): + m.c = pyo.SOSConstraint(var=m.x, sos=1, weights=m.p) diff --git a/pyomo/contrib/observer/tests/test_component_collector.py b/pyomo/contrib/observer/tests/test_component_collector.py new file mode 100644 index 00000000000..70d01a08ccf --- /dev/null +++ b/pyomo/contrib/observer/tests/test_component_collector.py @@ -0,0 +1,31 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# __________________________________________________________________________ + +import pyomo.environ as pyo +from pyomo.common import unittest +from pyomo.contrib.observer.component_collector import collect_components_from_expr +from pyomo.common.collections import ComponentSet + + +class TestComponentCollector(unittest.TestCase): + def test_nested_named_expressions(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.e1 = pyo.Expression(expr=m.x + m.y) + m.e2 = pyo.Expression(expr=m.e1 + m.z) + e = m.e2 * pyo.exp(m.e2) + (named_exprs, vars, params, external_funcs) = collect_components_from_expr(e) + self.assertEqual(len(named_exprs), 2) + named_exprs = ComponentSet(named_exprs) + self.assertIn(m.e1, named_exprs) + self.assertIn(m.e2, named_exprs) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index 63c1e97ffd6..9f6171b7773 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -321,22 +321,6 @@ def set_objective(self, obj: ObjectiveData): f"Derived class {self.__class__.__name__} failed to implement required method 'set_objective'." ) - def add_variables(self, variables: List[VarData]): - """ - Add variables to the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'add_variables'." - ) - - def add_parameters(self, params: List[ParamData]): - """ - Add parameters to the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'add_parameters'." - ) - def add_constraints(self, cons: List[ConstraintData]): """ Add constraints to the model. @@ -353,22 +337,6 @@ def add_block(self, block: BlockData): f"Derived class {self.__class__.__name__} failed to implement required method 'add_block'." ) - def remove_variables(self, variables: List[VarData]): - """ - Remove variables from the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'remove_variables'." - ) - - def remove_parameters(self, params: List[ParamData]): - """ - Remove parameters from the model. - """ - raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'remove_parameters'." - ) - def remove_constraints(self, cons: List[ConstraintData]): """ Remove constraints from the model. diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index acd78284103..430207a736c 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -12,9 +12,9 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt, LegacyIpoptSolver -from .solvers.gurobi_persistent import GurobiPersistent -from .solvers.gurobi_direct import GurobiDirect -from .solvers.gurobi_direct_minlp import GurobiDirectMINLP +from .solvers.gurobi.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_persistent import GurobiPersistent +from .solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP from .solvers.highs import Highs from .solvers.knitro.direct import KnitroDirectSolver @@ -39,7 +39,7 @@ def load(): doc='Direct interface to Gurobi accommodating general MINLP', )(GurobiDirectMINLP) SolverFactory.register( - name="highs", legacy_name="highs", doc="Persistent interface to HiGHS" + name="highs", legacy_name="highs_v2", doc="Persistent interface to HiGHS" )(Highs) SolverFactory.register( name="knitro_direct", diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py new file mode 100644 index 00000000000..0809846ebc3 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/__init__.py @@ -0,0 +1,3 @@ +from .gurobi_direct import GurobiDirect +from .gurobi_persistent import GurobiPersistent +from .gurobi_direct_minlp import GurobiDirectMINLP diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py new file mode 100644 index 00000000000..f17497c4901 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -0,0 +1,134 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import operator + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler + +from pyomo.contrib.solver.common.util import ( + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from .gurobi_direct_base import ( + GurobiDirectBase, + gurobipy, + GurobiDirectSolutionLoaderBase, +) +import logging + + +logger = logging.getLogger(__name__) + + +class GurobiDirectSolutionLoader(GurobiDirectSolutionLoaderBase): + def __del__(self): + super().__del__() + if python_is_shutting_down(): + return + # Free the associated model + if self._solver_model is not None: + self._var_map = None + self._con_map = None + # explicitly release the model + self._solver_model.dispose() + self._solver_model = None + + +class GurobiDirect(GurobiDirectBase): + _minimum_version = (9, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._gurobi_vars = None + self._pyomo_vars = None + + def _pyomo_gurobi_var_iter(self): + return zip(self._pyomo_vars, self._gurobi_vars) + + def _create_solver_model(self, pyomo_model): + timer = self.config.timer + + timer.start('compile_model') + repn = LinearStandardFormCompiler().write( + pyomo_model, mixed_form=True, set_sense=None + ) + timer.stop('compile_model') + + if len(repn.objectives) > 1: + raise IncompatibleModelError( + f"The {self.__class__.__name__} solver only supports models " + f"with zero or one objectives (received {len(repn.objectives)})." + ) + + timer.start('prepare_matrices') + inf = float('inf') + ninf = -inf + bounds = list(map(operator.attrgetter('bounds'), repn.columns)) + lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] + ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] + CON = gurobipy.GRB.CONTINUOUS + BIN = gurobipy.GRB.BINARY + INT = gurobipy.GRB.INTEGER + vtype = [ + ( + CON + if v.is_continuous() + else BIN if v.is_binary() else INT if v.is_integer() else '?' + ) + for v in repn.columns + ] + sense_type = list('=<>') # Note: ordering matches 0, 1, -1 + sense = [sense_type[r[1]] for r in repn.rows] + timer.stop('prepare_matrices') + + gurobi_model = gurobipy.Model(env=self.env()) + + timer.start('transfer_model') + x = gurobi_model.addMVar( + len(repn.columns), + lb=lb, + ub=ub, + obj=repn.c.todense()[0] if repn.c.shape[0] else 0, + vtype=vtype, + ) + A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) + if repn.c.shape[0]: + gurobi_model.setAttr('ObjCon', repn.c_offset[0]) + gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) + # Note: calling gurobi_model.update() here is not + # necessary (it will happen as part of optimize()): + # gurobi_model.update() + timer.stop('transfer_model') + + self._pyomo_vars = repn.columns + self._gurobi_vars = x.tolist() + + var_map = ComponentMap(zip(repn.columns, self._gurobi_vars)) + con_map = {} + for row, gc in zip(repn.rows, A.tolist()): + pc = row.constraint + if pc in con_map: + # range constraint + con_map[pc] = (con_map[pc], gc) + else: + con_map[pc] = gc + solution_loader = GurobiDirectSolutionLoader( + solver_model=gurobi_model, var_map=var_map, con_map=con_map + ) + has_obj = len(repn.objectives) > 0 + + return gurobi_model, solution_loader, has_obj diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py new file mode 100644 index 00000000000..da0a109a6d4 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -0,0 +1,529 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import os +import logging +from typing import Mapping, Optional, Sequence, Dict + +from pyomo.common.collections import ComponentMap +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager +from pyomo.core.base import VarData, ConstraintData + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase + + +logger = logging.getLogger(__name__) + +gurobipy, gurobipy_available = attempt_import('gurobipy') + + +class GurobiConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Gurobi.", + ), + ) + + +def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): + """ + solver_model: gurobipy.Model + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if ( + solver_model.getAttr('NumIntVars') == 0 + and solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') + original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] + solver_model.setParam('SolutionNumber', solution_number) + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] + vals = solver_model.getAttr("Xn", gurobi_vars_to_load) + res = ComponentMap(zip(vars_to_load, vals)) + solver_model.setParam('SolutionNumber', original_solution_number) + return res + + +def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + for v, val in _get_primals( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ).items(): + v.set_value(val, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + +def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Mapping[Vardata, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if solver_model.SolCount == 0: + raise NoSolutionError() + + if solution_number != 0: + return _load_suboptimal_mip_solution( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ) + + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] + vals = solver_model.getAttr("X", gurobi_vars_to_load) + + res = ComponentMap(zip(vars_to_load, vals)) + return res + + +def _get_reduced_costs(solver_model, var_map, vars_to_load): + """ + solver_model: gurobipy.Model + var_map: Mapping[VarData, gurobipy.Var] + Maps the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + if solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() + + gurobi_vars_to_load = [var_map[v] for v in vars_to_load] + vals = solver_model.getAttr("Rc", gurobi_vars_to_load) + + res = ComponentMap(zip(vars_to_load, vals)) + return res + + +def _get_duals(solver_model, con_map, cons_to_load): + """ + solver_model: gurobipy.Model + con_map: Dict[ConstraintData, gurobipy.Constr] + Maps the pyomo constraint to the gurobipy constraint + cons_to_load: List[ConstraintData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + if solver_model.IsMIP: + # this will also return True for continuous, nonconvex models + raise NoDualsError() + + qcons = set(solver_model.getQConstrs()) + + duals = {} + for c in cons_to_load: + gurobi_con = con_map[c] + if type(gurobi_con) is tuple: + # only linear range constraints are supported + gc1, gc2 = gurobi_con + d1 = gc1.Pi + d2 = gc2.Pi + if abs(d1) > abs(d2): + duals[c] = d1 + else: + duals[c] = d2 + else: + if gurobi_con in qcons: + duals[c] = gurobi_con.QCPi + else: + duals[c] = gurobi_con.Pi + + return duals + + +class GurobiDirectSolutionLoaderBase(SolutionLoaderBase): + def __init__(self, solver_model, var_map, con_map) -> None: + super().__init__() + self._solver_model = solver_model + self._var_map = var_map + self._con_map = con_map + GurobiDirectBase._register_env_client() + + def __del__(self): + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() + + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + ) -> None: + if vars_to_load is None: + vars_to_load = self._var_map + _load_vars( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_primals( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = self._var_map + return _get_primals( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_reduced_costs( + self, vars_to_load: Optional[Sequence[VarData]] = None + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = self._var_map + return _get_reduced_costs( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + ) + + def get_duals( + self, cons_to_load: Optional[Sequence[ConstraintData]] = None + ) -> Dict[ConstraintData, float]: + if cons_to_load is None: + cons_to_load = self._con_map + return _get_duals( + solver_model=self._solver_model, + con_map=self._con_map, + cons_to_load=cons_to_load, + ) + + +class GurobiDirectBase(SolverBase): + + _num_gurobipy_env_clients = 0 + _gurobipy_env = None + _available = None + _gurobipy_available = gurobipy_available + _tc_map = None + _minimum_version = (0, 0, 0) + + CONFIG = GurobiConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._register_env_client() + self._callback = None + + def __del__(self): + if not python_is_shutting_down(): + self._release_env_client() + + def available(self): + if self._available is None: + # this triggers the deferred import, and for the persistent + # interface, may update the _available flag + # + # Note that we set the _available flag on the *most derived + # class* and not on the instance, or on the base class. That + # allows different derived interfaces to have different + # availability (e.g., persistent has a minimum version + # requirement that the direct interface doesn't - is that true?) + if not self._gurobipy_available: + if self._available is None: + self.__class__._available = Availability.NotFound + else: + self.__class__._available = self._check_license() + if self.version() < self._minimum_version: + self.__class__._available = Availability.BadVersion + return self._available + + @staticmethod + def release_license(): + if GurobiDirectBase._gurobipy_env is None: + return + if GurobiDirectBase._num_gurobipy_env_clients: + logger.warning( + "Call to GurobiDirectBase.release_license() with %s remaining " + "environment clients." % (GurobiDirectBase._num_gurobipy_env_clients,) + ) + GurobiDirectBase._gurobipy_env.close() + GurobiDirectBase._gurobipy_env = None + + @staticmethod + def env(): + if GurobiDirectBase._gurobipy_env is None: + with capture_output(capture_fd=True): + GurobiDirectBase._gurobipy_env = gurobipy.Env() + return GurobiDirectBase._gurobipy_env + + @staticmethod + def _register_env_client(): + GurobiDirectBase._num_gurobipy_env_clients += 1 + + @staticmethod + def _release_env_client(): + GurobiDirectBase._num_gurobipy_env_clients -= 1 + if GurobiDirectBase._num_gurobipy_env_clients <= 0: + # Note that _num_gurobipy_env_clients should never be <0, + # but if it is, release_license will issue a warning (that + # we want to know about) + GurobiDirectBase.release_license() + + def _check_license(self): + try: + model = gurobipy.Model(env=self.env()) + except gurobipy.GurobiError: + return Availability.BadLicense + + model.setParam('OutputFlag', 0) + try: + model.addVars(range(2001)) + model.optimize() + return Availability.FullLicense + except gurobipy.GurobiError: + return Availability.LimitedLicense + finally: + model.dispose() + + def version(self): + version = ( + gurobipy.GRB.VERSION_MAJOR, + gurobipy.GRB.VERSION_MINOR, + gurobipy.GRB.VERSION_TECHNICAL, + ) + return version + + def _create_solver_model(self, pyomo_model): + # should return gurobi_model, solution_loader, has_objective + raise NotImplementedError('should be implemented by derived classes') + + def _pyomo_gurobi_var_iter(self): + # generator of tuples (pyomo_var, gurobi_var) + raise NotImplementedError('should be implemented by derived classes') + + def _mipstart(self): + for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + orig_config = self.config + orig_cwd = os.getcwd() + try: + config = self.config(value=kwds, preserve_implicit=True) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + + if config.working_dir: + os.chdir(config.working_dir) + with capture_output(TeeStream(*ostreams), capture_fd=False): + gurobi_model, solution_loader, has_obj = self._create_solver_model( + model + ) + options = config.solver_options + + gurobi_model.setParam('LogToConsole', 1) + + if config.threads is not None: + gurobi_model.setParam('Threads', config.threads) + if config.time_limit is not None: + gurobi_model.setParam('TimeLimit', config.time_limit) + if config.rel_gap is not None: + gurobi_model.setParam('MIPGap', config.rel_gap) + if config.abs_gap is not None: + gurobi_model.setParam('MIPGapAbs', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in options.items(): + gurobi_model.setParam(key, option) + + timer.start('optimize') + gurobi_model.optimize(self._callback) + timer.stop('optimize') + + res = self._postsolve( + grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj + ) + finally: + os.chdir(orig_cwd) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) + self.config = orig_config + + res.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + return res + + def _get_tc_map(self): + if GurobiDirectBase._tc_map is None: + grb = gurobipy.GRB + tc = TerminationCondition + GurobiDirectBase._tc_map = { + grb.LOADED: tc.unknown, # problem is loaded, but no solution + grb.OPTIMAL: tc.convergenceCriteriaSatisfied, + grb.INFEASIBLE: tc.provenInfeasible, + grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, + grb.UNBOUNDED: tc.unbounded, + grb.CUTOFF: tc.objectiveLimit, + grb.ITERATION_LIMIT: tc.iterationLimit, + grb.NODE_LIMIT: tc.iterationLimit, + grb.TIME_LIMIT: tc.maxTimeLimit, + grb.SOLUTION_LIMIT: tc.unknown, + grb.INTERRUPTED: tc.interrupted, + grb.NUMERIC: tc.unknown, + grb.SUBOPTIMAL: tc.unknown, + grb.USER_OBJ_LIMIT: tc.objectiveLimit, + } + return GurobiDirectBase._tc_map + + def _postsolve(self, grb_model, solution_loader, has_obj): + status = grb_model.Status + + results = Results() + results.solution_loader = solution_loader + results.timing_info.gurobi_time = grb_model.Runtime + + if grb_model.SolCount > 0: + if status == gurobipy.GRB.OPTIMAL: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible + else: + results.solution_status = SolutionStatus.noSolution + + results.termination_condition = self._get_tc_map().get( + status, TerminationCondition.unknown + ) + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and self.config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if math.isfinite(grb_model.ObjVal): + results.incumbent_objective = grb_model.ObjVal + else: + results.incumbent_objective = None + except (gurobipy.GurobiError, AttributeError): + results.incumbent_objective = None + try: + results.objective_bound = grb_model.ObjBound + except (gurobipy.GurobiError, AttributeError): + if grb_model.ModelSense == ObjectiveSense.minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf + else: + results.incumbent_objective = None + results.objective_bound = None + + results.extra_info.IterCount = grb_model.getAttr('IterCount') + results.extra_info.BarIterCount = grb_model.getAttr('BarIterCount') + results.extra_info.NodeCount = grb_model.getAttr('NodeCount') + + self.config.timer.start('load solution') + if self.config.load_solutions: + if grb_model.SolCount > 0: + results.solution_loader.load_vars() + else: + raise NoFeasibleSolutionError() + self.config.timer.stop('load solution') + + # self.config gets copied a the beginning of + # solve and restored at the end, so modifying + # results.solver_config will not actually + # modify self.config + results.solver_config = self.config + results.solver_name = self.name + results.solver_version = self.version() + + return results diff --git a/pyomo/contrib/solver/solvers/gurobi_direct_minlp.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py similarity index 89% rename from pyomo/contrib/solver/solvers/gurobi_direct_minlp.py rename to pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py index bc7b8362aea..2544e0acd40 100755 --- a/pyomo/contrib/solver/solvers/gurobi_direct_minlp.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_minlp.py @@ -24,10 +24,8 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.contrib.solver.common.util import NoSolutionError -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiDirect, - GurobiDirectSolutionLoader, -) +from .gurobi_direct_base import GurobiDirectBase +from .gurobi_direct import GurobiDirectSolutionLoader from pyomo.core.base import ( Binary, @@ -584,27 +582,18 @@ def write(self, model, **options): doc='Direct interface to Gurobi version 12 and up ' 'supporting general nonlinear expressions', ) -class GurobiDirectMINLP(GurobiDirect): - def solve(self, model, **kwds): - """Solve the model. +class GurobiDirectMINLP(GurobiDirectBase): + _minimum_version = (12, 0, 0) - Args: - model (Block): a Pyomo model or Block to be solved - """ - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer + def __init__(self, **kwds): + super().__init__(**kwds) + self._var_map = None - StaleFlagManager.mark_all_as_stale() + def _pyomo_gurobi_var_iter(self): + return self._var_map.items() + def _create_solver_model(self, pyomo_model): + timer = self.config.timer timer.start('compile_model') writer = GurobiMINLPWriter() @@ -614,50 +603,11 @@ def solve(self, model, **kwds): timer.stop('compile_model') - ostreams = [io.StringIO()] + config.tee - - # set options - options = config.solver_options - - grb_model.setParam('LogToConsole', 1) - - if config.threads is not None: - grb_model.setParam('Threads', config.threads) - if config.time_limit is not None: - grb_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - grb_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - grb_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - grb_model.setParam(key, option) - - grbsol = grb_model.optimize() - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - grb_model, - grb_cons=grb_cons, - grb_vars=var_map.values(), - pyo_cons=pyo_cons, - pyo_vars=var_map.keys(), - pyo_obj=pyo_obj, - ), - ) + self._var_map = var_map + con_map = dict(zip(pyo_cons, grb_cons)) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() + solution_loader = GurobiDirectSolutionLoader( + solver_model=grb_model, var_map=var_map, con_map=con_map + ) - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res + return grb_model, solution_loader, bool(pyo_obj) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py new file mode 100644 index 00000000000..1b4b58f3c02 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -0,0 +1,1329 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from __future__ import annotations +import logging +from typing import Dict, List, Optional, Sequence, Mapping +from collections.abc import Iterable + +from pyomo.common.collections import ComponentSet, OrderedSet, ComponentMap +from pyomo.common.errors import PyomoException +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.core.base.var import VarData +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.param import ParamData +from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types +from pyomo.repn import generate_standard_repn +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import IncompatibleModelError +from pyomo.contrib.solver.common.base import PersistentSolverBase +from pyomo.core.staleflag import StaleFlagManager +from .gurobi_direct_base import ( + GurobiDirectBase, + gurobipy, + GurobiConfig, + GurobiDirectSolutionLoaderBase, +) +from .gurobi_direct import GurobiDirectSolutionLoader +from pyomo.contrib.solver.common.util import get_objective +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, + Reason, +) + + +logger = logging.getLogger(__name__) + + +class GurobiPersistentSolutionLoader(GurobiDirectSolutionLoaderBase): + def __init__(self, solver_model, var_map, con_map) -> None: + super().__init__(solver_model, var_map, con_map) + self._valid = True + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> None: + self._assert_solution_still_valid() + return super().load_vars(vars_to_load, solution_id) + + def get_primals( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_primals(vars_to_load, solution_id) + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None + ) -> Dict[ConstraintData, float]: + self._assert_solution_still_valid() + return super().get_duals(cons_to_load) + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) + + +class _MutableLowerBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('lb', value(self.expr)) + + +class _MutableUpperBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('ub', value(self.expr)) + + +class _MutableLinearCoefficient: + def __init__(self, expr, pyomo_con, con_map, pyomo_var_id, var_map, gurobi_model): + self.expr = expr + self.pyomo_con = pyomo_con + self.pyomo_var_id = pyomo_var_id + self.con_map = con_map + self.var_map = var_map + self.gurobi_model = gurobi_model + + @property + def gurobi_var(self): + return self.var_map[self.pyomo_var_id] + + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] + + def update(self): + self.gurobi_model.chgCoeff(self.gurobi_con, self.gurobi_var, value(self.expr)) + + +class _MutableRangeConstant: + def __init__( + self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model + ): + self.lhs_expr = lhs_expr + self.rhs_expr = rhs_expr + self.pyomo_con = pyomo_con + self.con_map = con_map + self.slack_name = slack_name + self.gurobi_model = gurobi_model + + def update(self): + rhs_val = value(self.rhs_expr) + lhs_val = value(self.lhs_expr) + con = self.con_map[self.pyomo_con] + con.rhs = rhs_val + slack = self.gurobi_model.getVarByName(self.slack_name) + slack.ub = rhs_val - lhs_val + + +class _MutableConstant: + def __init__(self, expr, pyomo_con, con_map): + self.expr = expr + self.pyomo_con = pyomo_con + self.con_map = con_map + + def update(self): + con = self.con_map[self.pyomo_con] + con.rhs = value(self.expr) + + +class _MutableQuadraticConstraint: + def __init__( + self, gurobi_model, pyomo_con, con_map, constant, linear_coefs, quadratic_coefs + ): + self.pyomo_con = pyomo_con + self.con_map = con_map + self.gurobi_model = gurobi_model + self.constant = constant + self.last_constant_value = value(self.constant.expr) + self.linear_coefs = linear_coefs + self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] + + def get_updated_expression(self): + gurobi_expr = self.gurobi_model.getQCRow(self.gurobi_con) + for ndx, coef in enumerate(self.linear_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_linear_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.gurobi_var + self.last_linear_coef_values[ndx] = current_coef_value + for ndx, coef in enumerate(self.quadratic_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + def get_updated_rhs(self): + return value(self.constant.expr) + + +class _MutableObjective: + def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): + self.gurobi_model = gurobi_model + self.constant: _MutableConstant = constant + self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs + self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs + self.last_quadratic_coef_values: List[float] = [ + value(i.expr) for i in self.quadratic_coefs + ] + + def get_updated_expression(self): + for ndx, coef in enumerate(self.linear_coefs): + coef.gurobi_var.obj = value(coef.expr) + self.gurobi_model.ObjCon = value(self.constant.expr) + + gurobi_expr = None + for ndx, coef in enumerate(self.quadratic_coefs): + if value(coef.expr) != self.last_quadratic_coef_values[ndx]: + if gurobi_expr is None: + self.gurobi_model.update() + gurobi_expr = self.gurobi_model.getObjective() + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + +class _MutableQuadraticCoefficient: + def __init__(self, expr, v1id, v2id, var_map): + self.expr = expr + self.var_map = var_map + self.v1id = v1id + self.v2id = v2id + + @property + def var1(self): + return self.var_map[self.v1id] + + @property + def var2(self): + return self.var_map[self.v2id] + + +class GurobiPersistentConfig(GurobiConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + GurobiConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) + + +class GurobiPersistent(GurobiDirectBase, PersistentSolverBase, Observer): + _minimum_version = (7, 0, 0) + CONFIG = GurobiPersistentConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + # we actually want to only grab the license when + # set_instance is called + self._release_env_client() + self._solver_model = None + self._pyomo_var_to_solver_var_map = ComponentMap() + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None + self._change_detector = None + self._constraint_ndx = 0 + + def _clear(self): + release = False + if self._solver_model is not None: + release = True + self._solver_model = None + if release: + self._release_env_client() + self._pyomo_var_to_solver_var_map = ComponentMap() + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object = None + self._change_detector = None + self._constraint_ndx = 0 + + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: + self.update() + else: + self.set_instance(pyomo_model) + + solution_loader = GurobiPersistentSolutionLoader( + solver_model=self._solver_model, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + ) + has_obj = self._objective is not None + return self._solver_model, solution_loader, has_obj + + def _pyomo_gurobi_var_iter(self): + return self._pyomo_var_to_solver_var_map.items() + + def release_license(self): + self._clear() + super().release_license() + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + self._needs_updated = False + return res + + def _process_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError(f'Unrecognized domain: {var.domain}') + if var.fixed: + lb = var.value + ub = lb + else: + if var._lb is not None: + lb = max(lb, value(var._lb)) + if var._ub is not None: + ub = min(ub, value(var._ub)) + if not is_constant(var._lb): + mutable_lb = _MutableLowerBound( + id(var), var.lower, self._pyomo_var_to_solver_var_map + ) + self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) + if not is_constant(var._ub): + mutable_ub = _MutableUpperBound( + id(var), var.upper, self._pyomo_var_to_solver_var_map + ) + self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) + return lb, ub, vtype + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + vtypes = [] + lbs = [] + ubs = [] + for ndx, var in enumerate(variables): + lb, ub, vtype = self._process_domain_and_bounds(var) + vtypes.append(vtype) + lbs.append(lb) + ubs.append(ub) + + gurobi_vars = self._solver_model.addVars( + len(variables), lb=lbs, ub=ubs, vtype=vtypes + ).values() + + for pyomo_var, gurobi_var in zip(variables, gurobi_vars): + self._pyomo_var_to_solver_var_map[pyomo_var] = gurobi_var + self._vars_added_since_update.update(variables) + self._needs_updated = True + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._register_env_client() + self._pyomo_model = pyomo_model + self._solver_model = gurobipy.Model(env=self.env()) + timer.start('set_instance') + self._change_detector = ModelChangeDetector( + model=self._pyomo_model, observers=[self], **dict(self.config.auto_updates) + ) + self._change_detector.config = self.config.auto_updates + timer.stop('set_instance') + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + if self._needs_updated: + self._update_gurobi_model() + self._change_detector.update(timer=timer) + timer.stop('update') + + def _get_expr_from_pyomo_repn(self, repn): + if repn.nonlinear_expr is not None: + raise IncompatibleModelError( + f'GurobiPersistent only supports linear and quadratic expressions: {repn}.' + ) + + if len(repn.linear_vars) > 0: + coef_list = [value(i) for i in repn.linear_coefs] + vlist = [self._pyomo_var_to_solver_var_map[v] for v in repn.linear_vars] + new_expr = gurobipy.LinExpr(coef_list, vlist) + else: + new_expr = 0.0 + + if len(repn.quadratic_vars) > 0: + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + gurobi_x = self._pyomo_var_to_solver_var_map[x] + gurobi_y = self._pyomo_var_to_solver_var_map[y] + new_expr += value(coef) * gurobi_x * gurobi_y + + return new_expr + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + gurobi_expr_list = [] + for ndx, con in enumerate(cons): + lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) + repn = generate_standard_repn(body, quadratic=True, compute_values=False) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + mutable_constant = None + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " f"or an upper bound: {con} \n" + ) + elif lb is None: + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) + elif ub is None: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) + elif con.equality: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) + else: + assert ( + len(repn.quadratic_vars) == 0 + ), "Quadratic range constraints are not supported" + lhs_expr = lb - repn.constant + rhs_expr = ub - repn.constant + gurobi_expr_list.append( + gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))] + ) + if not is_constant(lhs_expr) or not is_constant(rhs_expr): + conname = f'c{self._constraint_ndx}[{ndx}]' + mutable_constant = _MutableRangeConstant( + lhs_expr, + rhs_expr, + con, + self._pyomo_con_to_solver_con_map, + 'Rg' + conname, + self._solver_model, + ) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient( + c, + con, + self._pyomo_con_to_solver_con_map, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) + mlc_list.append(mlc) + + if len(repn.quadratic_vars) == 0: + if len(mlc_list) > 0: + self._mutable_helpers[con] = mlc_list + if mutable_constant is not None: + if con not in self._mutable_helpers: + self._mutable_helpers[con] = [] + self._mutable_helpers[con].append(mutable_constant) + else: + if mutable_constant is None: + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) + mqc_list.append(mqc) + mqc = _MutableQuadraticConstraint( + self._solver_model, + con, + self._pyomo_con_to_solver_con_map, + mutable_constant, + mlc_list, + mqc_list, + ) + self._mutable_quadratic_helpers[con] = mqc + + gurobi_cons = list( + self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}', + ).values() + ) + self._constraint_ndx += 1 + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + for con in cons: + level = con.level + if level == 1: + sos_type = gurobipy.GRB.SOS_TYPE1 + elif level == 2: + sos_type = gurobipy.GRB.SOS_TYPE2 + else: + raise ValueError( + f"Solver does not support SOS level {level} constraints" + ) + + gurobi_vars = [] + weights = [] + + for v, w in con.get_items(): + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v]) + weights.append(w) + + gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) + self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + if obj is not self._objective: + raise RuntimeError( + 'tried to remove an objective that has not been added: ' + f'{str(obj)}' + ) + else: + self._invalidate_last_results() + self._solver_model.setObjective(0, sense=gurobipy.GRB.MINIMIZE) + # see PR #2454 + self._solver_model.update() + self._objective = None + self._needs_updated = False + + def _add_objectives(self, objs: List[ObjectiveData]): + if len(objs) > 1: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + f'only supports single-objective problems; got {len(objs)}: ' + f'{[str(i) for i in objs]}' + ) + + if len(objs) == 0: + return + + obj = objs[0] + + if self._objective is not None: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + 'only supports single-objective problems; tried to add ' + f'an objective ({str(obj)}), but there is already an ' + f'active objective ({str(self._objective)})' + ) + + self._invalidate_last_results() + + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) + mlc_list.append(mlc) + + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) + self._objective = obj + self._needs_updated = True + + def _update_gurobi_model(self): + self._solver_model.update() + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def _remove_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_con = self._pyomo_con_to_solver_con_map[con] + self._solver_model.remove(solver_con) + del self._pyomo_con_to_solver_con_map[con] + self._mutable_helpers.pop(con, None) + self._mutable_quadratic_helpers.pop(con, None) + self._needs_updated = True + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] + self._solver_model.remove(solver_sos_con) + del self._pyomo_sos_to_solver_sos_map[con] + self._needs_updated = True + + def _remove_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + v_id = id(var) + if var in self._vars_added_since_update: + self._update_gurobi_model() + solver_var = self._pyomo_var_to_solver_var_map.pop(var) + self._solver_model.remove(solver_var) + self._mutable_bounds.pop((v_id, 'lb'), None) + self._mutable_bounds.pop((v_id, 'ub'), None) + self._needs_updated = True + + def _update_variables(self, variables: Mapping[VarData, Reason]): + self._invalidate_last_results() + new_vars = [] + old_vars = [] + mod_vars = [] + for v, reason in variables.items(): + if reason & Reason.added: + new_vars.append(v) + elif reason & Reason.removed: + old_vars.append(v) + else: + mod_vars.append(v) + + if new_vars: + self._add_variables(new_vars) + if old_vars: + self._remove_variables(old_vars) + + cons_to_reprocess = OrderedSet() + cons_to_update = OrderedSet() + reprocess_obj = False + update_obj = False + + for v in mod_vars: + reason = variables[v] + if reason & (Reason.bounds | Reason.domain | Reason.fixed | Reason.value): + var_id = id(v) + self._mutable_bounds.pop((var_id, 'lb'), None) + self._mutable_bounds.pop((var_id, 'ub'), None) + gurobipy_var = self._pyomo_var_to_solver_var_map[v] + lb, ub, vtype = self._process_domain_and_bounds(v) + gurobipy_var.setAttr('lb', lb) + gurobipy_var.setAttr('ub', ub) + gurobipy_var.setAttr('vtype', vtype) + if reason & Reason.fixed: + cons_to_reprocess.update( + self._change_detector.get_constraints_impacted_by_var(v) + ) + objs = self._change_detector.get_objectives_impacted_by_var(v) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + reprocess_obj = True + elif (reason & Reason.value) and v.fixed: + cons_to_update.update( + self._change_detector.get_constraints_impacted_by_var(v) + ) + objs = self._change_detector.get_objectives_impacted_by_var(v) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + update_obj = True + + self._remove_constraints(cons_to_reprocess) + self._add_constraints(cons_to_reprocess) + cons_to_update -= cons_to_reprocess + for c in cons_to_update: + if c in self._mutable_helpers: + for i in self._mutable_helpers[c]: + i.update() + self._update_quadratic_constraint(c) + + if reprocess_obj: + obj = self._objective + self._remove_objectives([obj]) + self._add_objectives([obj]) + elif update_obj: + self._mutable_objective_update() + + self._needs_updated = True + + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + self._invalidate_last_results() + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.expr: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_constraints(old_cons) + if new_cons: + self._add_constraints(new_cons) + self._needs_updated = True + + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + self._invalidate_last_results() + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.sos_items: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_sos_constraints(old_cons) + if new_cons: + self._add_sos_constraints(new_cons) + self._needs_updated = True + + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + self._invalidate_last_results() + new_objs = [] + old_objs = [] + new_sense = [] + for obj, reason in objs.items(): + if reason & Reason.added: + new_objs.append(obj) + elif reason & Reason.removed: + old_objs.append(obj) + elif reason & Reason.expr: + old_objs.append(obj) + new_objs.append(obj) + elif reason & Reason.sense: + new_sense.append(obj) + + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) + if new_sense: + assert len(new_sense) == 1 + obj = new_sense[0] + assert obj is self._objective + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + self._solver_model.ModelSense = sense + + def _update_quadratic_constraint(self, c: ConstraintData): + if c in self._mutable_quadratic_helpers: + if c in self._constraints_added_since_update: + self._update_gurobi_model() + helper = self._mutable_quadratic_helpers[c] + gurobi_con = helper.gurobi_con + new_gurobi_expr = helper.get_updated_expression() + new_rhs = helper.get_updated_rhs() + new_sense = gurobi_con.qcsense + self._solver_model.remove(gurobi_con) + new_con = self._solver_model.addQConstr(new_gurobi_expr, new_sense, new_rhs) + self._pyomo_con_to_solver_con_map[c] = new_con + assert helper.pyomo_con is c + self._constraints_added_since_update.add(c) + + def _mutable_objective_update(self): + if self._mutable_objective is not None: + new_gurobi_expr = self._mutable_objective.get_updated_expression() + if new_gurobi_expr is not None: + if self._objective.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + else: + sense = gurobipy.GRB.MAXIMIZE + # TODO: need a test for when part of the object is linear + # and part of the objective is quadratic, but both + # parts have mutable coefficients + self._solver_model.setObjective(new_gurobi_expr, sense=sense) + + def _update_parameters(self, params: Mapping[ParamData, Reason]): + self._invalidate_last_results() + + cons_to_update = OrderedSet() + update_obj = False + vars_to_update = ComponentSet() + for p, reason in params.items(): + if reason & Reason.added: + continue + if reason & Reason.removed: + continue + if reason & Reason.value: + cons_to_update.update( + self._change_detector.get_constraints_impacted_by_param(p) + ) + objs = self._change_detector.get_objectives_impacted_by_param(p) + if objs: + assert len(objs) == 1 + assert objs[0] is self._objective + update_obj = True + vars_to_update.update( + self._change_detector.get_variables_impacted_by_param(p) + ) + + for c in cons_to_update: + if c in self._mutable_helpers: + for i in self._mutable_helpers[c]: + i.update() + self._update_quadratic_constraint(c) + + if update_obj: + self._mutable_objective_update() + + for v in vars_to_update: + vid = id(v) + if (vid, 'lb') in self._mutable_bounds: + self._mutable_bounds[(vid, 'lb')][1].update() + if (vid, 'ub') in self._mutable_bounds: + self._mutable_bounds[(vid, 'ub')][1].update() + + self._needs_updated = True + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def get_model_attr(self, attr): + """ + Get the value of an attribute on the Gurobi model. + + Parameters + ---------- + attr: str + The attribute to get. See Gurobi documentation for descriptions of the attributes. + """ + if self._needs_updated: + self._update_gurobi_model() + return self._solver_model.getAttr(attr) + + def write(self, filename): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + """ + self._solver_model.write(filename) + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def set_linear_constraint_attr(self, con, attr, val): + """ + Set the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be modified. + attr: str + The attribute to be modified. Options are: + CBasis + DStart + Lazy + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'Sense', 'RHS', 'ConstrName'}: + raise ValueError( + f'Linear constraint attr {attr} cannot be set with' + ' the set_linear_constraint_attr method. Please use' + ' the remove_constraint and add_constraint methods.' + ) + self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) + self._needs_updated = True + + def set_var_attr(self, var, attr, val): + """ + Set the value of an attribute on a gurobi variable. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be modified. + attr: str + The attribute to be modified. Options are: + Start + VarHintVal + VarHintPri + BranchPriority + VBasis + PStart + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'LB', 'UB', 'VType', 'VarName'}: + raise ValueError( + f'Var attr {attr} cannot be set with' + ' the set_var_attr method. Please use' + ' the update_var method.' + ) + if attr == 'Obj': + raise ValueError( + 'Var attr Obj cannot be set with' + ' the set_var_attr method. Please use' + ' the set_objective method.' + ) + self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) + self._needs_updated = True + + def get_var_attr(self, var, attr): + """ + Get the value of an attribute on a gurobi var. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be retrieved. + attr: str + The attribute to get. See gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) + + def get_linear_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def get_sos_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi sos constraint. + + Parameters + ---------- + con: pyomo.core.base.sos.SOSConstraintData + The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) + + def get_quadratic_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi quadratic constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def set_gurobi_param(self, param, val): + """ + Set a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to set. Options include any gurobi parameter. + Please see the Gurobi documentation for options. + val: any + The value to set the parameter to. See Gurobi documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_gurobi_param_info(self, param): + """ + Get information about a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to get info for. See Gurobi documentation for possible options. + + Returns + ------- + six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. + """ + return self._solver_model.getParamInfo(param) + + def _intermediate_callback(self): + def f(gurobi_model, where): + self._callback_func(self._pyomo_model, self, where) + + return f + + def set_callback(self, func=None): + """ + Specify a callback for gurobi to use. + + Parameters + ---------- + func: function + The function to call. The function should have three arguments. The first will be the pyomo model being + solved. The second will be the GurobiPersistent instance. The third will be an enum member of + gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For + example, suppose we want to solve + + .. math:: + + min 2*x + y + + s.t. + + y >= (x-2)**2 + + 0 <= x <= 4 + + y >= 0 + + y integer + + as an MILP using extended cutting planes in callbacks. + + >>> from gurobipy import GRB # doctest:+SKIP + >>> import pyomo.environ as pyo + >>> from pyomo.core.expr.taylor_series import taylor_series_expansion + >>> from pyomo.contrib import appsi + >>> + >>> m = pyo.ConcreteModel() + >>> m.x = pyo.Var(bounds=(0, 4)) + >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) + >>> m.obj = pyo.Objective(expr=2*m.x + m.y) + >>> m.cons = pyo.ConstraintList() # for the cutting planes + >>> + >>> def _add_cut(xval): + ... # a function to generate the cut + ... m.x.value = xval + ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) + ... + >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x + >>> _c = _add_cut(4) # this is an arbitrary choice + >>> + >>> opt = appsi.solvers.Gurobi() + >>> opt.config.stream_solver = True + >>> opt.set_instance(m) # doctest:+SKIP + >>> opt.gurobi_options['PreCrush'] = 1 + >>> opt.gurobi_options['LazyConstraints'] = 1 + >>> + >>> def my_callback(cb_m, cb_opt, cb_where): + ... if cb_where == GRB.Callback.MIPSOL: + ... cb_opt.cbGetSolution(variables=[m.x, m.y]) + ... if m.y.value < (m.x.value - 2)**2 - 1e-6: + ... cb_opt.cbLazy(_add_cut(m.x.value)) + ... + >>> opt.set_callback(my_callback) + >>> res = opt.solve(m) # doctest:+SKIP + + """ + if func is not None: + self._callback_func = func + self._callback = self._intermediate_callback() + else: + self._callback = None + self._callback_func = None + + def cbCut(self, con): + """ + Add a cut within a callback. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The cut to add + """ + if not con.active: + raise ValueError('cbCut expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbCut expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbCut.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbGet(self, what): + return self._solver_model.cbGet(what) + + def cbGetNodeRel(self, variables): + """ + Parameters + ---------- + variables: Var or iterable of Var + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetNodeRel(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbGetSolution(self, variables): + """ + Parameters + ---------- + variables: iterable of vars + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetSolution(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbLazy(self, con): + """ + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The lazy constraint to add + """ + if not con.active: + raise ValueError('cbLazy expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbLazy expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbLazy.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbSetSolution(self, variables, solution): + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + self._solver_model.cbSetSolution(gurobi_vars, solution) + + def cbUseSolution(self): + return self._solver_model.cbUseSolution() + + def reset(self): + self._solver_model.reset() + + def add_constraints(self, cons): + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData): + self._change_detector.add_objectives([obj]) + + def remove_constraints(self, cons): + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + self._change_detector.remove_sos_constraints(cons) + + def update_variables(self, variables): + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + self._change_detector.update_parameters(params) diff --git a/pyomo/contrib/solver/solvers/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi_direct.py deleted file mode 100644 index 2a27eba893a..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_direct.py +++ /dev/null @@ -1,481 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import datetime -import io -import math -import operator -import os -import logging - -from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler - -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase - -logger = logging.getLogger(__name__) - -gurobipy, gurobipy_available = attempt_import('gurobipy') - - -class GurobiConfigMixin: - """ - Mixin class for Gurobi-specific configurations - """ - - def __init__(self): - self.use_mipstart: bool = self.declare( - 'use_mipstart', - ConfigValue( - default=False, - domain=bool, - description="If True, the current values of the integer variables " - "will be passed to Gurobi.", - ), - ) - - -class GurobiConfig(BranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - BranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyo_obj): - self._grb_model = grb_model - self._grb_cons = grb_cons - self._grb_vars = grb_vars - self._pyo_cons = pyo_cons - self._pyo_vars = pyo_vars - self._pyo_obj = pyo_obj - GurobiDirect._register_env_client() - - def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._grb_model is not None: - self._grb_cons = None - self._grb_vars = None - self._pyo_cons = None - self._pyo_vars = None - self._pyo_obj = None - # explicitly release the model - self._grb_model.dispose() - self._grb_model = None - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirect._release_env_client() - - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, map(operator.attrgetter('x'), self._grb_vars)) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - for p_var, g_var in iterator: - p_var.set_value(g_var, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, map(operator.attrgetter('x'), self._grb_vars)) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - def get_duals(self, cons_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - def dedup(_iter): - last = None - for con_info_dual in _iter: - if not con_info_dual[1] and con_info_dual[0] is last: - continue - last = con_info_dual[0] - yield con_info_dual - - iterator = dedup( - zip(self._pyo_cons, map(operator.attrgetter('Pi'), self._grb_cons)) - ) - if cons_to_load: - cons_to_load = set(cons_to_load) - iterator = filter( - lambda con_info_dual: con_info_dual[0] in cons_to_load, iterator - ) - return {con_info: dual for con_info, dual in iterator} - - def get_reduced_costs(self, vars_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - iterator = zip(self._pyo_vars, map(operator.attrgetter('Rc'), self._grb_vars)) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - -class GurobiSolverMixin: - """ - gurobi_direct and gurobi_persistent check availability and set versions - in the same way. This moves the logic to a central location to reduce - duplicate code. - """ - - _num_gurobipy_env_clients = 0 - _gurobipy_env = None - _available = None - _gurobipy_available = gurobipy_available - - def available(self): - if self._available is None: - # this triggers the deferred import, and for the persistent - # interface, may update the _available flag - # - # Note that we set the _available flag on the *most derived - # class* and not on the instance, or on the base class. That - # allows different derived interfaces to have different - # availability (e.g., persistent has a minimum version - # requirement that the direct interface doesn't) - if not self._gurobipy_available: - if self._available is None: - self.__class__._available = Availability.NotFound - else: - self.__class__._available = self._check_license() - return self._available - - @staticmethod - def release_license(): - if GurobiSolverMixin._gurobipy_env is None: - return - if GurobiSolverMixin._num_gurobipy_env_clients: - logger.warning( - "Call to GurobiSolverMixin.release_license() with %s remaining " - "environment clients." % (GurobiSolverMixin._num_gurobipy_env_clients,) - ) - GurobiSolverMixin._gurobipy_env.close() - GurobiSolverMixin._gurobipy_env = None - - @staticmethod - def env(): - if GurobiSolverMixin._gurobipy_env is None: - with capture_output(capture_fd=True): - GurobiSolverMixin._gurobipy_env = gurobipy.Env() - return GurobiSolverMixin._gurobipy_env - - @staticmethod - def _register_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients += 1 - - @staticmethod - def _release_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients -= 1 - if GurobiSolverMixin._num_gurobipy_env_clients <= 0: - # Note that _num_gurobipy_env_clients should never be <0, - # but if it is, release_license will issue a warning (that - # we want to know about) - GurobiSolverMixin.release_license() - - def _check_license(self): - try: - model = gurobipy.Model(env=self.env()) - except gurobipy.GurobiError: - return Availability.BadLicense - - model.setParam('OutputFlag', 0) - try: - model.addVars(range(2001)) - model.optimize() - return Availability.FullLicense - except gurobipy.GurobiError: - return Availability.LimitedLicense - finally: - model.dispose() - - def version(self): - version = ( - gurobipy.GRB.VERSION_MAJOR, - gurobipy.GRB.VERSION_MINOR, - gurobipy.GRB.VERSION_TECHNICAL, - ) - return version - - -class GurobiDirect(GurobiSolverMixin, SolverBase): - """ - Interface to Gurobi using gurobipy - """ - - CONFIG = GurobiConfig() - - _tc_map = None - - def __init__(self, **kwds): - super().__init__(**kwds) - self._register_env_client() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - - timer.start('compile_model') - repn = LinearStandardFormCompiler().write( - model, mixed_form=True, set_sense=None - ) - timer.stop('compile_model') - - if len(repn.objectives) > 1: - raise IncompatibleModelError( - f"The {self.__class__.__name__} solver only supports models " - f"with zero or one objectives (received {len(repn.objectives)})." - ) - - timer.start('prepare_matrices') - inf = float('inf') - ninf = -inf - bounds = list(map(operator.attrgetter('bounds'), repn.columns)) - lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] - ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] - CON = gurobipy.GRB.CONTINUOUS - BIN = gurobipy.GRB.BINARY - INT = gurobipy.GRB.INTEGER - vtype = [ - ( - CON - if v.is_continuous() - else BIN if v.is_binary() else INT if v.is_integer() else '?' - ) - for v in repn.columns - ] - sense_type = list('=<>') # Note: ordering matches 0, 1, -1 - sense = [sense_type[r[1]] for r in repn.rows] - timer.stop('prepare_matrices') - - ostreams = [io.StringIO()] + config.tee - res = Results() - - orig_cwd = os.getcwd() - try: - if config.working_dir: - os.chdir(config.working_dir) - with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model = gurobipy.Model(env=self.env()) - - timer.start('transfer_model') - x = gurobi_model.addMVar( - len(repn.columns), - lb=lb, - ub=ub, - obj=repn.c.todense()[0] if repn.c.shape[0] else 0, - vtype=vtype, - ) - A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) - if repn.c.shape[0]: - gurobi_model.setAttr('ObjCon', repn.c_offset[0]) - gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) - # Note: calling gurobi_model.update() here is not - # necessary (it will happen as part of optimize()): - # gurobi_model.update() - timer.stop('transfer_model') - - options = config.solver_options - - gurobi_model.setParam('LogToConsole', 1) - - if config.threads is not None: - gurobi_model.setParam('Threads', config.threads) - if config.time_limit is not None: - gurobi_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - gurobi_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - gurobi_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - gurobi_model.setParam(key, option) - - timer.start('optimize') - gurobi_model.optimize() - timer.stop('optimize') - finally: - os.chdir(orig_cwd) - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - gurobi_model, - A, - x.tolist(), - list(map(operator.itemgetter(0), repn.rows)), - repn.columns, - repn.objectives, - ), - ) - - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res - - def _postsolve(self, timer: HierarchicalTimer, config, loader): - grb_model = loader._grb_model - status = grb_model.Status - - results = Results() - results.solution_loader = loader - results.timing_info.gurobi_time = grb_model.Runtime - - if grb_model.SolCount > 0: - if status == gurobipy.GRB.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - results.termination_condition = self._get_tc_map().get( - status, TerminationCondition.unknown - ) - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - if loader._pyo_obj: - try: - if math.isfinite(grb_model.ObjVal): - results.incumbent_objective = grb_model.ObjVal - else: - results.incumbent_objective = None - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = grb_model.ObjBound - except (gurobipy.GurobiError, AttributeError): - if grb_model.ModelSense == ObjectiveSense.minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - else: - results.incumbent_objective = None - results.objective_bound = None - - results.extra_info.IterCount = grb_model.getAttr('IterCount') - results.extra_info.BarIterCount = grb_model.getAttr('BarIterCount') - results.extra_info.NodeCount = grb_model.getAttr('NodeCount') - - timer.start('load solution') - if config.load_solutions: - if grb_model.SolCount > 0: - results.solution_loader.load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _get_tc_map(self): - if GurobiDirect._tc_map is None: - grb = gurobipy.GRB - tc = TerminationCondition - GurobiDirect._tc_map = { - grb.LOADED: tc.unknown, # problem is loaded, but no solution - grb.OPTIMAL: tc.convergenceCriteriaSatisfied, - grb.INFEASIBLE: tc.provenInfeasible, - grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, - grb.UNBOUNDED: tc.unbounded, - grb.CUTOFF: tc.objectiveLimit, - grb.ITERATION_LIMIT: tc.iterationLimit, - grb.NODE_LIMIT: tc.iterationLimit, - grb.TIME_LIMIT: tc.maxTimeLimit, - grb.SOLUTION_LIMIT: tc.unknown, - grb.INTERRUPTED: tc.interrupted, - grb.NUMERIC: tc.unknown, - grb.SUBOPTIMAL: tc.unknown, - grb.USER_OBJ_LIMIT: tc.objectiveLimit, - } - return GurobiDirect._tc_map diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py deleted file mode 100644 index dd2739b46f7..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ /dev/null @@ -1,1411 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import io -import logging -import math -from typing import List, Optional -from collections.abc import Iterable - -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler -from pyomo.core.base.var import VarData -from pyomo.core.base.constraint import ConstraintData -from pyomo.core.base.sos import SOSConstraintData -from pyomo.core.base.param import ParamData -from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types -from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader -from pyomo.core.staleflag import StaleFlagManager - - -logger = logging.getLogger(__name__) - - -def _import_gurobipy(): - try: - import gurobipy - except ImportError: - GurobiPersistent._available = Availability.NotFound - raise - if gurobipy.GRB.VERSION_MAJOR < 7: - GurobiPersistent._available = Availability.BadVersion - raise ImportError('The Persistent Gurobi interface requires gurobipy>=7.0.0') - return gurobipy - - -gurobipy, gurobipy_available = attempt_import('gurobipy', importer=_import_gurobipy) - - -class GurobiConfig(PersistentBranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - PersistentBranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - def get_primals(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - -class _MutableLowerBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('lb', value(self.expr)) - - -class _MutableUpperBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('ub', value(self.expr)) - - -class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None - - def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) - - -class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None - - def update(self): - rhs_val = value(self.rhs_expr) - lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val - slack = self.gurobi_model.getVarByName(self.slack_name) - slack.ub = rhs_val - lhs_val - - -class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None - - def update(self): - self.con.rhs = value(self.expr) - - -class _MutableQuadraticConstraint: - def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs - ): - self.con = gurobi_con - self.gurobi_model = gurobi_model - self.constant = constant - self.last_constant_value = value(self.constant.expr) - self.linear_coefs = linear_coefs - self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) - for ndx, coef in enumerate(self.linear_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_linear_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var - self.last_linear_coef_values[ndx] = current_coef_value - for ndx, coef in enumerate(self.quadratic_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - def get_updated_rhs(self): - return value(self.constant.expr) - - -class _MutableObjective: - def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): - self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) - self.gurobi_model.ObjCon = value(self.constant.expr) - - gurobi_expr = None - for ndx, coef in enumerate(self.quadratic_coefs): - if value(coef.expr) != self.last_quadratic_coef_values[ndx]: - if gurobi_expr is None: - self.gurobi_model.update() - gurobi_expr = self.gurobi_model.getObjective() - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - -class _MutableQuadraticCoefficient: - def __init__(self): - self.expr = None - self.var1 = None - self.var2 = None - - -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - CONFIG = GurobiConfig() - _gurobipy_available = gurobipy_available - - def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - self._register_env_client() - self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None - - def release_license(self): - self._reinit() - self.__class__.release_license() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - @property - def symbol_map(self): - return self._symbol_map - - def _solve(self): - config = self._active_config - timer = config.timer - ostreams = [io.StringIO()] + config.tee - - with capture_output(TeeStream(*ostreams), capture_fd=False): - options = config.solver_options - - self._solver_model.setParam('LogToConsole', 1) - - if config.threads is not None: - self._solver_model.setParam('Threads', config.threads) - if config.time_limit is not None: - self._solver_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - self._solver_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - self._solver_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) - - for key, option in options.items(): - self._solver_model.setParam(key, option) - - timer.start('optimize') - self._solver_model.optimize(self._callback) - timer.stop('optimize') - - self._needs_updated = False - res = self._postsolve(timer) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - return res - - def _process_domain_and_bounds( - self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var - ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] - lb, ub, step = _domain_interval - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if _fixed: - lb = _value - ub = _value - else: - if _lb is not None: - if not is_constant(_lb): - mutable_bound = _MutableLowerBound(NPV_MaxExpression((_lb, lb))) - if gurobipy_var is None: - mutable_lbs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'lb'] = (var, mutable_bound) - lb = max(value(_lb), lb) - if _ub is not None: - if not is_constant(_ub): - mutable_bound = _MutableUpperBound(NPV_MinExpression((_ub, ub))) - if gurobipy_var is None: - mutable_ubs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'ub'] = (var, mutable_bound) - ub = min(value(_ub), ub) - - return lb, ub, vtype - - def _add_variables(self, variables: List[VarData]): - var_names = [] - vtypes = [] - lbs = [] - ubs = [] - mutable_lbs = {} - mutable_ubs = {} - for ndx, var in enumerate(variables): - varname = self._symbol_map.getSymbol(var, self._labeler) - lb, ub, vtype = self._process_domain_and_bounds( - var, id(var), mutable_lbs, mutable_ubs, ndx, None - ) - var_names.append(varname) - vtypes.append(vtype) - lbs.append(lb) - ubs.append(ub) - - gurobi_vars = self._solver_model.addVars( - len(variables), lb=lbs, ub=ubs, vtype=vtypes, name=var_names - ) - - for ndx, pyomo_var in enumerate(variables): - gurobi_var = gurobi_vars[ndx] - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - for ndx, mutable_bound in mutable_lbs.items(): - mutable_bound.var = gurobi_vars[ndx] - for ndx, mutable_bound in mutable_ubs.items(): - mutable_bound.var = gurobi_vars[ndx] - self._vars_added_since_update.update(variables) - self._needs_updated = True - - def _add_parameters(self, params: List[ParamData]): - pass - - def _reinit(self): - saved_config = self.config - saved_tmp_config = self._active_config - self.__init__(treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) - # Note that __init__ registers a new env client, so we need to - # release it here: - self._release_env_client() - self.config = saved_config - self._active_config = saved_tmp_config - - def set_instance(self, model): - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - self._reinit() - self._model = model - - if self.config.symbolic_solver_labels: - self._labeler = TextLabeler() - else: - self._labeler = NumericLabeler('x') - - self._solver_model = gurobipy.Model(name=model.name or '', env=self.env()) - - self.add_block(model) - if self._objective is None: - self.set_objective(None) - - def _get_expr_from_pyomo_expr(self, expr): - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - repn = generate_standard_repn(expr, quadratic=True, compute_values=False) - - degree = repn.polynomial_degree() - if (degree is None) or (degree > 2): - raise IncompatibleModelError( - f'GurobiAuto does not support expressions of degree {degree}.' - ) - - if len(repn.linear_vars) > 0: - linear_coef_vals = [] - for ndx, coef in enumerate(repn.linear_coefs): - if not is_constant(coef): - mutable_linear_coefficient = _MutableLinearCoefficient() - mutable_linear_coefficient.expr = coef - mutable_linear_coefficient.var = self._pyomo_var_to_solver_var_map[ - id(repn.linear_vars[ndx]) - ] - mutable_linear_coefficients.append(mutable_linear_coefficient) - linear_coef_vals.append(value(coef)) - new_expr = gurobipy.LinExpr( - linear_coef_vals, - [self._pyomo_var_to_solver_var_map[id(i)] for i in repn.linear_vars], - ) - else: - new_expr = 0.0 - - for ndx, v in enumerate(repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - coef = repn.quadratic_coefs[ndx] - if not is_constant(coef): - mutable_quadratic_coefficient = _MutableQuadraticCoefficient() - mutable_quadratic_coefficient.expr = coef - mutable_quadratic_coefficient.var1 = gurobi_x - mutable_quadratic_coefficient.var2 = gurobi_y - mutable_quadratic_coefficients.append(mutable_quadratic_coefficient) - coef_val = value(coef) - new_expr += coef_val * gurobi_x * gurobi_y - - return ( - new_expr, - repn.constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - - def _add_constraints(self, cons: List[ConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if ( - gurobi_expr.__class__ in {gurobipy.LinExpr, gurobipy.Var} - or gurobi_expr.__class__ in native_numeric_types - ): - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_lb() and con.has_ub(): - lhs_expr = con.lower - repn_constant - rhs_expr = con.upper - repn_constant - lhs_val = value(lhs_expr) - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addRange( - gurobi_expr, lhs_val, rhs_val, name=conname - ) - self._range_constraints.add(con) - if not is_constant(lhs_expr) or not is_constant(rhs_expr): - mutable_range_constant = _MutableRangeConstant() - mutable_range_constant.lhs_expr = lhs_expr - mutable_range_constant.rhs_expr = rhs_expr - mutable_range_constant.con = gurobipy_con - mutable_range_constant.slack_name = 'Rg' + conname - mutable_range_constant.gurobi_model = self._solver_model - self._mutable_helpers[con] = [mutable_range_constant] - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - for tmp in mutable_linear_coefficients: - tmp.con = gurobipy_con - tmp.gurobi_model = self._solver_model - if len(mutable_linear_coefficients) > 0: - if con not in self._mutable_helpers: - self._mutable_helpers[con] = mutable_linear_coefficients - else: - self._mutable_helpers[con].extend(mutable_linear_coefficients) - elif gurobi_expr.__class__ is gurobipy.QuadExpr: - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - elif con.has_lb() and con.has_ub(): - raise NotImplementedError( - 'Quadratic range constraints are not supported' - ) - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - if ( - len(mutable_linear_coefficients) > 0 - or len(mutable_quadratic_coefficients) > 0 - or not is_constant(repn_constant) - ): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_quadratic_constraint = _MutableQuadraticConstraint( - self._solver_model, - gurobipy_con, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_quadratic_helpers[con] = mutable_quadratic_constraint - else: - raise ValueError( - f'Unrecognized Gurobi expression type: {str(gurobi_expr.__class__)}' - ) - - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - level = con.level - if level == 1: - sos_type = gurobipy.GRB.SOS_TYPE1 - elif level == 2: - sos_type = gurobipy.GRB.SOS_TYPE2 - else: - raise ValueError( - f"Solver does not support SOS level {level} constraints" - ) - - gurobi_vars = [] - weights = [] - - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) - weights.append(w) - - gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) - self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _remove_constraints(self, cons: List[ConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_con = self._pyomo_con_to_solver_con_map[con] - self._solver_model.remove(solver_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_con_to_solver_con_map[con] - del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) - self._mutable_helpers.pop(con, None) - self._mutable_quadratic_helpers.pop(con, None) - self._needs_updated = True - - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] - self._solver_model.remove(solver_sos_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_sos_to_solver_sos_map[con] - self._needs_updated = True - - def _remove_variables(self, variables: List[VarData]): - for var in variables: - v_id = id(var) - if var in self._vars_added_since_update: - self._update_gurobi_model() - solver_var = self._pyomo_var_to_solver_var_map[v_id] - self._solver_model.remove(solver_var) - self._symbol_map.removeSymbol(var) - del self._pyomo_var_to_solver_var_map[v_id] - self._mutable_bounds.pop(v_id, None) - self._needs_updated = True - - def _remove_parameters(self, params: List[ParamData]): - pass - - def _update_variables(self, variables: List[VarData]): - for var in variables: - var_id = id(var) - if var_id not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f'The Var provided to update_var needs to be added first: {var}' - ) - self._mutable_bounds.pop((var_id, 'lb'), None) - self._mutable_bounds.pop((var_id, 'ub'), None) - gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] - lb, ub, vtype = self._process_domain_and_bounds( - var, var_id, None, None, None, gurobipy_var - ) - gurobipy_var.setAttr('lb', lb) - gurobipy_var.setAttr('ub', ub) - gurobipy_var.setAttr('vtype', vtype) - self._needs_updated = True - - def update_parameters(self): - for con, helpers in self._mutable_helpers.items(): - for helper in helpers: - helper.update() - for k, (v, helper) in self._mutable_bounds.items(): - helper.update() - - for con, helper in self._mutable_quadratic_helpers.items(): - if con in self._constraints_added_since_update: - self._update_gurobi_model() - gurobi_con = helper.con - new_gurobi_expr = helper.get_updated_expression() - new_rhs = helper.get_updated_rhs() - new_sense = gurobi_con.qcsense - pyomo_con = self._solver_con_to_pyomo_con_map[id(gurobi_con)] - name = self._symbol_map.getSymbol(pyomo_con, self._labeler) - self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, name=name - ) - self._pyomo_con_to_solver_con_map[id(pyomo_con)] = new_con - del self._solver_con_to_pyomo_con_map[id(gurobi_con)] - self._solver_con_to_pyomo_con_map[id(new_con)] = pyomo_con - helper.con = new_con - self._constraints_added_since_update.add(con) - - helper = self._mutable_objective - pyomo_obj = self._objective - new_gurobi_expr = helper.get_updated_expression() - if new_gurobi_expr is not None: - if pyomo_obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - else: - sense = gurobipy.GRB.MAXIMIZE - self._solver_model.setObjective(new_gurobi_expr, sense=sense) - - def _set_objective(self, obj): - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE - else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(obj.expr) - - mutable_constant = _MutableConstant() - mutable_constant.expr = repn_constant - mutable_objective = _MutableObjective( - self._solver_model, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_objective = mutable_objective - - # These two lines are needed as a workaround - # see PR #2454 - self._solver_model.setObjective(0) - self._solver_model.update() - - self._solver_model.setObjective(gurobi_expr + value(repn_constant), sense=sense) - self._needs_updated = True - - def _postsolve(self, timer: HierarchicalTimer): - config = self._active_config - - gprob = self._solver_model - grb = gurobipy.GRB - status = gprob.Status - - results = Results() - results.solution_loader = GurobiSolutionLoader(self) - results.timing_info.gurobi_time = gprob.Runtime - - if gprob.SolCount > 0: - if status == grb.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - if status == grb.LOADED: # problem is loaded, but no solution - results.termination_condition = TerminationCondition.unknown - elif status == grb.OPTIMAL: # optimal - results.termination_condition = ( - TerminationCondition.convergenceCriteriaSatisfied - ) - elif status == grb.INFEASIBLE: - results.termination_condition = TerminationCondition.provenInfeasible - elif status == grb.INF_OR_UNBD: - results.termination_condition = TerminationCondition.infeasibleOrUnbounded - elif status == grb.UNBOUNDED: - results.termination_condition = TerminationCondition.unbounded - elif status == grb.CUTOFF: - results.termination_condition = TerminationCondition.objectiveLimit - elif status == grb.ITERATION_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.NODE_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.TIME_LIMIT: - results.termination_condition = TerminationCondition.maxTimeLimit - elif status == grb.SOLUTION_LIMIT: - results.termination_condition = TerminationCondition.unknown - elif status == grb.INTERRUPTED: - results.termination_condition = TerminationCondition.interrupted - elif status == grb.NUMERIC: - results.termination_condition = TerminationCondition.unknown - elif status == grb.SUBOPTIMAL: - results.termination_condition = TerminationCondition.unknown - elif status == grb.USER_OBJ_LIMIT: - results.termination_condition = TerminationCondition.objectiveLimit - else: - results.termination_condition = TerminationCondition.unknown - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - results.incumbent_objective = None - results.objective_bound = None - if self._objective is not None: - try: - results.incumbent_objective = gprob.ObjVal - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = gprob.ObjBound - except (gurobipy.GurobiError, AttributeError): - if self._objective.sense == minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - - if results.incumbent_objective is not None and not math.isfinite( - results.incumbent_objective - ): - results.incumbent_objective = None - - results.extra_info.IterCount = gprob.getAttr('IterCount') - results.extra_info.BarIterCount = gprob.getAttr('BarIterCount') - results.extra_info.NodeCount = gprob.getAttr('NodeCount') - - timer.start('load solution') - if config.load_solutions: - if gprob.SolCount > 0: - self._load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _load_suboptimal_mip_solution(self, vars_to_load, solution_number): - if ( - self.get_model_attr('NumIntVars') == 0 - and self.get_model_attr('NumBinVars') == 0 - ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - original_solution_number = self.get_gurobi_param_info('SolutionNumber')[2] - self.set_gurobi_param('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = self._solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - self.set_gurobi_param('SolutionNumber', original_solution_number) - return res - - def _load_vars(self, vars_to_load=None, solution_number=0): - for v, val in self._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ).items(): - v.set_value(val, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def _get_primals(self, vars_to_load=None, solution_number=0): - if self._needs_updated: - self._update_gurobi_model() # this is needed to ensure that solutions cannot be loaded after the model has been changed - - if self._solver_model.SolCount == 0: - raise NoSolutionError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - if solution_number != 0: - return self._load_suboptimal_mip_solution( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("X", gurobi_vars_to_load) - - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - return res - - def _get_reduced_costs(self, vars_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - res = ComponentMap() - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("Rc", gurobi_vars_to_load) - - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - - return res - - def _get_duals(self, cons_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = {} - - if cons_to_load is None: - linear_cons_to_load = self._solver_model.getConstrs() - quadratic_cons_to_load = self._solver_model.getQConstrs() - else: - gurobi_cons_to_load = OrderedSet( - [con_map[pyomo_con] for pyomo_con in cons_to_load] - ) - linear_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getConstrs()) - ) - ) - quadratic_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getQConstrs()) - ) - ) - linear_vals = self._solver_model.getAttr("Pi", linear_cons_to_load) - quadratic_vals = self._solver_model.getAttr("QCPi", quadratic_cons_to_load) - - for gurobi_con, val in zip(linear_cons_to_load, linear_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - for gurobi_con, val in zip(quadratic_cons_to_load, quadratic_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - - return dual - - def update(self, timer: HierarchicalTimer = None): - if self._needs_updated: - self._update_gurobi_model() - super().update(timer=timer) - self._update_gurobi_model() - - def _update_gurobi_model(self): - self._solver_model.update() - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def get_model_attr(self, attr): - """ - Get the value of an attribute on the Gurobi model. - - Parameters - ---------- - attr: str - The attribute to get. See Gurobi documentation for descriptions of the attributes. - """ - if self._needs_updated: - self._update_gurobi_model() - return self._solver_model.getAttr(attr) - - def write(self, filename): - """ - Write the model to a file (e.g., and lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - """ - self._solver_model.write(filename) - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def set_linear_constraint_attr(self, con, attr, val): - """ - Set the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be modified. - attr: str - The attribute to be modified. Options are: - CBasis - DStart - Lazy - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'Sense', 'RHS', 'ConstrName'}: - raise ValueError( - f'Linear constraint attr {attr} cannot be set with' - ' the set_linear_constraint_attr method. Please use' - ' the remove_constraint and add_constraint methods.' - ) - self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) - self._needs_updated = True - - def set_var_attr(self, var, attr, val): - """ - Set the value of an attribute on a gurobi variable. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be modified. - attr: str - The attribute to be modified. Options are: - Start - VarHintVal - VarHintPri - BranchPriority - VBasis - PStart - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'LB', 'UB', 'VType', 'VarName'}: - raise ValueError( - f'Var attr {attr} cannot be set with' - ' the set_var_attr method. Please use' - ' the update_var method.' - ) - if attr == 'Obj': - raise ValueError( - 'Var attr Obj cannot be set with' - ' the set_var_attr method. Please use' - ' the set_objective method.' - ) - self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) - self._needs_updated = True - - def get_var_attr(self, var, attr): - """ - Get the value of an attribute on a gurobi var. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be retrieved. - attr: str - The attribute to get. See gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) - - def get_linear_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def get_sos_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi sos constraint. - - Parameters - ---------- - con: pyomo.core.base.sos.SOSConstraintData - The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) - - def get_quadratic_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi quadratic constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def set_gurobi_param(self, param, val): - """ - Set a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to set. Options include any gurobi parameter. - Please see the Gurobi documentation for options. - val: any - The value to set the parameter to. See Gurobi documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_gurobi_param_info(self, param): - """ - Get information about a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to get info for. See Gurobi documentation for possible options. - - Returns - ------- - six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. - """ - return self._solver_model.getParamInfo(param) - - def _intermediate_callback(self): - def f(gurobi_model, where): - self._callback_func(self._model, self, where) - - return f - - def set_callback(self, func=None): - """ - Specify a callback for gurobi to use. - - Parameters - ---------- - func: function - The function to call. The function should have three arguments. The first will be the pyomo model being - solved. The second will be the GurobiPersistent instance. The third will be an enum member of - gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For - example, suppose we want to solve - - .. math:: - - min 2*x + y - - s.t. - - y >= (x-2)**2 - - 0 <= x <= 4 - - y >= 0 - - y integer - - as an MILP using extended cutting planes in callbacks. - - >>> from gurobipy import GRB # doctest:+SKIP - >>> import pyomo.environ as pyo - >>> from pyomo.core.expr.taylor_series import taylor_series_expansion - >>> from pyomo.contrib import appsi - >>> - >>> m = pyo.ConcreteModel() - >>> m.x = pyo.Var(bounds=(0, 4)) - >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) - >>> m.obj = pyo.Objective(expr=2*m.x + m.y) - >>> m.cons = pyo.ConstraintList() # for the cutting planes - >>> - >>> def _add_cut(xval): - ... # a function to generate the cut - ... m.x.value = xval - ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) - ... - >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x - >>> _c = _add_cut(4) # this is an arbitrary choice - >>> - >>> opt = appsi.solvers.Gurobi() - >>> opt.config.stream_solver = True - >>> opt.set_instance(m) # doctest:+SKIP - >>> opt.gurobi_options['PreCrush'] = 1 - >>> opt.gurobi_options['LazyConstraints'] = 1 - >>> - >>> def my_callback(cb_m, cb_opt, cb_where): - ... if cb_where == GRB.Callback.MIPSOL: - ... cb_opt.cbGetSolution(variables=[m.x, m.y]) - ... if m.y.value < (m.x.value - 2)**2 - 1e-6: - ... cb_opt.cbLazy(_add_cut(m.x.value)) - ... - >>> opt.set_callback(my_callback) - >>> res = opt.solve(m) # doctest:+SKIP - - """ - if func is not None: - self._callback_func = func - self._callback = self._intermediate_callback() - else: - self._callback = None - self._callback_func = None - - def cbCut(self, con): - """ - Add a cut within a callback. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The cut to add - """ - if not con.active: - raise ValueError('cbCut expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbCut expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbCut.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbGet(self, what): - return self._solver_model.cbGet(what) - - def cbGetNodeRel(self, variables): - """ - Parameters - ---------- - variables: Var or iterable of Var - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetNodeRel(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbGetSolution(self, variables): - """ - Parameters - ---------- - variables: iterable of vars - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetSolution(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbLazy(self, con): - """ - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The lazy constraint to add - """ - if not con.active: - raise ValueError('cbLazy expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbLazy expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbLazy.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbSetSolution(self, variables, solution): - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - self._solver_model.cbSetSolution(gurobi_vars, solution) - - def cbUseSolution(self): - return self._solver_model.cbUseSolution() - - def reset(self): - self._solver_model.reset() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py index 30ddd7eca4b..31d90770aca 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py @@ -13,7 +13,7 @@ import pyomo.common.unittest as unittest from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.results import TerminationCondition, SolutionStatus -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import GurobiDirectMINLP +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP from pyomo.core.base.constraint import Constraint from pyomo.environ import ( diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py index 14eab91f09c..eeae8a7d96e 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py @@ -14,7 +14,7 @@ from pyomo.core.expr import ProductExpression, SumExpression from pyomo.common.errors import InvalidValueError import pyomo.common.unittest as unittest -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import GurobiMINLPVisitor +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiMINLPVisitor from pyomo.contrib.solver.tests.solvers.gurobi_to_pyomo_expressions import ( grb_nl_to_pyo_expr, ) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py index f86ebd975c4..ab68aae046c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py @@ -40,7 +40,7 @@ ) from pyomo.gdp import Disjunction from pyomo.opt import WriterFactory -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import ( +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import ( GurobiDirectMINLP, GurobiMINLPVisitor, ) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 8703ae9edff..98df7236f25 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -11,7 +11,7 @@ import pyomo.common.unittest as unittest import pyomo.environ as pyo -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.core.expr.taylor_series import taylor_series_expansion @@ -471,11 +471,11 @@ def test_solution_number(self): res = opt.solve(m) num_solutions = opt.get_model_attr('SolCount') self.assertEqual(num_solutions, 3) - res.solution_loader.load_vars(solution_number=0) + res.solution_loader.load_vars(solution_id=0) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.431184939357673) - res.solution_loader.load_vars(solution_number=1) + res.solution_loader.load_vars(solution_id=1) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.584793218502477) - res.solution_loader.load_vars(solution_number=2) + res.solution_loader.load_vars(solution_id=2) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.592304628123309) def test_zero_time_limit(self): @@ -496,11 +496,9 @@ def test_zero_time_limit(self): self.assertIsNone(res.incumbent_objective) -class TestManualModel(unittest.TestCase): +class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.config.auto_updates.check_for_new_or_removed_params = False - opt.config.auto_updates.check_for_new_or_removed_vars = False opt.config.auto_updates.check_for_new_or_removed_constraints = False opt.config.auto_updates.update_parameters = False opt.config.auto_updates.update_vars = False @@ -586,13 +584,6 @@ def test_basics(self): self.assertEqual(opt.get_model_attr('NumConstrs'), 1) self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) - m.z = pyo.Var() - opt.add_variables([m.z]) - self.assertEqual(opt.get_model_attr('NumVars'), 3) - opt.remove_variables([m.z]) - del m.z - self.assertEqual(opt.get_model_attr('NumVars'), 2) - def test_update1(self): m = pyo.ConcreteModel() m.x = pyo.Var() @@ -603,16 +594,13 @@ def test_update1(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) opt.remove_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) opt.add_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) def test_update2(self): m = pyo.ConcreteModel() @@ -625,16 +613,13 @@ def test_update2(self): opt = self.opt opt.config.symbolic_solver_labels = True opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) opt.remove_constraints([m.c2]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumConstrs'), 0) opt.add_constraints([m.c2]) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) def test_update3(self): m = pyo.ConcreteModel() @@ -684,16 +669,13 @@ def test_update5(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) opt.remove_sos_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) + self.assertEqual(opt.get_model_attr('NumSOS'), 0) opt.add_sos_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) def test_update6(self): m = pyo.ConcreteModel() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 8aa452444ec..76c7cb4be87 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,10 +30,14 @@ NoDualsError, NoReducedCostsError, NoSolutionError, + NoFeasibleSolutionError, + NoOptimalSolutionError, +) +from pyomo.contrib.solver.solvers.gurobi import ( + GurobiDirect, + GurobiPersistent, + GurobiDirectMINLP, ) -from pyomo.contrib.solver.solvers.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi_direct_minlp import GurobiDirectMINLP -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.solvers.highs import Highs from pyomo.contrib.solver.solvers.ipopt import Ipopt from pyomo.contrib.solver.solvers.knitro.direct import KnitroDirectSolver @@ -1087,10 +1091,12 @@ def test_results_infeasible( m.obj = pyo.Objective(expr=m.y) m.c1 = pyo.Constraint(expr=m.y >= m.x) m.c2 = pyo.Constraint(expr=m.y <= m.x - 1) - with self.assertRaises(Exception): + with self.assertRaises(NoOptimalSolutionError): res = opt.solve(m) - opt.config.load_solutions = False opt.config.raise_exception_on_nonoptimal_result = False + with self.assertRaises(NoFeasibleSolutionError): + res = opt.solve(m) + opt.config.load_solutions = False res = opt.solve(m) self.assertNotEqual(res.solution_status, SolutionStatus.optimal) if isinstance(opt, Ipopt): @@ -1293,55 +1299,50 @@ def test_mutable_quadratic_objective_qp( def test_fixed_vars( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): - for treat_fixed_vars_as_params in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=treat_fixed_vars_as_params) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - m = pyo.ConcreteModel() - m.x = pyo.Var() - m.x.fix(0) - m.y = pyo.Var() - a1 = 1 - a2 = -1 - b1 = 1 - b2 = 2 - m.obj = pyo.Objective(expr=m.y) - m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) - m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.unfix() - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) - self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - m.x.fix(0) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.value = 2 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2) - self.assertAlmostEqual(m.y.value, 3) - m.x.value = 0 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.x.fix(0) + m.y = pyo.Var() + a1 = 1 + a2 = -1 + b1 = 1 + b2 = 2 + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) + m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.unfix() + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) + self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) + m.x.fix(0) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.value = 2 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 2) + self.assertAlmostEqual(m.y.value, 3) + m.x.value = 0 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) @parameterized.expand(input=_load_tests(all_solvers)) def test_fixed_vars_2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1385,8 +1386,6 @@ def test_fixed_vars_3( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1409,8 +1408,6 @@ def test_fixed_vars_4( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1548,9 +1545,7 @@ def test_add_and_remove_vars( opt.config.auto_updates.update_vars = False opt.config.auto_updates.update_constraints = False opt.config.auto_updates.update_named_expressions = False - opt.config.auto_updates.check_for_new_or_removed_params = False opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.check_for_new_or_removed_vars = False opt.config.load_solutions = False res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) @@ -1875,7 +1870,11 @@ def test_objective_changes( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 3) if opt.is_persistent(): - opt.config.auto_updates.check_for_new_objective = False + # hack until we get everything ported to the observer + try: + opt.config.auto_updates.check_for_new_or_removed_objectives = False + except: + opt.config.auto_updates.check_for_new_objective = False m.e.expr = 4 res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 4) @@ -1964,10 +1963,7 @@ def test_fixed_binaries( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 1) - if opt.is_persistent(): - opt: SolverBase = opt_class(treat_fixed_vars_as_params=False) - else: - opt = opt_class() + opt = opt_class() m.x.fix(0) res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 0) @@ -2121,33 +2117,30 @@ def test_bug_2(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) This test is for a bug where an objective containing a fixed variable does not get updated properly when the variable is unfixed. """ - for fixed_var_option in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=fixed_var_option) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - - m = pyo.ConcreteModel() - m.x = pyo.Var(bounds=(-10, 10)) - m.y = pyo.Var() - m.obj = pyo.Objective(expr=3 * m.y - m.x) - m.c = pyo.Constraint(expr=m.y >= m.x) - - m.x.fix(1) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, 2, 5) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False - m.x.unfix() - m.x.setlb(-9) - m.x.setub(9) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, -18, 5) + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(-10, 10)) + m.y = pyo.Var() + m.obj = pyo.Objective(expr=3 * m.y - m.x) + m.c = pyo.Constraint(expr=m.y >= m.x) + + m.x.fix(1) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, 2, 5) + + m.x.unfix() + m.x.setlb(-9) + m.x.setub(9) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, -18, 5) @parameterized.expand(input=_load_tests(nl_solvers)) def test_presolve_with_zero_coef( diff --git a/pyomo/contrib/solver/tests/unit/test_base.py b/pyomo/contrib/solver/tests/unit/test_base.py index 217b02b9999..08245d37f5e 100644 --- a/pyomo/contrib/solver/tests/unit/test_base.py +++ b/pyomo/contrib/solver/tests/unit/test_base.py @@ -74,15 +74,11 @@ def test_class_method_list(self): '_load_vars', 'add_block', 'add_constraints', - 'add_parameters', - 'add_variables', 'api_version', 'available', 'is_persistent', 'remove_block', 'remove_constraints', - 'remove_parameters', - 'remove_variables', 'set_instance', 'set_objective', 'solve', @@ -103,18 +99,10 @@ def test_init(self): self.assertEqual(instance.api_version(), SolverAPIVersion.V2) with self.assertRaises(NotImplementedError): self.assertEqual(instance.set_instance(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.add_variables(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.add_parameters(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.add_constraints(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.add_block(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.remove_variables(None), None) - with self.assertRaises(NotImplementedError): - self.assertEqual(instance.remove_parameters(None), None) with self.assertRaises(NotImplementedError): self.assertEqual(instance.remove_constraints(None), None) with self.assertRaises(NotImplementedError):