From 28b3fd96735839f05a7b3fd21fbaa15fbbb716ea Mon Sep 17 00:00:00 2001 From: Jeremy Sadler <53983960+jezsadler@users.noreply.github.com> Date: Tue, 19 Mar 2024 02:05:50 +0000 Subject: [PATCH 1/2] OmltVar wrapper class --- src/omlt/base.py | 319 +++++++++++++++++++ src/omlt/block.py | 6 +- src/omlt/formulation.py | 8 +- src/omlt/gbt/gbt_formulation.py | 5 +- src/omlt/linear_tree/lt_formulation.py | 5 +- src/omlt/neuralnet/activations/relu.py | 3 +- src/omlt/neuralnet/layers/full_space.py | 5 +- src/omlt/neuralnet/layers/partition_based.py | 12 +- src/omlt/neuralnet/nn_formulation.py | 9 +- tests/neuralnet/test_nn_formulation.py | 9 +- 10 files changed, 359 insertions(+), 22 deletions(-) create mode 100644 src/omlt/base.py diff --git a/src/omlt/base.py b/src/omlt/base.py new file mode 100644 index 00000000..223c76b5 --- /dev/null +++ b/src/omlt/base.py @@ -0,0 +1,319 @@ +""" +Abstraction layer of classes used by OMLT. Underneath these are +objects in a choice of modeling languages: Pyomo (default), +MathOptInterface, or Smoke (not yet implemented). + + +""" + +from abc import ABC, abstractmethod +import pyomo.environ as pyo + + +class OmltVar(ABC): + def __new__(cls, *indexes, **kwargs): + + if not indexes: + instance = OmltScalar.__new__(OmltScalar, **kwargs) + else: + instance = OmltIndexed.__new__(OmltIndexed, *indexes, **kwargs) + return instance + + +class OmltScalar(OmltVar): + def __new__(cls, *args, format="pyomo", **kwargs): + subclass_map = {subclass.format: subclass for subclass in cls.__subclasses__()} + if format not in subclass_map: + raise ValueError( + f"Variable format %s not recognized. Supported formats " + "are 'pyomo' or 'moi'.", + format, + ) + subclass = subclass_map[format] + instance = super(OmltVar, subclass).__new__(subclass) + + instance.__init__(*args, **kwargs) + return instance + + def __init__(self, *args, **kwargs): + pass + + @abstractmethod + def construct(self, data): + pass + + @abstractmethod + def fix(self, value, skip_validation): + pass + + @property + @abstractmethod + def bounds(self): + pass + + @bounds.setter + @abstractmethod + def bounds(self, val): + pass + + @property + @abstractmethod + def lb(self): + pass + + @lb.setter + @abstractmethod + def lb(self, val): + pass + + @property + @abstractmethod + def ub(self): + pass + + @ub.setter + @abstractmethod + def ub(self, val): + pass + + # @abstractmethod + # def __mul__(self, other): + # pass + + # @abstractmethod + # def __rmul__(self, other): + # pass + + +class OmltScalarPyomo(pyo.ScalarVar, OmltScalar): + format = "pyomo" + + def __init__(self, *args, **kwargs): + pyo.ScalarVar.__init__(self, *args, **kwargs) + + def construct(self, data): + super().construct(data) + + def fix(self, value=None, skip_validation=False): + self.fixed = True + if value is None: + super().fix(skip_validation) + else: + super().fix(value, skip_validation) + + @property + def bounds(self): + return super().bounds + + @bounds.setter + def bounds(self, val): + super().bounds = val + + @property + def ub(self): + return super().ub + + @ub.setter + def ub(self, val): + super().ub = val + + @property + def lb(self): + return super().__get__(self.lb) + + @lb.setter + def lb(self, val): + super().__setattr__(self.lb, val) + + def __lt__(self, other): + return pyo.NumericValue.__lt__(self, other) + + def __gt__(self, other): + return pyo.NumericValue.__gt__(self, other) + + def __le__(self, other): + return pyo.NumericValue.__le__(self, other) + + def __ge__(self, other): + return pyo.NumericValue.__ge__(self, other) + + def __eq__(self, other): + return pyo.NumericValue.__eq__(self, other) + + def __add__(self, other): + return pyo.NumericValue.__add__(self, other) + + def __sub__(self, other): + return pyo.NumericValue.__sub__(self, other) + + # def __mul__(self,other): + # return pyo.NumericValue.__mul__(self,other) + + def __div__(self, other): + return pyo.NumericValue.__div__(self, other) + + def __truediv__(self, other): + return pyo.NumericValue.__truediv__(self, other) + + def __pow__(self, other): + return pyo.NumericValue.__pow__(self, other) + + def __radd__(self, other): + return pyo.NumericValue.__radd__(self, other) + + def __rsub__(self, other): + return pyo.NumericValue.__rsub__(self, other) + + # def __rmul__(self,other): + # return self._ComponentDataClass.__rmul__(self,other) + + def __rdiv__(self, other): + return pyo.NumericValue.__rdiv__(self, other) + + def __rtruediv__(self, other): + return pyo.NumericValue.__rtruediv__(self, other) + + def __rpow__(self, other): + return pyo.NumericValue.__rpow__(self, other) + + def __iadd__(self, other): + return pyo.NumericValue.__iadd__(self, other) + + def __isub__(self, other): + return pyo.NumericValue.__isub__(self, other) + + def __imul__(self, other): + return pyo.NumericValue.__imul__(self, other) + + def __idiv__(self, other): + return pyo.NumericValue.__idiv__(self, other) + + def __itruediv__(self, other): + return pyo.NumericValue.__itruediv__(self, other) + + def __ipow__(self, other): + return pyo.NumericValue.__ipow__(self, other) + + def __neg__(self): + return pyo.NumericValue.__neg__(self) + + def __pos__(self): + return pyo.NumericValue.__pos__(self) + + def __abs__(self): + return pyo.NumericValue.__abs__(self) + + +""" +Future formats to implement. +""" + + +class OmltScalarMOI(OmltScalar): + format = "moi" + + +class OmltScalarSmoke(OmltScalar): + format = "smoke" + + def __init__(self, *args, **kwargs): + raise ValueError( + "Storing variables in Smoke format is not currently implemented." + ) + + +class OmltScalarGurobi(OmltScalar): + format = "gurobi" + + def __init__(self, *args, **kwargs): + raise ValueError( + "Storing variables in Gurobi format is not currently implemented." + ) + + +class OmltIndexed(OmltVar): + def __new__(cls, *indexes, format="pyomo", **kwargs): + subclass_map = {subclass.format: subclass for subclass in cls.__subclasses__()} + if format not in subclass_map: + raise ValueError( + f"Variable format %s not recognized. Supported formats are 'pyomo'" + " or 'moi'.", + format, + ) + subclass = subclass_map[format] + instance = super(OmltVar, subclass).__new__(subclass) + instance.__init__(*indexes, **kwargs) + return instance + + @abstractmethod + def fix(self, value=None, skip_validation=False): + pass + + @abstractmethod + def setub(self, value): + pass + + @abstractmethod + def setlb(self, value): + pass + + +class OmltIndexedPyomo(pyo.Var, OmltIndexed): + format = "pyomo" + + def __init__(self, *indexes, **kwargs): + super().__init__(*indexes, **kwargs) + + def fix(self, value=None, skip_validation=False): + self.fixed = True + if value is None: + for vardata in self.values(): + vardata.fix(skip_validation) + else: + for vardata in self.values(): + vardata.fix(value, skip_validation) + + def setub(self, value): + for vardata in self.values(): + vardata.ub = value + + def setlb(self, value): + for vardata in self.values(): + vardata.lb = value + + +""" +Future formats to implement. +""" + + +class OmltIndexedMOI(OmltIndexed): + format = "moi" + + +class OmltIndexedSmoke(OmltIndexed): + format = "smoke" + + def __init__(self, *args, **kwargs): + raise ValueError( + "Storing variables in Smoke format is not currently implemented." + ) + + +class OmltIndexedGurobi(OmltIndexed): + format = "gurobi" + + def __init__(self, *args, **kwargs): + raise ValueError( + "Storing variables in Gurobi format is not currently implemented." + ) + + +class OmltSet: + def __init__(self): + pass + + +class OmltExpression: + def __init__(self): + pass diff --git a/src/omlt/block.py b/src/omlt/block.py index a6c7bbf2..04932e41 100644 --- a/src/omlt/block.py +++ b/src/omlt/block.py @@ -25,6 +25,8 @@ class is used in combination with a formulation object to construct the import warnings +from omlt.base import OmltVar + import pyomo.environ as pyo from pyomo.core.base.block import _BlockData, declare_custom_block @@ -59,9 +61,9 @@ def _setup_inputs_outputs(self, *, input_indexes, output_indexes): ) self.inputs_set = pyo.Set(initialize=input_indexes) - self.inputs = pyo.Var(self.inputs_set, initialize=0) + self.inputs = OmltVar(self.inputs_set, initialize=0) self.outputs_set = pyo.Set(initialize=output_indexes) - self.outputs = pyo.Var(self.outputs_set, initialize=0) + self.outputs = OmltVar(self.outputs_set, initialize=0) def build_formulation(self, formulation): """ diff --git a/src/omlt/formulation.py b/src/omlt/formulation.py index fd83ae86..0d054ca9 100644 --- a/src/omlt/formulation.py +++ b/src/omlt/formulation.py @@ -2,7 +2,7 @@ import weakref import pyomo.environ as pyo - +from omlt.base import OmltVar class _PyomoFormulationInterface(abc.ABC): """ @@ -82,11 +82,11 @@ def _setup_scaled_inputs_outputs(block, scaler=None, scaled_input_bounds=None): k: (float(scaled_input_bounds[k][0]), float(scaled_input_bounds[k][1])) for k in block.inputs_set } - block.scaled_inputs = pyo.Var(block.inputs_set, initialize=0, bounds=bnds) + block.scaled_inputs = OmltVar(block.inputs_set, initialize=0, bounds=bnds) else: - block.scaled_inputs = pyo.Var(block.inputs_set, initialize=0) + block.scaled_inputs = OmltVar(block.inputs_set, initialize=0) - block.scaled_outputs = pyo.Var(block.outputs_set, initialize=0) + block.scaled_outputs = OmltVar(block.outputs_set, initialize=0) if scaled_input_bounds is not None and scaler is None: # set the bounds on the inputs to be the same as the scaled inputs diff --git a/src/omlt/gbt/gbt_formulation.py b/src/omlt/gbt/gbt_formulation.py index f2d01296..58c133fd 100644 --- a/src/omlt/gbt/gbt_formulation.py +++ b/src/omlt/gbt/gbt_formulation.py @@ -3,6 +3,7 @@ import numpy as np import pyomo.environ as pe +from omlt.base import OmltVar from omlt.formulation import _PyomoFormulation, _setup_scaled_inputs_outputs from omlt.gbt.model import GradientBoostedTreeModel @@ -148,7 +149,7 @@ def add_formulation_to_block(block, model_definition, input_vars, output_vars): var = input_vars[var_idx] continuous_vars[var_idx] = var - block.z_l = pe.Var( + block.z_l = OmltVar( list(zip(nodes_tree_ids[nodes_leaf_mask], nodes_node_ids[nodes_leaf_mask])), bounds=(0, None), domain=pe.Reals, @@ -167,7 +168,7 @@ def add_formulation_to_block(block, model_definition, input_vars, output_vars): for f in continuous_vars.keys() for bi, _ in enumerate(branch_value_by_feature_id[f]) ] - block.y = pe.Var(y_index, domain=pe.Binary) + block.y = OmltVar(y_index, domain=pe.Binary) @block.Constraint(tree_ids) def single_leaf(b, tree_id): diff --git a/src/omlt/linear_tree/lt_formulation.py b/src/omlt/linear_tree/lt_formulation.py index 4f83e7f3..76312278 100644 --- a/src/omlt/linear_tree/lt_formulation.py +++ b/src/omlt/linear_tree/lt_formulation.py @@ -2,6 +2,7 @@ import pyomo.environ as pe from pyomo.gdp import Disjunct +from omlt.base import OmltVar from omlt.formulation import _PyomoFormulation, _setup_scaled_inputs_outputs @@ -250,7 +251,7 @@ def _add_gdp_formulation_to_block( block.scaled_outputs.setub(output_bounds[1]) block.scaled_outputs.setlb(output_bounds[0]) - block.intermediate_output = pe.Var( + block.intermediate_output = OmltVar( tree_ids, bounds=(output_bounds[0], output_bounds[1]) ) @@ -329,7 +330,7 @@ def _add_hybrid_formulation_to_block(block, model_definition, input_vars, output # Create the intermeditate variables. z is binary that indicates which leaf # in tree t is returned. intermediate_output is the output of tree t and # the total output of the model is the sum of the intermediate_output vars - block.z = pe.Var(t_l, within=pe.Binary) + block.z = OmltVar(t_l, within=pe.Binary) block.intermediate_output = pe.Var(tree_ids) @block.Constraint(features, tree_ids) diff --git a/src/omlt/neuralnet/activations/relu.py b/src/omlt/neuralnet/activations/relu.py index 427be19a..8ac42aa0 100644 --- a/src/omlt/neuralnet/activations/relu.py +++ b/src/omlt/neuralnet/activations/relu.py @@ -1,6 +1,7 @@ import pyomo.environ as pyo import pyomo.mpec as mpec +from omlt.base import OmltVar def bigm_relu_activation_constraint(net_block, net, layer_block, layer): r""" @@ -38,7 +39,7 @@ def bigm_relu_activation_constraint(net_block, net, layer_block, layer): The lower bound of :math:`y` is :math:`\max(0,l)`, and the upper bound of :math:`y` is :math:`\max(0,u)`. """ - layer_block.q_relu = pyo.Var(layer.output_indexes, within=pyo.Binary) + layer_block.q_relu = OmltVar(layer.output_indexes, within=pyo.Binary) layer_block._z_lower_bound_relu = pyo.Constraint(layer.output_indexes) layer_block._z_lower_bound_zhat_relu = pyo.Constraint(layer.output_indexes) diff --git a/src/omlt/neuralnet/layers/full_space.py b/src/omlt/neuralnet/layers/full_space.py index 8970bc69..c8ac1bf5 100644 --- a/src/omlt/neuralnet/layers/full_space.py +++ b/src/omlt/neuralnet/layers/full_space.py @@ -2,6 +2,7 @@ import pyomo.environ as pyo from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr +from omlt.base import OmltVar from omlt.neuralnet.activations import NON_INCREASING_ACTIVATIONS from omlt.neuralnet.layer import ConvLayer2D, IndexMapper, PoolingLayer2D @@ -86,7 +87,7 @@ def full_space_gnn_layer(net_block, net, layer_block, layer): input_layer, input_layer_block = _input_layer_and_block(net_block, net, layer) - input_layer_block.zbar = pyo.Var( + input_layer_block.zbar = OmltVar( pyo.Set(initialize=layer.input_indexes), pyo.Set(initialize=range(layer.N)), initialize=0, @@ -276,7 +277,7 @@ def full_space_maxpool2d_layer(net_block, net, layer_block, layer): for kernel_index, _ in layer.kernel_index_with_input_indexes(0, 0, 0) ) ) - layer_block.q_maxpool = pyo.Var( + layer_block.q_maxpool = OmltVar( layer.output_indexes, layer_block._kernel_indexes, within=pyo.Binary ) layer_block._q_sum_maxpool = pyo.Constraint(layer.output_indexes) diff --git a/src/omlt/neuralnet/layers/partition_based.py b/src/omlt/neuralnet/layers/partition_based.py index f29cadd2..b67c796b 100644 --- a/src/omlt/neuralnet/layers/partition_based.py +++ b/src/omlt/neuralnet/layers/partition_based.py @@ -2,6 +2,8 @@ import pyomo.environ as pyo from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr +from omlt.base import OmltVar + def default_partition_split_func(w, n): r""" @@ -81,8 +83,8 @@ def output_node_block(b, *output_index): splits = split_func(weights) num_splits = len(splits) - b.sig = pyo.Var(domain=pyo.Binary) - b.z2 = pyo.Var(range(num_splits)) + b.sig = OmltVar(domain=pyo.Binary) + b.z2 = OmltVar(range(num_splits)) mapper = layer.input_index_mapper @@ -109,6 +111,12 @@ def output_node_block(b, *output_index): expr += prev_layer_block.z[input_index] * w lb, ub = compute_bounds_on_expr(expr) + print("inside function") + print(expr) + print(w) + print(prev_layer_block.z[input_index]) + print(prev_layer_block.z[input_index].lb) + print(prev_layer_block.z[input_index].ub) if lb is None: raise ValueError("Expression is unbounded below.") if ub is None: diff --git a/src/omlt/neuralnet/nn_formulation.py b/src/omlt/neuralnet/nn_formulation.py index 042b14fe..1ff4d4fb 100644 --- a/src/omlt/neuralnet/nn_formulation.py +++ b/src/omlt/neuralnet/nn_formulation.py @@ -1,6 +1,7 @@ import numpy as np import pyomo.environ as pyo +from omlt.base import OmltVar from omlt.formulation import _PyomoFormulation, _setup_scaled_inputs_outputs from omlt.neuralnet.activations import ( ACTIVATION_FUNCTION_MAP as _DEFAULT_ACTIVATION_FUNCTIONS, @@ -162,7 +163,7 @@ def _build_neural_network_formulation( @block.Block(block.layers) def layer(b, layer_id): net_layer = net.layer(layer_id) - b.z = pyo.Var(net_layer.output_indexes, initialize=0) + b.z = OmltVar(net_layer.output_indexes, initialize=0) if isinstance(net_layer, InputLayer): for index in net_layer.output_indexes: input_var = block.scaled_inputs[index] @@ -171,7 +172,7 @@ def layer(b, layer_id): z_var.setub(input_var.ub) else: # add zhat only to non input layers - b.zhat = pyo.Var(net_layer.output_indexes, initialize=0) + b.zhat = OmltVar(net_layer.output_indexes, initialize=0) return b @@ -488,7 +489,7 @@ def _build_formulation(self): @block.Block(block.layers) def layer(b, layer_id): net_layer = net.layer(layer_id) - b.z = pyo.Var(net_layer.output_indexes, initialize=0) + b.z = OmltVar(net_layer.output_indexes, initialize=0) if isinstance(net_layer, InputLayer): for index in net_layer.output_indexes: input_var = block.scaled_inputs[index] @@ -497,7 +498,7 @@ def layer(b, layer_id): z_var.setub(input_var.ub) else: # add zhat only to non input layers - b.zhat = pyo.Var(net_layer.output_indexes, initialize=0) + b.zhat = OmltVar(net_layer.output_indexes, initialize=0) return b diff --git a/tests/neuralnet/test_nn_formulation.py b/tests/neuralnet/test_nn_formulation.py index ad3b2b0f..2e1d6cb5 100644 --- a/tests/neuralnet/test_nn_formulation.py +++ b/tests/neuralnet/test_nn_formulation.py @@ -537,18 +537,19 @@ def test_partition_based_unbounded_below(): m.neural_net_block = OmltBlock() net, y = two_node_network(None, -2.0) test_layer = list(net.layers)[2] + test_layer_id = id(test_layer) prev_layer_id = id(list(net.layers)[1]) formulation = ReluPartitionFormulation(net) m.neural_net_block.build_formulation(formulation) prev_layer_block = m.neural_net_block.layer[prev_layer_id] prev_layer_block.z.setlb(-interval.inf) - + split_func = lambda w: default_partition_split_func(w, 2) with pytest.raises(ValueError) as excinfo: partition_based_dense_relu_layer( - m.neural_net_block, net, m.neural_net_block, test_layer, split_func + m.neural_net_block, net, m.neural_net_block.layer[test_layer_id], test_layer, split_func ) expected_msg = "Expression is unbounded below." assert str(excinfo.value) == expected_msg @@ -559,6 +560,7 @@ def test_partition_based_unbounded_above(): m.neural_net_block = OmltBlock() net, y = two_node_network(None, -2.0) test_layer = list(net.layers)[2] + test_layer_id = id(test_layer) prev_layer_id = id(list(net.layers)[1]) formulation = ReluPartitionFormulation(net) @@ -566,11 +568,12 @@ def test_partition_based_unbounded_above(): prev_layer_block = m.neural_net_block.layer[prev_layer_id] prev_layer_block.z.setub(interval.inf) + split_func = lambda w: default_partition_split_func(w, 2) with pytest.raises(ValueError) as excinfo: partition_based_dense_relu_layer( - m.neural_net_block, net, m.neural_net_block, test_layer, split_func + m.neural_net_block, net, m.neural_net_block.layer[test_layer_id], test_layer, split_func ) expected_msg = "Expression is unbounded above." assert str(excinfo.value) == expected_msg From 45ac09582b658d690c3c0f252527b39504527879 Mon Sep 17 00:00:00 2001 From: Jeremy Sadler <53983960+jezsadler@users.noreply.github.com> Date: Tue, 19 Mar 2024 02:05:50 +0000 Subject: [PATCH 2/2] OmltVar wrapper class --- src/omlt/neuralnet/layers/partition_based.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/omlt/neuralnet/layers/partition_based.py b/src/omlt/neuralnet/layers/partition_based.py index b67c796b..5f99e706 100644 --- a/src/omlt/neuralnet/layers/partition_based.py +++ b/src/omlt/neuralnet/layers/partition_based.py @@ -111,12 +111,7 @@ def output_node_block(b, *output_index): expr += prev_layer_block.z[input_index] * w lb, ub = compute_bounds_on_expr(expr) - print("inside function") - print(expr) - print(w) - print(prev_layer_block.z[input_index]) - print(prev_layer_block.z[input_index].lb) - print(prev_layer_block.z[input_index].ub) + if lb is None: raise ValueError("Expression is unbounded below.") if ub is None: