From 4d47410b59938f36f9d9577cc8ff29877a2b1ac4 Mon Sep 17 00:00:00 2001 From: Chelsea Date: Thu, 11 Nov 2021 18:06:43 -0800 Subject: [PATCH 1/6] Adding constraints so that binary variables reflect correct activation status --- src/optimization/utils/constraints.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/optimization/utils/constraints.jl b/src/optimization/utils/constraints.jl index e1846a36..877c54c5 100644 --- a/src/optimization/utils/constraints.jl +++ b/src/optimization/utils/constraints.jl @@ -86,6 +86,7 @@ end # For an Id Layer, any encoding type defaults to this: function encode_layer!(::AbstractLinearProgram, model::Model, layer::Layer{Id}, ẑᵢ, zᵢ, args...) @constraint(model, zᵢ .== ẑᵢ) + # how do we access δᵢⱼ variable so that it can be properly constrained to 1? nothing end @@ -105,6 +106,7 @@ end function encode_ij(LP, model, i, j) + # where is this function used? Needs documentation. L = model[:network].layers[i] params = model_params(LP, model, i) if L.activation isa Id @@ -124,6 +126,7 @@ function encode_relu(::SlackLP, model, ẑᵢⱼ, zᵢⱼ, δᵢⱼ, sᵢⱼ) if δᵢⱼ @constraint(model, zᵢⱼ == ẑᵢⱼ + sᵢⱼ) @constraint(model, ẑᵢⱼ + sᵢⱼ >= 0.0) + # how to encode necessary constraint on δᵢⱼ here? else @constraint(model, zᵢⱼ == sᵢⱼ) @constraint(model, ẑᵢⱼ <= sᵢⱼ) @@ -133,8 +136,10 @@ end function encode_relu(::BoundedMixedIntegerLP, model, ẑᵢⱼ, zᵢⱼ, δᵢⱼ, l̂ᵢⱼ, ûᵢⱼ) if l̂ᵢⱼ >= 0.0 @constraint(model, zᵢⱼ == ẑᵢⱼ) + @constraint(model, δᵢⱼ == 1) elseif ûᵢⱼ <= 0.0 @constraint(model, zᵢⱼ == 0.0) + @constraint(model, δᵢⱼ == 0) else @constraints(model, begin zᵢⱼ >= 0.0 @@ -152,8 +157,10 @@ end function encode_relu(::TriangularRelaxedLP, model, ẑᵢⱼ, zᵢⱼ, l̂ᵢⱼ, ûᵢⱼ) if l̂ᵢⱼ > 0.0 @constraint(model, zᵢⱼ == ẑᵢⱼ) + @constraint(model, δᵢⱼ == 1) elseif ûᵢⱼ < 0.0 @constraint(model, zᵢⱼ == 0.0) + @constraint(model, δᵢⱼ == 0) else @constraints(model, begin zᵢⱼ >= 0.0 @@ -164,16 +171,18 @@ function encode_relu(::TriangularRelaxedLP, model, ẑᵢⱼ, zᵢⱼ, l̂ᵢⱼ end function encode_relu(::LinearRelaxedLP, model, ẑᵢⱼ, zᵢⱼ, δᵢⱼ) - @constraint(model, zᵢⱼ == (δᵢⱼ ? ẑᵢⱼ : 0.0)) + @constraint(model, zᵢⱼ == (δᵢⱼ ? ẑᵢⱼ : 0.0)) # what does this mean? how can you use ? in a constraint? end function encode_relu(::StandardLP, model, ẑᵢⱼ, zᵢⱼ, δᵢⱼ) if δᵢⱼ @constraint(model, ẑᵢⱼ >= 0.0) @constraint(model, zᵢⱼ == ẑᵢⱼ) + @constraint(model, δᵢⱼ == 1) else @constraint(model, ẑᵢⱼ <= 0.0) @constraint(model, zᵢⱼ == 0.0) + @constraint(model, δᵢⱼ == 0) end end From a8e731d9ad1e90e3987f41a95c8343ee337487f3 Mon Sep 17 00:00:00 2001 From: Chelsea Date: Thu, 11 Nov 2021 23:34:07 -0800 Subject: [PATCH 2/6] adding fix for ID layers and binary variables --- src/optimization/utils/constraints.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/optimization/utils/constraints.jl b/src/optimization/utils/constraints.jl index 877c54c5..0fb8f812 100644 --- a/src/optimization/utils/constraints.jl +++ b/src/optimization/utils/constraints.jl @@ -86,7 +86,6 @@ end # For an Id Layer, any encoding type defaults to this: function encode_layer!(::AbstractLinearProgram, model::Model, layer::Layer{Id}, ẑᵢ, zᵢ, args...) @constraint(model, zᵢ .== ẑᵢ) - # how do we access δᵢⱼ variable so that it can be properly constrained to 1? nothing end @@ -104,6 +103,13 @@ function encode_layer!(SLP::SlackLP, model::Model, layer::Layer{Id}, ẑᵢ, z return nothing end +# need to fix δᵢⱼ for BoundedMixedIntegerLP and possibly other types +function encode_layer!(::BoundedMixedIntegerLP, model::Model, layer::Layer{Id}, ẑᵢ, zᵢ, δᵢⱼ, args...) + println("Using new case!") + @constraint(model, zᵢ .== ẑᵢ) + @constraint(model, δᵢⱼ == 1) + return nothing +end function encode_ij(LP, model, i, j) # where is this function used? Needs documentation. From c0f5465e1af9f4b5114e5b8c95dbd54a4f667526 Mon Sep 17 00:00:00 2001 From: Chelsea Date: Thu, 11 Nov 2021 23:44:05 -0800 Subject: [PATCH 3/6] bug fix --- src/optimization/utils/constraints.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/optimization/utils/constraints.jl b/src/optimization/utils/constraints.jl index 0fb8f812..e3eb12e8 100644 --- a/src/optimization/utils/constraints.jl +++ b/src/optimization/utils/constraints.jl @@ -89,6 +89,7 @@ function encode_layer!(::AbstractLinearProgram, model::Model, layer::Layer{Id}, nothing end +# All ReLU layers pass through this function encode_layer!(LP::AbstractLinearProgram, model::Model, layer::Layer{ReLU}, args...) encode_relu.(LP, model, args...) nothing @@ -104,10 +105,10 @@ function encode_layer!(SLP::SlackLP, model::Model, layer::Layer{Id}, ẑᵢ, z end # need to fix δᵢⱼ for BoundedMixedIntegerLP and possibly other types -function encode_layer!(::BoundedMixedIntegerLP, model::Model, layer::Layer{Id}, ẑᵢ, zᵢ, δᵢⱼ, args...) - println("Using new case!") +function encode_layer!(::BoundedMixedIntegerLP, model::Model, layer::Layer{Id}, ẑᵢ, zᵢ, δᵢ, args...) + println("Using new case! δᵢⱼ = $(δᵢⱼ)") @constraint(model, zᵢ .== ẑᵢ) - @constraint(model, δᵢⱼ == 1) + @constraint(model, δᵢ .== 1) return nothing end From 24ae4cfb979ce0893c41158d8a0b1525a78fdb72 Mon Sep 17 00:00:00 2001 From: Chelsea Date: Thu, 11 Nov 2021 23:47:31 -0800 Subject: [PATCH 4/6] bug fix --- src/optimization/utils/constraints.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimization/utils/constraints.jl b/src/optimization/utils/constraints.jl index e3eb12e8..c63f4c6f 100644 --- a/src/optimization/utils/constraints.jl +++ b/src/optimization/utils/constraints.jl @@ -106,7 +106,7 @@ end # need to fix δᵢⱼ for BoundedMixedIntegerLP and possibly other types function encode_layer!(::BoundedMixedIntegerLP, model::Model, layer::Layer{Id}, ẑᵢ, zᵢ, δᵢ, args...) - println("Using new case! δᵢⱼ = $(δᵢⱼ)") + println("Using new case! δᵢ = $(δᵢ)") @constraint(model, zᵢ .== ẑᵢ) @constraint(model, δᵢ .== 1) return nothing From 81fa4e3e3d556723121f9e8f323f1c41797d7a0e Mon Sep 17 00:00:00 2001 From: Chelsea Date: Fri, 12 Nov 2021 00:22:18 -0800 Subject: [PATCH 5/6] deleting debug statements --- src/optimization/utils/constraints.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/optimization/utils/constraints.jl b/src/optimization/utils/constraints.jl index c63f4c6f..88d62532 100644 --- a/src/optimization/utils/constraints.jl +++ b/src/optimization/utils/constraints.jl @@ -106,7 +106,6 @@ end # need to fix δᵢⱼ for BoundedMixedIntegerLP and possibly other types function encode_layer!(::BoundedMixedIntegerLP, model::Model, layer::Layer{Id}, ẑᵢ, zᵢ, δᵢ, args...) - println("Using new case! δᵢ = $(δᵢ)") @constraint(model, zᵢ .== ẑᵢ) @constraint(model, δᵢ .== 1) return nothing From 92d712816e554b7715a796f4b82d40dec1bdfeb0 Mon Sep 17 00:00:00 2001 From: Chelsea Date: Fri, 12 Nov 2021 11:57:11 -0800 Subject: [PATCH 6/6] cleaning up delta_ij changes --- src/optimization/utils/constraints.jl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/optimization/utils/constraints.jl b/src/optimization/utils/constraints.jl index 88d62532..bab6d4a4 100644 --- a/src/optimization/utils/constraints.jl +++ b/src/optimization/utils/constraints.jl @@ -132,7 +132,6 @@ function encode_relu(::SlackLP, model, ẑᵢⱼ, zᵢⱼ, δᵢⱼ, sᵢⱼ) if δᵢⱼ @constraint(model, zᵢⱼ == ẑᵢⱼ + sᵢⱼ) @constraint(model, ẑᵢⱼ + sᵢⱼ >= 0.0) - # how to encode necessary constraint on δᵢⱼ here? else @constraint(model, zᵢⱼ == sᵢⱼ) @constraint(model, ẑᵢⱼ <= sᵢⱼ) @@ -163,10 +162,8 @@ end function encode_relu(::TriangularRelaxedLP, model, ẑᵢⱼ, zᵢⱼ, l̂ᵢⱼ, ûᵢⱼ) if l̂ᵢⱼ > 0.0 @constraint(model, zᵢⱼ == ẑᵢⱼ) - @constraint(model, δᵢⱼ == 1) elseif ûᵢⱼ < 0.0 @constraint(model, zᵢⱼ == 0.0) - @constraint(model, δᵢⱼ == 0) else @constraints(model, begin zᵢⱼ >= 0.0 @@ -177,18 +174,16 @@ function encode_relu(::TriangularRelaxedLP, model, ẑᵢⱼ, zᵢⱼ, l̂ᵢⱼ end function encode_relu(::LinearRelaxedLP, model, ẑᵢⱼ, zᵢⱼ, δᵢⱼ) - @constraint(model, zᵢⱼ == (δᵢⱼ ? ẑᵢⱼ : 0.0)) # what does this mean? how can you use ? in a constraint? + @constraint(model, zᵢⱼ == (δᵢⱼ ? ẑᵢⱼ : 0.0)) # in LinearRelaxedLP δᵢⱼ is a constant not a variable end function encode_relu(::StandardLP, model, ẑᵢⱼ, zᵢⱼ, δᵢⱼ) - if δᵢⱼ + if δᵢⱼ # in StandardLP δᵢⱼ is a constant, not a variable @constraint(model, ẑᵢⱼ >= 0.0) @constraint(model, zᵢⱼ == ẑᵢⱼ) - @constraint(model, δᵢⱼ == 1) else @constraint(model, ẑᵢⱼ <= 0.0) @constraint(model, zᵢⱼ == 0.0) - @constraint(model, δᵢⱼ == 0) end end