From 48a6b5ca22052ab4236adc10f927b909ed23babb Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 13:09:50 -0600 Subject: [PATCH 01/10] Update for single/double precision, add tests --- .../engine/nn/layers/AveragePooling2DLayer.m | 16 +++++++-- code/nnv/engine/nn/layers/Conv2DLayer.m | 32 ++++++++++++----- code/nnv/engine/nn/layers/ReluLayer.m | 2 +- .../test_layers_averagePooling.m | 35 ++++++++++++------- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m index af9b374c2b..bbfb10c271 100644 --- a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m +++ b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m @@ -228,8 +228,20 @@ function set_padding(obj, padding) % author: Dung Tran % date: 12/10/2018 % update: 7/26/2019 - - y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); + + outputType = "double"; % default + + if isa(input, "single") + input = double(input); % vn_nnconv requires inputs to be of double precision + outputType = "single"; + end + + y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); + + % Return output with correct precision + if strcmp(outputType, "single") + y = single(y); + end end diff --git a/code/nnv/engine/nn/layers/Conv2DLayer.m b/code/nnv/engine/nn/layers/Conv2DLayer.m index 8f2b82d2d4..29987ca5bb 100644 --- a/code/nnv/engine/nn/layers/Conv2DLayer.m +++ b/code/nnv/engine/nn/layers/Conv2DLayer.m @@ -435,11 +435,27 @@ function set_name(obj, name) function y = evaluate(obj, input) % @input: 3-dimensional array, for example, input(:, :, :) % @y: high-dimensional array (output volume), depth of output = number of filters + + outputType = "double"; % default - % author: Dung Tran - % date: 7/18/2019 - - y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + if isa(input, "single") + input = double(input); % vn_nnconv requires inputs to be of double precision + outputType = "single"; + end + + % Compute evaluation + if isa(obj.Weights, "single") + y = vl_nnconv(input, double(obj.Weights), double(obj.Bias), 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + elseif isa(obj.Weights, "double") + y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + else + error("Input and layer parameters must be single or double precision.") + end + + % Return output with correct precision + if strcmp(outputType, "single") + y = single(y); + end end @@ -501,8 +517,8 @@ function set_name(obj, name) end % compute output sets - c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - V = vl_nnconv(input.V(:,:,:,2:input.numPred + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + c = obj.evaluate(input.V(:,:,:,1)); + V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); Y = cat(4, c, V); S = ImageStar(Y, input.C, input.d, input.pred_lb, input.pred_ub); @@ -522,8 +538,8 @@ function set_name(obj, name) end % compute output sets - c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - V = vl_nnconv(input.V(:,:,:,2:input.numPreds + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + c = obj.evaluate(input.V(:,:,:,1)); + V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); Y = cat(4, c, V); Z = ImageZono(Y); diff --git a/code/nnv/engine/nn/layers/ReluLayer.m b/code/nnv/engine/nn/layers/ReluLayer.m index f7f060102f..13ff7fca11 100644 --- a/code/nnv/engine/nn/layers/ReluLayer.m +++ b/code/nnv/engine/nn/layers/ReluLayer.m @@ -105,7 +105,7 @@ w = in_image.width; c = in_image.numChannel; - Y = PosLin.reach(in_image.toStar, method, [], relaxFactor); % reachable set computation with ReLU + Y = PosLin.reach(in_image.toStar, method, [], relaxFactor, dis_opt, lp_solver); % reachable set computation with ReLU n = length(Y); images(n) = ImageStar; % transform back to ImageStar diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 71e8b89895..9e5f5a390f 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -1,18 +1,18 @@ %% test 1: AveragePooling2DLayer Compute averageMap -I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input -L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); -averageMap = L.compute_averageMap(I); - -checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; -assert(isequal(checker, averageMap)) - - -%% test 2: AveragePooling2DLayer constructor -L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); -L2 = AveragePooling2DLayer(); -L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); +% I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input +% L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); +% averageMap = L.compute_averageMap(I); +% +% checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; +% assert(isequal(checker, averageMap)) +% +% +% %% test 2: AveragePooling2DLayer constructor +% L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); +% L2 = AveragePooling2DLayer(); +% L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); %% test 3: AveragePooling2DLayer evaluation @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(inputVol); +y = L.evaluate(single(inputVol)); for i=1:3 for j=1:2 @@ -39,6 +39,15 @@ % display(inputVol(:,:,i)); end +%% test 3b: AveragePooling2DLayer evaluation (single precision) +% original input volume: color image with 3 channels +inputVol(:, :, 1) = [0 0 2 0 0; 1 2 0 2 0; 0 0 2 2 0; 0 2 2 2 2; 2 2 2 1 1]; % channel 1 input matrix +inputVol(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix +inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix + +L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); +y = L.evaluate(single(inputVol)); + %% test 4: AveragePooling2DLayer get zero padding input From 8e2e5691e017bcebd12c28a434ac6489a7e5362a Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 15:49:27 -0600 Subject: [PATCH 02/10] Fix test --- .../layers/AveragePooling2DLayer/test_layers_averagePooling.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 9e5f5a390f..08fb53f976 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(single(inputVol)); +y = L.evaluate(inputVol); for i=1:3 for j=1:2 From eb75ea13fd7c3e0719611989a89029d587f24dd1 Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 18:57:55 -0600 Subject: [PATCH 03/10] Revert "Fix test" This reverts commit 8e2e5691e017bcebd12c28a434ac6489a7e5362a. --- .../layers/AveragePooling2DLayer/test_layers_averagePooling.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 08fb53f976..9e5f5a390f 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(inputVol); +y = L.evaluate(single(inputVol)); for i=1:3 for j=1:2 From 79f5808176ae10929b79a02c614ec0b8a09869db Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 18:58:00 -0600 Subject: [PATCH 04/10] Revert "Update for single/double precision, add tests" This reverts commit 48a6b5ca22052ab4236adc10f927b909ed23babb. --- .../engine/nn/layers/AveragePooling2DLayer.m | 16 ++------- code/nnv/engine/nn/layers/Conv2DLayer.m | 32 +++++------------ code/nnv/engine/nn/layers/ReluLayer.m | 2 +- .../test_layers_averagePooling.m | 35 +++++++------------ 4 files changed, 24 insertions(+), 61 deletions(-) diff --git a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m index bbfb10c271..af9b374c2b 100644 --- a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m +++ b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m @@ -228,20 +228,8 @@ function set_padding(obj, padding) % author: Dung Tran % date: 12/10/2018 % update: 7/26/2019 - - outputType = "double"; % default - - if isa(input, "single") - input = double(input); % vn_nnconv requires inputs to be of double precision - outputType = "single"; - end - - y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); - - % Return output with correct precision - if strcmp(outputType, "single") - y = single(y); - end + + y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); end diff --git a/code/nnv/engine/nn/layers/Conv2DLayer.m b/code/nnv/engine/nn/layers/Conv2DLayer.m index 29987ca5bb..8f2b82d2d4 100644 --- a/code/nnv/engine/nn/layers/Conv2DLayer.m +++ b/code/nnv/engine/nn/layers/Conv2DLayer.m @@ -435,27 +435,11 @@ function set_name(obj, name) function y = evaluate(obj, input) % @input: 3-dimensional array, for example, input(:, :, :) % @y: high-dimensional array (output volume), depth of output = number of filters - - outputType = "double"; % default - if isa(input, "single") - input = double(input); % vn_nnconv requires inputs to be of double precision - outputType = "single"; - end - - % Compute evaluation - if isa(obj.Weights, "single") - y = vl_nnconv(input, double(obj.Weights), double(obj.Bias), 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - elseif isa(obj.Weights, "double") - y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - else - error("Input and layer parameters must be single or double precision.") - end - - % Return output with correct precision - if strcmp(outputType, "single") - y = single(y); - end + % author: Dung Tran + % date: 7/18/2019 + + y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); end @@ -517,8 +501,8 @@ function set_name(obj, name) end % compute output sets - c = obj.evaluate(input.V(:,:,:,1)); - V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); + c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + V = vl_nnconv(input.V(:,:,:,2:input.numPred + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); Y = cat(4, c, V); S = ImageStar(Y, input.C, input.d, input.pred_lb, input.pred_ub); @@ -538,8 +522,8 @@ function set_name(obj, name) end % compute output sets - c = obj.evaluate(input.V(:,:,:,1)); - V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); + c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + V = vl_nnconv(input.V(:,:,:,2:input.numPreds + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); Y = cat(4, c, V); Z = ImageZono(Y); diff --git a/code/nnv/engine/nn/layers/ReluLayer.m b/code/nnv/engine/nn/layers/ReluLayer.m index 13ff7fca11..f7f060102f 100644 --- a/code/nnv/engine/nn/layers/ReluLayer.m +++ b/code/nnv/engine/nn/layers/ReluLayer.m @@ -105,7 +105,7 @@ w = in_image.width; c = in_image.numChannel; - Y = PosLin.reach(in_image.toStar, method, [], relaxFactor, dis_opt, lp_solver); % reachable set computation with ReLU + Y = PosLin.reach(in_image.toStar, method, [], relaxFactor); % reachable set computation with ReLU n = length(Y); images(n) = ImageStar; % transform back to ImageStar diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 9e5f5a390f..71e8b89895 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -1,18 +1,18 @@ %% test 1: AveragePooling2DLayer Compute averageMap -% I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input -% L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); -% averageMap = L.compute_averageMap(I); -% -% checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; -% assert(isequal(checker, averageMap)) -% -% -% %% test 2: AveragePooling2DLayer constructor -% L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); -% L2 = AveragePooling2DLayer(); -% L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); +I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input +L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); +averageMap = L.compute_averageMap(I); + +checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; +assert(isequal(checker, averageMap)) + + +%% test 2: AveragePooling2DLayer constructor +L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); +L2 = AveragePooling2DLayer(); +L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); %% test 3: AveragePooling2DLayer evaluation @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(single(inputVol)); +y = L.evaluate(inputVol); for i=1:3 for j=1:2 @@ -39,15 +39,6 @@ % display(inputVol(:,:,i)); end -%% test 3b: AveragePooling2DLayer evaluation (single precision) -% original input volume: color image with 3 channels -inputVol(:, :, 1) = [0 0 2 0 0; 1 2 0 2 0; 0 0 2 2 0; 0 2 2 2 2; 2 2 2 1 1]; % channel 1 input matrix -inputVol(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix -inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix - -L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(single(inputVol)); - %% test 4: AveragePooling2DLayer get zero padding input From 5d57d0adfd9d0f51f678d8996ba38847d1857d7e Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 19:11:03 -0600 Subject: [PATCH 05/10] process 3d medmnist results --- .../medmnist/multipleAttacks_3D_results.txt | 648 ++++++++++++++++++ .../examples/NN/medmnist/process_results.m | 2 +- 2 files changed, 649 insertions(+), 1 deletion(-) diff --git a/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt b/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt index 9922e6425f..7664b23526 100644 --- a/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt +++ b/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt @@ -1,5 +1,23 @@ ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_adrenalmnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.089655 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.091389 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -9,6 +27,24 @@ Number of unknown images = 10 Number of missclassified images = 38 Average computation time of 0.097665 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.094814 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.088423 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -18,6 +54,24 @@ Number of unknown images = 10 Number of missclassified images = 38 Average computation time of 0.093128 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.082751 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.090661 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -27,6 +81,24 @@ Number of unknown images = 10 Number of missclassified images = 38 Average computation time of 0.10202 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.10423 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.21532 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -36,6 +108,24 @@ Number of unknown images = 2 Number of missclassified images = 38 Average computation time of 0.23374 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.15527 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.15155 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -45,6 +135,24 @@ Number of unknown images = 2 Number of missclassified images = 38 Average computation time of 0.19122 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 157 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 38 +Average computation time of 0.15655 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 157 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 38 +Average computation time of 0.24745 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -57,6 +165,24 @@ Average computation time of 0.26513 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_fracturemnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 64 +Number of not robust images = 0 +Number of unknown images = 21 +Number of missclassified images = 115 +Average computation time of 0.066963 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 64 +Number of not robust images = 0 +Number of unknown images = 21 +Number of missclassified images = 115 +Average computation time of 0.071886 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -66,6 +192,24 @@ Number of unknown images = 21 Number of missclassified images = 115 Average computation time of 0.077774 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 62 +Number of not robust images = 0 +Number of unknown images = 23 +Number of missclassified images = 115 +Average computation time of 0.070278 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 62 +Number of not robust images = 0 +Number of unknown images = 23 +Number of missclassified images = 115 +Average computation time of 0.07056 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -75,6 +219,24 @@ Number of unknown images = 23 Number of missclassified images = 115 Average computation time of 0.084388 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 61 +Number of not robust images = 0 +Number of unknown images = 24 +Number of missclassified images = 115 +Average computation time of 0.062923 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 61 +Number of not robust images = 0 +Number of unknown images = 24 +Number of missclassified images = 115 +Average computation time of 0.068888 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -84,6 +246,24 @@ Number of unknown images = 24 Number of missclassified images = 115 Average computation time of 0.078273 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 85 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 115 +Average computation time of 0.11543 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 85 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 115 +Average computation time of 0.25427 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -93,6 +273,24 @@ Number of unknown images = 0 Number of missclassified images = 115 Average computation time of 0.27232 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 84 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 115 +Average computation time of 0.081964 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 84 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 115 +Average computation time of 0.20942 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -102,6 +300,24 @@ Number of unknown images = 1 Number of missclassified images = 115 Average computation time of 0.23118 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 115 +Average computation time of 0.081765 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 115 +Average computation time of 0.20911 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -114,6 +330,24 @@ Average computation time of 0.22384 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_nodulemnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 90 +Number of not robust images = 0 +Number of unknown images = 83 +Number of missclassified images = 27 +Average computation time of 0.091647 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 90 +Number of not robust images = 0 +Number of unknown images = 83 +Number of missclassified images = 27 +Average computation time of 0.10601 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -123,6 +357,24 @@ Number of unknown images = 83 Number of missclassified images = 27 Average computation time of 0.11423 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 93 +Number of missclassified images = 27 +Average computation time of 0.094546 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 93 +Number of missclassified images = 27 +Average computation time of 0.1101 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -132,6 +384,24 @@ Number of unknown images = 93 Number of missclassified images = 27 Average computation time of 0.12654 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 81 +Number of not robust images = 0 +Number of unknown images = 92 +Number of missclassified images = 27 +Average computation time of 0.10503 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 81 +Number of not robust images = 0 +Number of unknown images = 92 +Number of missclassified images = 27 +Average computation time of 0.116 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -141,6 +411,24 @@ Number of unknown images = 92 Number of missclassified images = 27 Average computation time of 0.13058 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 12 +Number of missclassified images = 27 +Average computation time of 0.10913 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 12 +Number of missclassified images = 27 +Average computation time of 0.1208 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -150,6 +438,24 @@ Number of unknown images = 12 Number of missclassified images = 27 Average computation time of 0.14372 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 138 +Number of not robust images = 0 +Number of unknown images = 35 +Number of missclassified images = 27 +Average computation time of 0.10546 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 138 +Number of not robust images = 0 +Number of unknown images = 35 +Number of missclassified images = 27 +Average computation time of 0.12612 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -159,6 +465,24 @@ Number of unknown images = 35 Number of missclassified images = 27 Average computation time of 0.14722 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 116 +Number of not robust images = 0 +Number of unknown images = 57 +Number of missclassified images = 27 +Average computation time of 0.11492 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 116 +Number of not robust images = 0 +Number of unknown images = 57 +Number of missclassified images = 27 +Average computation time of 0.14412 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -171,6 +495,24 @@ Average computation time of 0.17602 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_organmnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.14054 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.14117 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -180,6 +522,24 @@ Number of unknown images = 173 Number of missclassified images = 25 Average computation time of 0.14552 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.13693 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.13875 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -189,6 +549,24 @@ Number of unknown images = 173 Number of missclassified images = 25 Average computation time of 0.14997 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.1751 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.162 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -198,6 +576,24 @@ Number of unknown images = 173 Number of missclassified images = 25 Average computation time of 0.16166 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 28 +Number of not robust images = 0 +Number of unknown images = 147 +Number of missclassified images = 25 +Average computation time of 0.15954 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 28 +Number of not robust images = 0 +Number of unknown images = 147 +Number of missclassified images = 25 +Average computation time of 0.17367 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -207,6 +603,24 @@ Number of unknown images = 147 Number of missclassified images = 25 Average computation time of 0.18946 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 23 +Number of not robust images = 0 +Number of unknown images = 152 +Number of missclassified images = 25 +Average computation time of 0.15496 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 23 +Number of not robust images = 0 +Number of unknown images = 152 +Number of missclassified images = 25 +Average computation time of 0.19699 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -216,6 +630,24 @@ Number of unknown images = 152 Number of missclassified images = 25 Average computation time of 0.22786 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 16 +Number of not robust images = 0 +Number of unknown images = 159 +Number of missclassified images = 25 +Average computation time of 0.17645 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 16 +Number of not robust images = 0 +Number of unknown images = 159 +Number of missclassified images = 25 +Average computation time of 0.23706 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -228,6 +660,24 @@ Average computation time of 0.28619 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_synapsemnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.1092 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.10928 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -237,6 +687,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.10852 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.10622 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.11437 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -246,6 +714,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.117 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.10604 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.11188 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -255,6 +741,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.12074 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.11667 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.12212 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -264,6 +768,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.15097 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.12318 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.13739 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -273,6 +795,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.15911 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.14024 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.15435 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -285,6 +825,24 @@ Average computation time of 0.20001 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_vesselmnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 150 +Number of not robust images = 0 +Number of unknown images = 14 +Number of missclassified images = 36 +Average computation time of 0.09203 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 150 +Number of not robust images = 0 +Number of unknown images = 14 +Number of missclassified images = 36 +Average computation time of 0.10962 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -294,6 +852,24 @@ Number of unknown images = 14 Number of missclassified images = 36 Average computation time of 0.11734 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 154 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 36 +Average computation time of 0.10295 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 154 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 36 +Average computation time of 0.10439 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -303,6 +879,24 @@ Number of unknown images = 10 Number of missclassified images = 36 Average computation time of 0.11599 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 156 +Number of not robust images = 0 +Number of unknown images = 8 +Number of missclassified images = 36 +Average computation time of 0.10085 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 156 +Number of not robust images = 0 +Number of unknown images = 8 +Number of missclassified images = 36 +Average computation time of 0.09907 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -312,6 +906,24 @@ Number of unknown images = 8 Number of missclassified images = 36 Average computation time of 0.10129 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 164 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 36 +Average computation time of 0.10218 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 164 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 36 +Average computation time of 0.11383 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -321,6 +933,24 @@ Number of unknown images = 0 Number of missclassified images = 36 Average computation time of 0.19815 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 163 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 36 +Average computation time of 0.1618 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 163 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 36 +Average computation time of 0.20755 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -330,6 +960,24 @@ Number of unknown images = 1 Number of missclassified images = 36 Average computation time of 0.18294 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 3 +Number of missclassified images = 36 +Average computation time of 0.14705 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 3 +Number of missclassified images = 36 +Average computation time of 0.2101 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. diff --git a/code/nnv/examples/NN/medmnist/process_results.m b/code/nnv/examples/NN/medmnist/process_results.m index f5bf64507b..7d80990b79 100644 --- a/code/nnv/examples/NN/medmnist/process_results.m +++ b/code/nnv/examples/NN/medmnist/process_results.m @@ -39,7 +39,7 @@ function process_multiple_attacks(results) noise_vals = [1;2;3]; for i = 1:n(3) for j = 1:n(4) - for k = n(5) + for k = 1:n(5) % Print what results we are looking into disp("... processing " + names(i) + " attack with " ... +string(max_pixels(j)) + " pixels perturbed with noise of "+ string(noise_vals(k))); From 4106f205326ac9aacd61c7009cf777e1b1c8c95a Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 20:16:43 -0600 Subject: [PATCH 06/10] Add test to average pooling --- .../AveragePooling2DLayer/test_layers_averagePooling.m | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 71e8b89895..8f4314c194 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -39,6 +39,15 @@ % display(inputVol(:,:,i)); end +%% test 3b: AveragePooling2DLayer evaluation - single precision +% original input volume: color image with 3 channels +inputVol(:, :, 1) = [0 0 2 0 0; 1 2 0 2 0; 0 0 2 2 0; 0 2 2 2 2; 2 2 2 1 1]; % channel 1 input matrix +inputVol(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix +inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix + +L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); +y = L.evaluate(single(inputVol)); + %% test 4: AveragePooling2DLayer get zero padding input From a4d20ca614042a9ae253eb41cfd16ba8c1e6692b Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Wed, 13 Dec 2023 12:18:36 -0600 Subject: [PATCH 07/10] Fix parsing error for vnnlib from vnncomp --- code/nnv/engine/utils/load_vnnlib.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/engine/utils/load_vnnlib.m b/code/nnv/engine/utils/load_vnnlib.m index cbb3b5e2ba..cb96228e7f 100644 --- a/code/nnv/engine/utils/load_vnnlib.m +++ b/code/nnv/engine/utils/load_vnnlib.m @@ -374,7 +374,7 @@ if contains(x, 'X') [lb_input, ub_input] = process_input_constraint(x, lb_input, ub_input); else - [H, g] = process_output_combo_constraint(tline, H, g, output_dim); + [H, g] = process_output_combo_constraint(x, H, g, output_dim); end end Hg = HalfSpace(H,g); From d13897dd37a33de18f9e7f4ae86da9bd8c9d5462 Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Wed, 20 Dec 2023 11:21:34 -0600 Subject: [PATCH 08/10] Add reach options for tansig --- code/nnv/engine/nn/funcs/TanSig.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/engine/nn/funcs/TanSig.m b/code/nnv/engine/nn/funcs/TanSig.m index 1664f7159f..36fee9785e 100644 --- a/code/nnv/engine/nn/funcs/TanSig.m +++ b/code/nnv/engine/nn/funcs/TanSig.m @@ -59,7 +59,7 @@ if ~isa(I, 'Star') error('Input set is not a star set'); end - if strcmp(method, 'approx-star-no-split') || strcmp(method, 'approx-star') + if strcmp(method, 'approx-star-no-split') || strcmp(method, 'approx-star') || contains(method, 'relax-star') if relaxFactor == 0 S = TanSig.multiStepTanSig_NoSplit(I, dis_opt, lp_solver); else From 044b59afc22ded4ffedf715e2ba28e036698526c Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Thu, 21 Dec 2023 10:47:32 -0600 Subject: [PATCH 09/10] Fix bug on exporting index to value for vnnlib --- code/nnv/engine/utils/export2vnnlib.m | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/nnv/engine/utils/export2vnnlib.m b/code/nnv/engine/utils/export2vnnlib.m index 03e1b29d83..624a2eecdf 100644 --- a/code/nnv/engine/utils/export2vnnlib.m +++ b/code/nnv/engine/utils/export2vnnlib.m @@ -106,7 +106,10 @@ function export2vnnlib(lb, ub, outsize, property, name) % Outputs a string to write in the vnnlib file locs = find(hRow ~= 0); % Find indexes that are not zero - if hVal == 0 % Compare two indexes + if length(locs) > 1 + if hVal ~= 0 % Compare two indexes + error("Only allowed index to index comparison, or 1 index to value, but not both.") + end if hRow(locs(1)) > 0 % str = "(>= Y_"+string(locs(2)-1) + " " + "Y_"+string(locs(1)-1)+ "))"; else From d75dfde1694f8eebb8dff067b61139e3d9957255 Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Thu, 11 Jan 2024 18:14:19 -0600 Subject: [PATCH 10/10] Fix precision error on CAV23, add tests and other small fixes --- code/nnv/engine/nn/NN.m | 23 +++- code/nnv/engine/set/ImageStar.m | 24 ++++ code/nnv/engine/set/ImageZono.m | 17 ++- code/nnv/engine/set/Star.m | 34 ++++- code/nnv/engine/set/VolumeStar.m | 25 +++- code/nnv/engine/set/Zono.m | 14 ++ code/nnv/engine/utils/load_vnnlib.m | 14 +- .../Tutorial/other/set_representations.m | 5 +- .../layers/Conv2DLayer/test_layers_Conv2D.m | 122 +++++++++++------- code/nnv/tests/set/star/test_star_getZono.m | 1 + .../tests/set/star/test_star_plotBoxes_3D.m | 1 + 11 files changed, 219 insertions(+), 61 deletions(-) diff --git a/code/nnv/engine/nn/NN.m b/code/nnv/engine/nn/NN.m index b7f10d78ba..5e2977d6aa 100644 --- a/code/nnv/engine/nn/NN.m +++ b/code/nnv/engine/nn/NN.m @@ -237,13 +237,15 @@ if strcmp(obj.dis_opt, 'display') fprintf('\nPerform reachability analysis for the network %s \n', obj.Name); end + + % ensure NN parameters and input set share same precision + inputSet = obj.consistentPrecision(inputSet); % change only input, this can be changed in the future % Perform reachability based on connections or assume no skip/sparse connections if isempty(obj.Connections) outputSet = obj.reach_noConns(inputSet); else outputSet = obj.reach_withConns(inputSet); - end end @@ -945,6 +947,25 @@ reachOptions.numCores = 1; end end + + % Ensure input and parameter precision is the same + function inputSet = consistentPrecision(obj, inputSet) + % (assume parameters have same precision across layers) + % approach: change input precision based on network parameters + inputPrecision = class(inputSet.V); + netPrecision = 'double'; % default + for i=1:length(obj.Layers) + if isa(obj.Layers{i}, "FullyConnectedLayer") || isa(obj.Layers{i}, "Conv2DLayer") + netPrecision = class(obj.Layers{i}.Weights); + break; + end + end + if ~strcmp(inputPrecision, netPrecision) + % input and parameter precision does not match + warning("Changing input set precision to "+string(netPrecision)); + inputSet = inputSet.changeVarsPrecision(netPrecision); + end + end % Create input set based on input vector and bounds function R = create_input_set(obj, x_in, disturbance, lb_allowable, ub_allowable) % assume tol is applied to every vale of the input diff --git a/code/nnv/engine/set/ImageStar.m b/code/nnv/engine/set/ImageStar.m index 46f130373a..44bed518ad 100644 --- a/code/nnv/engine/set/ImageStar.m +++ b/code/nnv/engine/set/ImageStar.m @@ -656,6 +656,30 @@ end + % change variable precision + function S = changeVarsPrecision(obj, precision) + S = obj; + if strcmp(precision, 'single') + S.V = single(S.V); + S.C = single(S.C); + S.d = single(S.d); + S.pred_lb = single(S.pred_lb); + S.pred_ub = single(S.pred_lb); + S.im_lb = single(S.im_lb); + S.im_ub = single(S.im_ub); + elseif strcmp(precision, 'double') + S.V = double(S.V); + S.C = double(S.C); + S.d = double(S.d); + S.pred_lb = double(S.pred_lb); + S.pred_ub = double(S.pred_lb); + S.im_lb = double(S.im_lb); + S.im_ub = double(S.im_ub); + else + error("Only single or double precision arrays allowed. GpuArray/dlarray are coming.") + end + end + end diff --git a/code/nnv/engine/set/ImageZono.m b/code/nnv/engine/set/ImageZono.m index a3ca88f01b..111bad7316 100644 --- a/code/nnv/engine/set/ImageZono.m +++ b/code/nnv/engine/set/ImageZono.m @@ -280,7 +280,22 @@ S = obj.toImageStar; S = S.toStar; end - + + % change variable precision + function S = changeVarsPrecision(obj, precision) + S = obj; + if strcmp(precision, 'single') + S.V = single(S.V); + S.lb_image = single(S.lb_image); + S.ub_image = single(S.ub_image); + elseif strcmp(precision, 'double') + S.V = double(S.V); + S.lb_image = double(S.lb_image); + S.ub_image = double(S.ub_image); + else + error("Only single or double precision arrays allowed. GpuArray/dlarray are coming.") + end + end end diff --git a/code/nnv/engine/set/Star.m b/code/nnv/engine/set/Star.m index bdc91248f1..f02941c02e 100644 --- a/code/nnv/engine/set/Star.m +++ b/code/nnv/engine/set/Star.m @@ -1448,6 +1448,30 @@ end + % change variable precision + function S = changeVarsPrecision(obj, precision) + S = obj; + if strcmp(precision, 'single') + S.V = single(S.V); + S.C = single(S.C); + S.d = single(S.d); + S.predicate_lb = single(S.predicate_lb); + S.predicate_ub = single(S.predicate_lb); + S.state_lb = single(S.state_lb); + S.state_ub = single(S.state_ub); + elseif strcmp(precision, 'double') + S.V = double(S.V); + S.C = double(S.C); + S.d = double(S.d); + S.predicate_lb = double(S.predicate_lb); + S.predicate_ub = double(S.predicate_lb); + S.state_lb = double(S.state_lb); + S.state_ub = double(S.state_ub); + else + error("Only single or double precision arrays allowed. GpuArray/dlarray are coming.") + end + end + end @@ -1526,8 +1550,14 @@ function plot(varargin) end end else - P = obj.toPolyhedron; - P.plot('color', color); + if isa(obj.V, 'single') || isa(obj.C,'single') || isa(obj.d, 'single') + S = obj.changeVarsPrecision('double'); + P = S.toPolyhedron; + P.plot('color', color); + else + P = obj.toPolyhedron; + P.plot('color', color); + end end else diff --git a/code/nnv/engine/set/VolumeStar.m b/code/nnv/engine/set/VolumeStar.m index db2a0d396b..bec37d75c2 100644 --- a/code/nnv/engine/set/VolumeStar.m +++ b/code/nnv/engine/set/VolumeStar.m @@ -523,7 +523,30 @@ end - % TODO: add a 3D projection to ImageStar + % change variable precision + function S = changeVarsPrecision(obj, precision) + S = obj; + if strcmp(precision, 'single') + S.V = single(S.V); + S.C = single(S.C); + S.d = single(S.d); + S.pred_lb = single(S.pred_lb); + S.pred_ub = single(S.pred_lb); + S.vol_lb = single(S.vol_lb); + S.vol_ub = single(S.vol_ub); + elseif strcmp(precision, 'double') + S.V = double(S.V); + S.C = double(S.C); + S.d = double(S.d); + S.pred_lb = double(S.pred_lb); + S.pred_ub = double(S.pred_lb); + S.vol_lb = double(S.vol_lb); + S.vol_ub = double(S.vol_ub); + else + error("Only single or double precision arrays allowed. GpuArray/dlarray are coming.") + end + end + end diff --git a/code/nnv/engine/set/Zono.m b/code/nnv/engine/set/Zono.m index 5745181d67..7970d0d19a 100644 --- a/code/nnv/engine/set/Zono.m +++ b/code/nnv/engine/set/Zono.m @@ -343,6 +343,20 @@ imageStar = im1.toImageStar(height, width, numChannels); end + + % change variable precision + function S = changeVarsPrecision(obj, precision) + S = obj; + if strcmp(precision, 'single') + S.V = single(S.V); + S.c = single(S.c); + elseif strcmp(precision, 'double') + S.V = double(S.V); + S.c = double(S.c); + else + error("Only single or double precision arrays allowed. GpuArray/dlarray are coming.") + end + end end diff --git a/code/nnv/engine/utils/load_vnnlib.m b/code/nnv/engine/utils/load_vnnlib.m index cb96228e7f..c850736151 100644 --- a/code/nnv/engine/utils/load_vnnlib.m +++ b/code/nnv/engine/utils/load_vnnlib.m @@ -35,8 +35,8 @@ dim = dim + 1; % only have seen inputs defined as vectors, so this should work % the more general approach would require some extra work, but should be easy as well elseif contains(tline, "declare-const") && contains(tline, "Y_") - lb_template = zeros(dim,1); - ub_template = zeros(dim,1); + lb_template = zeros(dim,1,'single'); + ub_template = zeros(dim,1,'single'); dim = 0; % reset dimension counter phase = "DeclareOutput"; continue; % redo this line in correct phase @@ -51,7 +51,7 @@ dim = 1; % reset dimension counter phase = "DefineInput"; % Initialize variables - lb_input = lb_template; + lb_input = lb_template; ub_input = ub_template; continue; % redo this line in correct phase end @@ -240,7 +240,7 @@ H(idx2) = -1; else var2 = split(var2, ')'); - g = str2double(var2{1}); + g = single(str2double(var2{1})); end else H(idx1) = -1; @@ -252,7 +252,7 @@ H(idx2) = 1; else var2 = split(var2, ')'); - g = -str2double(var2{1}); + g = -single(str2double(var2{1})); end end % Add constraint (H, g) to assertion variable (ast) @@ -346,7 +346,7 @@ dim = split(t{2},'_'); dim = str2double(dim{2})+1; value = split(t{3},')'); - value = str2double(value{1}); + value = single(str2double(value{1})); if contains(t{1},">=") lb_input(dim) = value; else @@ -401,7 +401,7 @@ dim = split(t{2},'_'); dim = str2double(dim{2})+1; value = split(t{3},')'); - value = str2double(value{1}); + value = single(str2double(value{1})); if contains(t{1},">=") || contains(t{1}, ">") Hvec(dim) = -1; gval = -value; diff --git a/code/nnv/examples/Tutorial/other/set_representations.m b/code/nnv/examples/Tutorial/other/set_representations.m index 78ad4d44d8..08b748fe08 100644 --- a/code/nnv/examples/Tutorial/other/set_representations.m +++ b/code/nnv/examples/Tutorial/other/set_representations.m @@ -64,11 +64,12 @@ S1 = S.affineMap([1 0 0 0 0; 0 1 0 0 0], []); % 2D, dims 1 and 2 figure; -Star.plot(S1); +Star.plotBoxes_2D(S1,1,2,'r'); +hold on; S3 = S.affineMap([1 0 0 0 0; 0 1 0 0 0; 0 0 1 0 0], []); % 3D, dims 1,2 and 3 figure; -Star.plot(S3); +Star.plotBoxes_3D(S3,1,2,3,'r'); % 2) Plot an overapproximation (box around Star) of the set and plot diff --git a/code/nnv/tests/nn/layers/Conv2DLayer/test_layers_Conv2D.m b/code/nnv/tests/nn/layers/Conv2DLayer/test_layers_Conv2D.m index 6e3712613b..3042dc4fb7 100644 --- a/code/nnv/tests/nn/layers/Conv2DLayer/test_layers_Conv2D.m +++ b/code/nnv/tests/nn/layers/Conv2DLayer/test_layers_Conv2D.m @@ -10,13 +10,9 @@ -%___________________________________________________________________________________________________ -%tests below originally taken from test_Conv2DLayer_compute_featureMap.m - %% test 1: Conv2DLayer feature map - I = [1 1 1 0 0; 0 1 1 1 0; 0 0 1 1 1; 0 0 1 1 0; 0 1 1 0 0]; % input W = [1 0 1; 0 1 0; 1 0 1]; % filter @@ -24,21 +20,13 @@ stride = [1 1]; dilation = [1 1]; - featureMap = Conv2DLayer.compute_featureMap(I, W, padding, stride, dilation); - checker=[I(1, 1)+I(1, 3)+I(2, 2)+I(3, 1)+I(3, 3), I(1, 2)+I(1, 4)+I(2, 3)+I(3, 2)+I(3, 4), I(1, 3)+I(1, 5)+I(2, 4)+I(3, 3)+I(3, 5); I(2, 1)+I(2, 3)+I(3, 2)+I(4, 1)+I(4, 3), I(2, 2)+I(2, 4)+I(3, 3)+I(4, 2)+I(4, 4), I(2, 3)+I(2, 5)+I(3, 4)+I(4, 3)+I(4, 5); I(3, 1)+I(3, 3)+I(4, 2)+I(5, 1)+I(5, 3), I(3, 2)+I(3, 4)+I(4, 3)+I(5, 2)+I(5, 4), I(3, 3)+I(3, 5)+I(4, 4)+I(5, 3)+I(5, 5);]; assert(isequal(checker, featureMap)) - - -%___________________________________________________________________________________________________ -%tests below originally taken from test_Conv2DLayer_constructor.m - - %% test 2: Conv2DLayer constructor @@ -50,6 +38,7 @@ W(:,:, 1, 1) = [1 1 1; 1 1 -1; 0 -1 0]; % channel 1 W(:, :, 2, 1) = [0 1 1; 0 -1 0; 0 0 -1]; % channel 2 W(:, :, 3, 1) = [1 -1 0; 1 1 0; -1 0 1]; % channel 3 + % filter 2 weight matrix with 3 channels W(:, :, 1, 2) = [1 1 -1; -1 0 1; 1 -1 -1]; % channel 1 W(:, :, 2, 2) = [-1 1 1; 0 1 1; 0 -1 1]; % channel 2 @@ -59,27 +48,17 @@ b(:, :, 1) = 1; % filter 1 b(:, :, 2) = 0; % filter 2 - L0 = Conv2DLayer(W, b); L1 = Conv2DLayer('Test_Cov2DLayer', W, b, [1 1 1 1], [1 1], [1 1]); - - -%___________________________________________________________________________________________________ -%tests below originally taken from test_Conv2DLayer_get_zero_padding_input.m - - %% test 3: Conv2DLayer zero padding - - % original input volume: color image with 3 channels inputVol(:, :, 1) = [2 0 1 2 1; 1 0 2 2 2; 1 2 2 0 2; 1 2 0 0 1; 1 0 1 1 2]; % channel 1 input matrix inputVol(:, :, 2) = [0 0 1 0 1; 0 0 2 1 1; 1 1 0 1 1; 1 1 0 2 2; 2 1 2 0 0]; % channel 2 input matrix inputVol(:, :, 3) = [1 2 2 1 0; 2 0 0 2 0; 0 0 1 0 1; 1 2 0 2 0; 1 0 2 1 0]; % channel 3 input matrix - % construct input with padding operation paddingSize = [1 1 1 1]; I = Conv2DLayer.get_zero_padding_input(inputVol, paddingSize); @@ -95,11 +74,6 @@ assert(isempty(find(left_right))) - -%___________________________________________________________________________________________________ -%tests below originally taken from test_Conv2DLayer_reach.m - - %% test 4: Conv2DLayer reach % construct a Conv2DLayer object @@ -110,6 +84,7 @@ W(:,:, 1, 1) = [1 1 1; 1 1 -1; 0 -1 0]; % channel 1 W(:, :, 2, 1) = [0 1 1; 0 -1 0; 0 0 -1]; % channel 2 W(:, :, 3, 1) = [1 -1 0; 1 1 0; -1 0 1]; % channel 3 + % filter 2 weight matrix with 3 channels W(:, :, 1, 2) = [1 1 -1; -1 0 1; 1 -1 -1]; % channel 1 W(:, :, 2, 2) = [-1 1 1; 0 1 1; 0 -1 1]; % channel 2 @@ -119,7 +94,6 @@ b(:, :, 1) = 1; % filter 1 b(:, :, 2) = 0; % filter 2 - L0 = Conv2DLayer(W, b); LB(:,:,1) = [-0.1 -0.2 0 0; 0 0 0 0; 0 0 0 0; 0 0 0 0]; % attack on pixel (1,1) and (1,2) @@ -130,21 +104,49 @@ UB(:,:,2) = [0.1 0.15 0 0; 0 0 0 0; 0 0 0 0; 0 0 0 0]; UB(:,:,3) = UB(:,:,2); -image = ImageZono(LB, UB); +image = ImageStar(LB, UB); + +L0.reach(image); + +%% %% test 4b: Conv2DLayer reach (single precision) -%Z = L0.reach(image); %THIS TRIGGERS AN ERROR, BECAUSE IT'S NOT AN IMAGE +% construct a Conv2DLayer object +% this convolution layer is from the link: +% http://cs231n.github.io/convolutional-networks/ + +% filter 1 weight matrix with 3 channels +W(:,:, 1, 1) = [1 1 1; 1 1 -1; 0 -1 0]; % channel 1 +W(:, :, 2, 1) = [0 1 1; 0 -1 0; 0 0 -1]; % channel 2 +W(:, :, 3, 1) = [1 -1 0; 1 1 0; -1 0 1]; % channel 3 + +% filter 2 weight matrix with 3 channels +W(:, :, 1, 2) = [1 1 -1; -1 0 1; 1 -1 -1]; % channel 1 +W(:, :, 2, 2) = [-1 1 1; 0 1 1; 0 -1 1]; % channel 2 +W(:, :, 3, 2) = [-1 1 -1; -1 0 -1; -1 -1 -1]; % channel 3 + +% biases for 2 filters +b(:, :, 1) = 1; % filter 1 +b(:, :, 2) = 0; % filter 2 -S = L0.reach(image.toImageStar); +L0 = Conv2DLayer(single(W), single(b)); +LB(:,:,1) = [-0.1 -0.2 0 0; 0 0 0 0; 0 0 0 0; 0 0 0 0]; % attack on pixel (1,1) and (1,2) +LB(:,:,2) = [-0.1 -0.15 0 0; 0 0 0 0; 0 0 0 0; 0 0 0 0]; +LB(:,:,3) = LB(:,:,2); +UB(:,:,1) = [0 0.2 0 0; 0 0 0 0; 0 0 0 0; 0 0 0 0]; +UB(:,:,2) = [0.1 0.15 0 0; 0 0 0 0; 0 0 0 0; 0 0 0 0]; +UB(:,:,3) = UB(:,:,2); -%___________________________________________________________________________________________________ -%tests below originally taken from test_Conv2DLayer_reach_star_exact_single_input.m +image = ImageStar(LB, UB); +image = image.changeVarsPrecision('single'); -%% test 5: Conv2DLayer reach star exact +L0.reach(image); +%% test 5: Conv2DLayer reach star exact + % construct a Conv2DLayer object % this convolution layer is from the link: % http://cs231n.github.io/convolutional-networks/ @@ -164,12 +166,8 @@ b(:, :, 1) = 1; % filter 1 b(:, :, 2) = 0; % filter 2 - - - L = Conv2DLayer(W, b); -L.set_weights_biases(W, b); - +% L.set_weights_biases(W, b); % the convolutional layer has 2 filters of size 3x3 padding = 1; @@ -185,7 +183,6 @@ IM(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix IM(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix - LB(:,:,1) = [-0.1 -0.2 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0]; % attack on pixel (1,1) and (1,2) LB(:,:,2) = [-0.1 -0.15 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0]; LB(:,:,3) = LB(:,:,2); @@ -196,24 +193,55 @@ input = ImageStar(IM, LB, UB); - Y = L.reach_star_single_input(input); +%% test 5b: Conv2DLayer reach star exact (single precision input) +% construct a Conv2DLayer object +% this convolution layer is from the link: +% http://cs231n.github.io/convolutional-networks/ +% filter has a size of 3 x 3 +% filter 1 weight matrix with 3 channels +W(:,:, 1, 1) = [-1 0 0; 1 1 0; -1 0 -1]; % channel 1 +W(:, :, 2, 1) = [-1 0 -1; 1 -1 1; 1 1 0]; % channel 2 +W(:, :, 3, 1) = [-1 1 1; -1 -1 -1; 0 0 -1]; % channel 3 +% filter 2 weight matrix with 3 channels +W(:, :, 1, 2) = [-1 1 0; 0 0 -1; 1 0 0]; % channel 1 +W(:, :, 2, 2) = [1 0 0; 0 0 -1; 0 0 1]; % channel 2 +W(:, :, 3, 2) = [1 -1 -1; 1 1 0; -1 0 1]; % channel 3 +% biases for 2 filters +b(:, :, 1) = 1; % filter 1 +b(:, :, 2) = 0; % filter 2 +L = Conv2DLayer(single(W), single(b)); +% L.set_weights_biases(W, b); +% the convolutional layer has 2 filters of size 3x3 +padding = 1; +stride = 2; +dilation = 1; +L.set_stride(stride); +L.set_padding(padding); +L.set_dilation(dilation); +% original input volume: color image with 3 channels +IM(:, :, 1) = [0 0 2 0 0; 1 2 0 2 0; 0 0 2 2 0; 0 2 2 2 2; 2 2 2 1 1]; % channel 1 input matrix +IM(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix +IM(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix +LB(:,:,1) = [-0.1 -0.2 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0]; % attack on pixel (1,1) and (1,2) +LB(:,:,2) = [-0.1 -0.15 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0]; +LB(:,:,3) = LB(:,:,2); +UB(:,:,1) = [0.1 0.2 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0;0 0 0 0 0]; +UB(:,:,2) = [0.1 0.15 0 0 0; 0 0 0 0 0; 0 0 0 0 0; 0 0 0 0 0;0 0 0 0 0]; +UB(:,:,3) = UB(:,:,2); +input = ImageStar(IM, LB, UB); +input = input.changeVarsPrecision('single'); - - - - - - +Y = L.reach_star_single_input(input); \ No newline at end of file diff --git a/code/nnv/tests/set/star/test_star_getZono.m b/code/nnv/tests/set/star/test_star_getZono.m index 179ed60855..e309740f0a 100644 --- a/code/nnv/tests/set/star/test_star_getZono.m +++ b/code/nnv/tests/set/star/test_star_getZono.m @@ -1,3 +1,4 @@ +rng(3); center = [1; 1]; V = [1 0 1; 0 1 1]; P = ExamplePoly.randHrep('d',3); diff --git a/code/nnv/tests/set/star/test_star_plotBoxes_3D.m b/code/nnv/tests/set/star/test_star_plotBoxes_3D.m index b6d08f9b7e..80d873f943 100644 --- a/code/nnv/tests/set/star/test_star_plotBoxes_3D.m +++ b/code/nnv/tests/set/star/test_star_plotBoxes_3D.m @@ -1,3 +1,4 @@ +rng(1); center = [1; 1; 1]; V = [1 0 1; 0 1 1; 1 0 0]; P = ExamplePoly.randHrep('d',3);