From 48a6b5ca22052ab4236adc10f927b909ed23babb Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 13:09:50 -0600 Subject: [PATCH 1/9] Update for single/double precision, add tests --- .../engine/nn/layers/AveragePooling2DLayer.m | 16 +++++++-- code/nnv/engine/nn/layers/Conv2DLayer.m | 32 ++++++++++++----- code/nnv/engine/nn/layers/ReluLayer.m | 2 +- .../test_layers_averagePooling.m | 35 ++++++++++++------- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m index af9b374c2b..bbfb10c271 100644 --- a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m +++ b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m @@ -228,8 +228,20 @@ function set_padding(obj, padding) % author: Dung Tran % date: 12/10/2018 % update: 7/26/2019 - - y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); + + outputType = "double"; % default + + if isa(input, "single") + input = double(input); % vn_nnconv requires inputs to be of double precision + outputType = "single"; + end + + y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); + + % Return output with correct precision + if strcmp(outputType, "single") + y = single(y); + end end diff --git a/code/nnv/engine/nn/layers/Conv2DLayer.m b/code/nnv/engine/nn/layers/Conv2DLayer.m index 8f2b82d2d4..29987ca5bb 100644 --- a/code/nnv/engine/nn/layers/Conv2DLayer.m +++ b/code/nnv/engine/nn/layers/Conv2DLayer.m @@ -435,11 +435,27 @@ function set_name(obj, name) function y = evaluate(obj, input) % @input: 3-dimensional array, for example, input(:, :, :) % @y: high-dimensional array (output volume), depth of output = number of filters + + outputType = "double"; % default - % author: Dung Tran - % date: 7/18/2019 - - y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + if isa(input, "single") + input = double(input); % vn_nnconv requires inputs to be of double precision + outputType = "single"; + end + + % Compute evaluation + if isa(obj.Weights, "single") + y = vl_nnconv(input, double(obj.Weights), double(obj.Bias), 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + elseif isa(obj.Weights, "double") + y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + else + error("Input and layer parameters must be single or double precision.") + end + + % Return output with correct precision + if strcmp(outputType, "single") + y = single(y); + end end @@ -501,8 +517,8 @@ function set_name(obj, name) end % compute output sets - c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - V = vl_nnconv(input.V(:,:,:,2:input.numPred + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + c = obj.evaluate(input.V(:,:,:,1)); + V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); Y = cat(4, c, V); S = ImageStar(Y, input.C, input.d, input.pred_lb, input.pred_ub); @@ -522,8 +538,8 @@ function set_name(obj, name) end % compute output sets - c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - V = vl_nnconv(input.V(:,:,:,2:input.numPreds + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + c = obj.evaluate(input.V(:,:,:,1)); + V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); Y = cat(4, c, V); Z = ImageZono(Y); diff --git a/code/nnv/engine/nn/layers/ReluLayer.m b/code/nnv/engine/nn/layers/ReluLayer.m index f7f060102f..13ff7fca11 100644 --- a/code/nnv/engine/nn/layers/ReluLayer.m +++ b/code/nnv/engine/nn/layers/ReluLayer.m @@ -105,7 +105,7 @@ w = in_image.width; c = in_image.numChannel; - Y = PosLin.reach(in_image.toStar, method, [], relaxFactor); % reachable set computation with ReLU + Y = PosLin.reach(in_image.toStar, method, [], relaxFactor, dis_opt, lp_solver); % reachable set computation with ReLU n = length(Y); images(n) = ImageStar; % transform back to ImageStar diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 71e8b89895..9e5f5a390f 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -1,18 +1,18 @@ %% test 1: AveragePooling2DLayer Compute averageMap -I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input -L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); -averageMap = L.compute_averageMap(I); - -checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; -assert(isequal(checker, averageMap)) - - -%% test 2: AveragePooling2DLayer constructor -L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); -L2 = AveragePooling2DLayer(); -L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); +% I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input +% L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); +% averageMap = L.compute_averageMap(I); +% +% checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; +% assert(isequal(checker, averageMap)) +% +% +% %% test 2: AveragePooling2DLayer constructor +% L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); +% L2 = AveragePooling2DLayer(); +% L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); %% test 3: AveragePooling2DLayer evaluation @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(inputVol); +y = L.evaluate(single(inputVol)); for i=1:3 for j=1:2 @@ -39,6 +39,15 @@ % display(inputVol(:,:,i)); end +%% test 3b: AveragePooling2DLayer evaluation (single precision) +% original input volume: color image with 3 channels +inputVol(:, :, 1) = [0 0 2 0 0; 1 2 0 2 0; 0 0 2 2 0; 0 2 2 2 2; 2 2 2 1 1]; % channel 1 input matrix +inputVol(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix +inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix + +L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); +y = L.evaluate(single(inputVol)); + %% test 4: AveragePooling2DLayer get zero padding input From 8e2e5691e017bcebd12c28a434ac6489a7e5362a Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 15:49:27 -0600 Subject: [PATCH 2/9] Fix test --- .../layers/AveragePooling2DLayer/test_layers_averagePooling.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 9e5f5a390f..08fb53f976 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(single(inputVol)); +y = L.evaluate(inputVol); for i=1:3 for j=1:2 From eb75ea13fd7c3e0719611989a89029d587f24dd1 Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 18:57:55 -0600 Subject: [PATCH 3/9] Revert "Fix test" This reverts commit 8e2e5691e017bcebd12c28a434ac6489a7e5362a. --- .../layers/AveragePooling2DLayer/test_layers_averagePooling.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 08fb53f976..9e5f5a390f 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(inputVol); +y = L.evaluate(single(inputVol)); for i=1:3 for j=1:2 From 79f5808176ae10929b79a02c614ec0b8a09869db Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 18:58:00 -0600 Subject: [PATCH 4/9] Revert "Update for single/double precision, add tests" This reverts commit 48a6b5ca22052ab4236adc10f927b909ed23babb. --- .../engine/nn/layers/AveragePooling2DLayer.m | 16 ++------- code/nnv/engine/nn/layers/Conv2DLayer.m | 32 +++++------------ code/nnv/engine/nn/layers/ReluLayer.m | 2 +- .../test_layers_averagePooling.m | 35 +++++++------------ 4 files changed, 24 insertions(+), 61 deletions(-) diff --git a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m index bbfb10c271..af9b374c2b 100644 --- a/code/nnv/engine/nn/layers/AveragePooling2DLayer.m +++ b/code/nnv/engine/nn/layers/AveragePooling2DLayer.m @@ -228,20 +228,8 @@ function set_padding(obj, padding) % author: Dung Tran % date: 12/10/2018 % update: 7/26/2019 - - outputType = "double"; % default - - if isa(input, "single") - input = double(input); % vn_nnconv requires inputs to be of double precision - outputType = "single"; - end - - y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); - - % Return output with correct precision - if strcmp(outputType, "single") - y = single(y); - end + + y = vl_nnpool(input, obj.PoolSize, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Method', 'avg'); end diff --git a/code/nnv/engine/nn/layers/Conv2DLayer.m b/code/nnv/engine/nn/layers/Conv2DLayer.m index 29987ca5bb..8f2b82d2d4 100644 --- a/code/nnv/engine/nn/layers/Conv2DLayer.m +++ b/code/nnv/engine/nn/layers/Conv2DLayer.m @@ -435,27 +435,11 @@ function set_name(obj, name) function y = evaluate(obj, input) % @input: 3-dimensional array, for example, input(:, :, :) % @y: high-dimensional array (output volume), depth of output = number of filters - - outputType = "double"; % default - if isa(input, "single") - input = double(input); % vn_nnconv requires inputs to be of double precision - outputType = "single"; - end - - % Compute evaluation - if isa(obj.Weights, "single") - y = vl_nnconv(input, double(obj.Weights), double(obj.Bias), 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - elseif isa(obj.Weights, "double") - y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); - else - error("Input and layer parameters must be single or double precision.") - end - - % Return output with correct precision - if strcmp(outputType, "single") - y = single(y); - end + % author: Dung Tran + % date: 7/18/2019 + + y = vl_nnconv(input, obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); end @@ -517,8 +501,8 @@ function set_name(obj, name) end % compute output sets - c = obj.evaluate(input.V(:,:,:,1)); - V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); + c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + V = vl_nnconv(input.V(:,:,:,2:input.numPred + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); Y = cat(4, c, V); S = ImageStar(Y, input.C, input.d, input.pred_lb, input.pred_ub); @@ -538,8 +522,8 @@ function set_name(obj, name) end % compute output sets - c = obj.evaluate(input.V(:,:,:,1)); - V = obj.evaluate(input.V(:,:,:,2:input.numPred + 1)); + c = vl_nnconv(input.V(:,:,:,1), obj.Weights, obj.Bias, 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); + V = vl_nnconv(input.V(:,:,:,2:input.numPreds + 1), obj.Weights, [], 'Stride', obj.Stride, 'Pad', obj.PaddingSize, 'Dilate', obj.DilationFactor); Y = cat(4, c, V); Z = ImageZono(Y); diff --git a/code/nnv/engine/nn/layers/ReluLayer.m b/code/nnv/engine/nn/layers/ReluLayer.m index 13ff7fca11..f7f060102f 100644 --- a/code/nnv/engine/nn/layers/ReluLayer.m +++ b/code/nnv/engine/nn/layers/ReluLayer.m @@ -105,7 +105,7 @@ w = in_image.width; c = in_image.numChannel; - Y = PosLin.reach(in_image.toStar, method, [], relaxFactor, dis_opt, lp_solver); % reachable set computation with ReLU + Y = PosLin.reach(in_image.toStar, method, [], relaxFactor); % reachable set computation with ReLU n = length(Y); images(n) = ImageStar; % transform back to ImageStar diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 9e5f5a390f..71e8b89895 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -1,18 +1,18 @@ %% test 1: AveragePooling2DLayer Compute averageMap -% I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input -% L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); -% averageMap = L.compute_averageMap(I); -% -% checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; -% assert(isequal(checker, averageMap)) -% -% -% %% test 2: AveragePooling2DLayer constructor -% L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); -% L2 = AveragePooling2DLayer(); -% L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); +I = [1 0 2 3; 4 6 6 8; 3 1 1 0; 1 2 2 4]; % input +L = AveragePooling2DLayer([2 2], [2 2], [0 0 0 0]); +averageMap = L.compute_averageMap(I); + +checker=[(I(1, 1)+I(1, 2)+I(2, 1)+I(2, 2))/4, (I(1, 3)+I(1, 4)+I(2, 3)+I(2, 4))/4; (I(3, 1)+I(3, 2)+I(4, 1)+I(4, 2))/4, (I(3, 3)+I(3, 4)+I(4, 3)+I(4, 4))/4]; +assert(isequal(checker, averageMap)) + + +%% test 2: AveragePooling2DLayer constructor +L1 = AveragePooling2DLayer('test_average_pooling_2d_layer', [2 2], [1 1], [0 0 0 0]); +L2 = AveragePooling2DLayer(); +L3 = AveragePooling2DLayer([3 3], [1 1], [0 0 0 0]); %% test 3: AveragePooling2DLayer evaluation @@ -22,7 +22,7 @@ inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(single(inputVol)); +y = L.evaluate(inputVol); for i=1:3 for j=1:2 @@ -39,15 +39,6 @@ % display(inputVol(:,:,i)); end -%% test 3b: AveragePooling2DLayer evaluation (single precision) -% original input volume: color image with 3 channels -inputVol(:, :, 1) = [0 0 2 0 0; 1 2 0 2 0; 0 0 2 2 0; 0 2 2 2 2; 2 2 2 1 1]; % channel 1 input matrix -inputVol(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix -inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix - -L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); -y = L.evaluate(single(inputVol)); - %% test 4: AveragePooling2DLayer get zero padding input From 5d57d0adfd9d0f51f678d8996ba38847d1857d7e Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 19:11:03 -0600 Subject: [PATCH 5/9] process 3d medmnist results --- .../medmnist/multipleAttacks_3D_results.txt | 648 ++++++++++++++++++ .../examples/NN/medmnist/process_results.m | 2 +- 2 files changed, 649 insertions(+), 1 deletion(-) diff --git a/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt b/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt index 9922e6425f..7664b23526 100644 --- a/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt +++ b/code/nnv/examples/NN/medmnist/multipleAttacks_3D_results.txt @@ -1,5 +1,23 @@ ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_adrenalmnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.089655 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.091389 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -9,6 +27,24 @@ Number of unknown images = 10 Number of missclassified images = 38 Average computation time of 0.097665 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.094814 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.088423 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -18,6 +54,24 @@ Number of unknown images = 10 Number of missclassified images = 38 Average computation time of 0.093128 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.082751 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 152 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 38 +Average computation time of 0.090661 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -27,6 +81,24 @@ Number of unknown images = 10 Number of missclassified images = 38 Average computation time of 0.10202 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.10423 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.21532 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -36,6 +108,24 @@ Number of unknown images = 2 Number of missclassified images = 38 Average computation time of 0.23374 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.15527 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 160 +Number of not robust images = 0 +Number of unknown images = 2 +Number of missclassified images = 38 +Average computation time of 0.15155 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -45,6 +135,24 @@ Number of unknown images = 2 Number of missclassified images = 38 Average computation time of 0.19122 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 157 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 38 +Average computation time of 0.15655 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 157 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 38 +Average computation time of 0.24745 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -57,6 +165,24 @@ Average computation time of 0.26513 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_fracturemnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 64 +Number of not robust images = 0 +Number of unknown images = 21 +Number of missclassified images = 115 +Average computation time of 0.066963 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 64 +Number of not robust images = 0 +Number of unknown images = 21 +Number of missclassified images = 115 +Average computation time of 0.071886 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -66,6 +192,24 @@ Number of unknown images = 21 Number of missclassified images = 115 Average computation time of 0.077774 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 62 +Number of not robust images = 0 +Number of unknown images = 23 +Number of missclassified images = 115 +Average computation time of 0.070278 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 62 +Number of not robust images = 0 +Number of unknown images = 23 +Number of missclassified images = 115 +Average computation time of 0.07056 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -75,6 +219,24 @@ Number of unknown images = 23 Number of missclassified images = 115 Average computation time of 0.084388 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 61 +Number of not robust images = 0 +Number of unknown images = 24 +Number of missclassified images = 115 +Average computation time of 0.062923 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 61 +Number of not robust images = 0 +Number of unknown images = 24 +Number of missclassified images = 115 +Average computation time of 0.068888 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -84,6 +246,24 @@ Number of unknown images = 24 Number of missclassified images = 115 Average computation time of 0.078273 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 85 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 115 +Average computation time of 0.11543 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 85 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 115 +Average computation time of 0.25427 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -93,6 +273,24 @@ Number of unknown images = 0 Number of missclassified images = 115 Average computation time of 0.27232 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 84 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 115 +Average computation time of 0.081964 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 84 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 115 +Average computation time of 0.20942 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -102,6 +300,24 @@ Number of unknown images = 1 Number of missclassified images = 115 Average computation time of 0.23118 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 115 +Average computation time of 0.081765 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 5 +Number of missclassified images = 115 +Average computation time of 0.20911 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -114,6 +330,24 @@ Average computation time of 0.22384 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_nodulemnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 90 +Number of not robust images = 0 +Number of unknown images = 83 +Number of missclassified images = 27 +Average computation time of 0.091647 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 90 +Number of not robust images = 0 +Number of unknown images = 83 +Number of missclassified images = 27 +Average computation time of 0.10601 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -123,6 +357,24 @@ Number of unknown images = 83 Number of missclassified images = 27 Average computation time of 0.11423 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 93 +Number of missclassified images = 27 +Average computation time of 0.094546 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 80 +Number of not robust images = 0 +Number of unknown images = 93 +Number of missclassified images = 27 +Average computation time of 0.1101 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -132,6 +384,24 @@ Number of unknown images = 93 Number of missclassified images = 27 Average computation time of 0.12654 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 81 +Number of not robust images = 0 +Number of unknown images = 92 +Number of missclassified images = 27 +Average computation time of 0.10503 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 81 +Number of not robust images = 0 +Number of unknown images = 92 +Number of missclassified images = 27 +Average computation time of 0.116 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -141,6 +411,24 @@ Number of unknown images = 92 Number of missclassified images = 27 Average computation time of 0.13058 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 12 +Number of missclassified images = 27 +Average computation time of 0.10913 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 12 +Number of missclassified images = 27 +Average computation time of 0.1208 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -150,6 +438,24 @@ Number of unknown images = 12 Number of missclassified images = 27 Average computation time of 0.14372 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 138 +Number of not robust images = 0 +Number of unknown images = 35 +Number of missclassified images = 27 +Average computation time of 0.10546 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 138 +Number of not robust images = 0 +Number of unknown images = 35 +Number of missclassified images = 27 +Average computation time of 0.12612 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -159,6 +465,24 @@ Number of unknown images = 35 Number of missclassified images = 27 Average computation time of 0.14722 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 116 +Number of not robust images = 0 +Number of unknown images = 57 +Number of missclassified images = 27 +Average computation time of 0.11492 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 116 +Number of not robust images = 0 +Number of unknown images = 57 +Number of missclassified images = 27 +Average computation time of 0.14412 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -171,6 +495,24 @@ Average computation time of 0.17602 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_organmnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.14054 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.14117 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -180,6 +522,24 @@ Number of unknown images = 173 Number of missclassified images = 25 Average computation time of 0.14552 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.13693 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.13875 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -189,6 +549,24 @@ Number of unknown images = 173 Number of missclassified images = 25 Average computation time of 0.14997 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.1751 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 2 +Number of not robust images = 0 +Number of unknown images = 173 +Number of missclassified images = 25 +Average computation time of 0.162 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -198,6 +576,24 @@ Number of unknown images = 173 Number of missclassified images = 25 Average computation time of 0.16166 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 28 +Number of not robust images = 0 +Number of unknown images = 147 +Number of missclassified images = 25 +Average computation time of 0.15954 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 28 +Number of not robust images = 0 +Number of unknown images = 147 +Number of missclassified images = 25 +Average computation time of 0.17367 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -207,6 +603,24 @@ Number of unknown images = 147 Number of missclassified images = 25 Average computation time of 0.18946 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 23 +Number of not robust images = 0 +Number of unknown images = 152 +Number of missclassified images = 25 +Average computation time of 0.15496 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 23 +Number of not robust images = 0 +Number of unknown images = 152 +Number of missclassified images = 25 +Average computation time of 0.19699 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -216,6 +630,24 @@ Number of unknown images = 152 Number of missclassified images = 25 Average computation time of 0.22786 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 16 +Number of not robust images = 0 +Number of unknown images = 159 +Number of missclassified images = 25 +Average computation time of 0.17645 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 16 +Number of not robust images = 0 +Number of unknown images = 159 +Number of missclassified images = 25 +Average computation time of 0.23706 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -228,6 +660,24 @@ Average computation time of 0.28619 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_synapsemnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.1092 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.10928 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -237,6 +687,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.10852 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.10622 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.11437 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -246,6 +714,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.117 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.10604 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.11188 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -255,6 +741,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.12074 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.11667 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.12212 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -264,6 +768,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.15097 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.12318 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.13739 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -273,6 +795,24 @@ Number of unknown images = 0 Number of missclassified images = 0 Average computation time of 0.15911 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.14024 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 200 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 0 +Average computation time of 0.15435 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -285,6 +825,24 @@ Average computation time of 0.20001 ******************************************************* ******************************************************* ================= PROCESSING RESULTS: results/verification_multipleAttacks_vesselmnist3d.mat ... +... processing dark attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 150 +Number of not robust images = 0 +Number of unknown images = 14 +Number of missclassified images = 36 +Average computation time of 0.09203 + +... processing dark attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 150 +Number of not robust images = 0 +Number of unknown images = 14 +Number of missclassified images = 36 +Average computation time of 0.10962 + ... processing dark attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -294,6 +852,24 @@ Number of unknown images = 14 Number of missclassified images = 36 Average computation time of 0.11734 +... processing dark attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 154 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 36 +Average computation time of 0.10295 + +... processing dark attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 154 +Number of not robust images = 0 +Number of unknown images = 10 +Number of missclassified images = 36 +Average computation time of 0.10439 + ... processing dark attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -303,6 +879,24 @@ Number of unknown images = 10 Number of missclassified images = 36 Average computation time of 0.11599 +... processing dark attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 156 +Number of not robust images = 0 +Number of unknown images = 8 +Number of missclassified images = 36 +Average computation time of 0.10085 + +... processing dark attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 156 +Number of not robust images = 0 +Number of unknown images = 8 +Number of missclassified images = 36 +Average computation time of 0.09907 + ... processing dark attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -312,6 +906,24 @@ Number of unknown images = 8 Number of missclassified images = 36 Average computation time of 0.10129 +... processing bright attack with 50 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 164 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 36 +Average computation time of 0.10218 + +... processing bright attack with 50 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 164 +Number of not robust images = 0 +Number of unknown images = 0 +Number of missclassified images = 36 +Average computation time of 0.11383 + ... processing bright attack with 50 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -321,6 +933,24 @@ Number of unknown images = 0 Number of missclassified images = 36 Average computation time of 0.19815 +... processing bright attack with 100 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 163 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 36 +Average computation time of 0.1618 + +... processing bright attack with 100 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 163 +Number of not robust images = 0 +Number of unknown images = 1 +Number of missclassified images = 36 +Average computation time of 0.20755 + ... processing bright attack with 100 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. @@ -330,6 +960,24 @@ Number of unknown images = 1 Number of missclassified images = 36 Average computation time of 0.18294 +... processing bright attack with 200 pixels perturbed with noise of 1 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 3 +Number of missclassified images = 36 +Average computation time of 0.14705 + +... processing bright attack with 200 pixels perturbed with noise of 2 +----------- ROBUSTNESS RESULTS ------------- +Verification results of 200 images. +Number of robust images = 161 +Number of not robust images = 0 +Number of unknown images = 3 +Number of missclassified images = 36 +Average computation time of 0.2101 + ... processing bright attack with 200 pixels perturbed with noise of 3 ----------- ROBUSTNESS RESULTS ------------- Verification results of 200 images. diff --git a/code/nnv/examples/NN/medmnist/process_results.m b/code/nnv/examples/NN/medmnist/process_results.m index f5bf64507b..7d80990b79 100644 --- a/code/nnv/examples/NN/medmnist/process_results.m +++ b/code/nnv/examples/NN/medmnist/process_results.m @@ -39,7 +39,7 @@ function process_multiple_attacks(results) noise_vals = [1;2;3]; for i = 1:n(3) for j = 1:n(4) - for k = n(5) + for k = 1:n(5) % Print what results we are looking into disp("... processing " + names(i) + " attack with " ... +string(max_pixels(j)) + " pixels perturbed with noise of "+ string(noise_vals(k))); From 4106f205326ac9aacd61c7009cf777e1b1c8c95a Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Tue, 12 Dec 2023 20:16:43 -0600 Subject: [PATCH 6/9] Add test to average pooling --- .../AveragePooling2DLayer/test_layers_averagePooling.m | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m index 71e8b89895..8f4314c194 100644 --- a/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m +++ b/code/nnv/tests/nn/layers/AveragePooling2DLayer/test_layers_averagePooling.m @@ -39,6 +39,15 @@ % display(inputVol(:,:,i)); end +%% test 3b: AveragePooling2DLayer evaluation - single precision +% original input volume: color image with 3 channels +inputVol(:, :, 1) = [0 0 2 0 0; 1 2 0 2 0; 0 0 2 2 0; 0 2 2 2 2; 2 2 2 1 1]; % channel 1 input matrix +inputVol(:, :, 2) = [1 2 2 1 2; 2 1 2 0 2; 2 2 2 0 1; 1 1 1 0 0; 1 0 2 2 1]; % channel 2 input matrix +inputVol(:, :, 3) = [0 0 2 2 1; 0 2 1 1 2; 0 2 0 0 1; 0 2 1 0 1; 1 2 1 0 0]; % channel 3 input matrix + +L = AveragePooling2DLayer([3 3], [2 2], [0 0 0 0]); +y = L.evaluate(single(inputVol)); + %% test 4: AveragePooling2DLayer get zero padding input From a4d20ca614042a9ae253eb41cfd16ba8c1e6692b Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Wed, 13 Dec 2023 12:18:36 -0600 Subject: [PATCH 7/9] Fix parsing error for vnnlib from vnncomp --- code/nnv/engine/utils/load_vnnlib.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/engine/utils/load_vnnlib.m b/code/nnv/engine/utils/load_vnnlib.m index cbb3b5e2ba..cb96228e7f 100644 --- a/code/nnv/engine/utils/load_vnnlib.m +++ b/code/nnv/engine/utils/load_vnnlib.m @@ -374,7 +374,7 @@ if contains(x, 'X') [lb_input, ub_input] = process_input_constraint(x, lb_input, ub_input); else - [H, g] = process_output_combo_constraint(tline, H, g, output_dim); + [H, g] = process_output_combo_constraint(x, H, g, output_dim); end end Hg = HalfSpace(H,g); From d13897dd37a33de18f9e7f4ae86da9bd8c9d5462 Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Wed, 20 Dec 2023 11:21:34 -0600 Subject: [PATCH 8/9] Add reach options for tansig --- code/nnv/engine/nn/funcs/TanSig.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/nnv/engine/nn/funcs/TanSig.m b/code/nnv/engine/nn/funcs/TanSig.m index 1664f7159f..36fee9785e 100644 --- a/code/nnv/engine/nn/funcs/TanSig.m +++ b/code/nnv/engine/nn/funcs/TanSig.m @@ -59,7 +59,7 @@ if ~isa(I, 'Star') error('Input set is not a star set'); end - if strcmp(method, 'approx-star-no-split') || strcmp(method, 'approx-star') + if strcmp(method, 'approx-star-no-split') || strcmp(method, 'approx-star') || contains(method, 'relax-star') if relaxFactor == 0 S = TanSig.multiStepTanSig_NoSplit(I, dis_opt, lp_solver); else From 044b59afc22ded4ffedf715e2ba28e036698526c Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Thu, 21 Dec 2023 10:47:32 -0600 Subject: [PATCH 9/9] Fix bug on exporting index to value for vnnlib --- code/nnv/engine/utils/export2vnnlib.m | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/nnv/engine/utils/export2vnnlib.m b/code/nnv/engine/utils/export2vnnlib.m index 03e1b29d83..624a2eecdf 100644 --- a/code/nnv/engine/utils/export2vnnlib.m +++ b/code/nnv/engine/utils/export2vnnlib.m @@ -106,7 +106,10 @@ function export2vnnlib(lb, ub, outsize, property, name) % Outputs a string to write in the vnnlib file locs = find(hRow ~= 0); % Find indexes that are not zero - if hVal == 0 % Compare two indexes + if length(locs) > 1 + if hVal ~= 0 % Compare two indexes + error("Only allowed index to index comparison, or 1 index to value, but not both.") + end if hRow(locs(1)) > 0 % str = "(>= Y_"+string(locs(2)-1) + " " + "Y_"+string(locs(1)-1)+ "))"; else