From eb619b45c032134acc2bb0b4a332f1765fabf68c Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Wed, 19 Jun 2024 13:22:38 -0500 Subject: [PATCH] Working on 3d verification, add gurobi as solver option --- code/nnv/engine/utils/lpsolver.m | 32 ++- .../WiP_3d/functions/L_inf_attack.m | 7 - .../Submission/WiP_3d/functions/dark_attack.m | 8 +- .../WiP_3d/functions/summarize_results.m | 2 + .../WiP_3d/functions/verify_instance_3d.m | 21 +- code/nnv/examples/Submission/WiP_3d/run_all.m | 60 ++--- .../Submission/WiP_3d/verify_adrenal.m | 1 + .../Submission/WiP_3d/verify_fracture.m | 209 ++++------------- .../Submission/WiP_3d/verify_nodule.m | 210 ++++-------------- .../examples/Submission/WiP_3d/verify_organ.m | 209 ++++------------- .../Submission/WiP_3d/verify_synapse.m | 209 ++++------------- .../Submission/WiP_3d/verify_vessel.m | 2 +- .../Submission/WiP_3d/visualize_results.m | 47 ++++ 13 files changed, 266 insertions(+), 751 deletions(-) delete mode 100644 code/nnv/examples/Submission/WiP_3d/functions/L_inf_attack.m create mode 100644 code/nnv/examples/Submission/WiP_3d/functions/summarize_results.m create mode 100644 code/nnv/examples/Submission/WiP_3d/visualize_results.m diff --git a/code/nnv/engine/utils/lpsolver.m b/code/nnv/engine/utils/lpsolver.m index 923c1f2636..39fbf0a1ae 100644 --- a/code/nnv/engine/utils/lpsolver.m +++ b/code/nnv/engine/utils/lpsolver.m @@ -32,8 +32,38 @@ Aeq = double(Aeq); Beq = double(Beq); ub = double(ub); end + if strcmp(lp_solver, 'gurobi') % no backup solver, should be better than the others + % Create gurobi model + model.obj = f; % objective function + model.A = [sparse(A); sparse(Aeq)]; % A must be sparse + model.sense = [repmat('<',size(A,1),1); repmat('=',size(Aeq,1),1)]; + model.rhs = full([b(:); Beq(:)]); % rhs must be dense + if ~isempty(lb) + model.lb = lb; + else + model.lb = -inf(size(model.A,2),1); % default lb for MATLAB is -inf + end + if ~isempty(ub) + model.ub = ub; + end + % Define solver parameters + params = struct; % for now, leave default options/params + params.OutputFlag = 0; % no display + result = gurobi(model, params); + fval = result.objval; % get fval value from results + % get exitflag and match those of linprog for easier parsing + if strcmp(result.status,'OPTIMAL') + exitflag = "l1"; % converged to a solution + elseif strcmp(result.status,'UNBOUNDED') + exitflag = "l-5"; % problem is unbounded + elseif strcmp(result.status,'ITERATION_LIMIT') + exitflag = "l-2"; % maximum number of iterations reached + else + exitflag = "l-2"; % no feasible point found + end + % Solve using linprog (glpk as backup) - if strcmp(lp_solver, 'linprog') + elseif strcmp(lp_solver, 'linprog') options = optimoptions(@linprog, 'Display','none'); options.OptimalityTolerance = 1e-10; % set tolerance % first try solving using linprog diff --git a/code/nnv/examples/Submission/WiP_3d/functions/L_inf_attack.m b/code/nnv/examples/Submission/WiP_3d/functions/L_inf_attack.m deleted file mode 100644 index 70fb5f06c5..0000000000 --- a/code/nnv/examples/Submission/WiP_3d/functions/L_inf_attack.m +++ /dev/null @@ -1,7 +0,0 @@ -function [outputArg1,outputArg2] = Linf_attack(inputArg1,inputArg2) -%LINF_ATTACK Summary of this function goes here -% Detailed explanation goes here -outputArg1 = inputArg1; -outputArg2 = inputArg2; -end - diff --git a/code/nnv/examples/Submission/WiP_3d/functions/dark_attack.m b/code/nnv/examples/Submission/WiP_3d/functions/dark_attack.m index e11e7c1dd6..ec0d83d93e 100644 --- a/code/nnv/examples/Submission/WiP_3d/functions/dark_attack.m +++ b/code/nnv/examples/Submission/WiP_3d/functions/dark_attack.m @@ -32,10 +32,10 @@ % Define input set as VolumeStar dif_vol = vol - at_vol; noise = -dif_vol; - V(:,:,:,:,1) = vol; % center of set - V(:,:,:,:,2) = noise; % basis vectors - C = [1; -1]; % constraints - d = [1; noise_disturbance-1]; + V(:,:,:,:,1) = vol; % center of set + V(:,:,:,:,2) = noise; % basis vectors + C = [1; -1]; % constraints + d = [1; noise_disturbance-1]; % constraints I = VolumeStar(V, C, d, 1-noise_disturbance, 1); % input set end diff --git a/code/nnv/examples/Submission/WiP_3d/functions/summarize_results.m b/code/nnv/examples/Submission/WiP_3d/functions/summarize_results.m new file mode 100644 index 0000000000..139597f9cb --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/functions/summarize_results.m @@ -0,0 +1,2 @@ + + diff --git a/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_3d.m b/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_3d.m index 0e46547069..20fa9eee1c 100644 --- a/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_3d.m +++ b/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_3d.m @@ -1,13 +1,10 @@ -function results = verify_instance_3d(net, vol, target, attack, reachOptions, max_value, min_value) +function results = verify_instance_3d(net, vol, target, attack, reachOptions) % verify medmnist with inputs (input images), targets (labels) and attack % (struct with adversarial attack info) % results = verify_medmnist(net, matlabNet, inputs, targets, attack, max_value*, min_value*) % Check what type of attack to consider - if strcmp(attack.Name, 'linf') - epsilon = attack.epsilon; - max_pixels = attack.max_pixels; - elseif strcmp(attack.Name, 'dark') || strcmp(attack.Name, 'bright') + if strcmp(attack.Name, 'dark') || strcmp(attack.Name, 'bright') max_pixels = attack.max_pixels; threshold = attack.threshold; noise_disturbance = attack.noise_de; @@ -15,18 +12,8 @@ error("Adversarial attack not supported."); end - % check if max_value and min_value are provided - if ~exist("max_value", 'var') - max_value = inf; - end - if ~exist("min_value", 'var') - min_value = -inf; - end - % Choose attack - if strcmp(attack.Name,'linf') - I = L_inf_attack(vol, epsilon, max_pixels, max_value, min_value); - elseif strcmp(attack.Name, 'dark') + if strcmp(attack.Name, 'dark') I = dark_attack(vol, max_pixels, threshold, noise_disturbance); elseif strcmp(attack.Name, 'bright') I = bright_attack(vol, max_pixels, threshold, noise_disturbance); @@ -48,7 +35,7 @@ return end - % Check for falsification (TODO) + % Check for falsification n_samples = 100; % number of random samples to try for falsification xRand = I.sample(n_samples); for k = 1:n_samples diff --git a/code/nnv/examples/Submission/WiP_3d/run_all.m b/code/nnv/examples/Submission/WiP_3d/run_all.m index e8d5391e05..e87c0011fa 100644 --- a/code/nnv/examples/Submission/WiP_3d/run_all.m +++ b/code/nnv/examples/Submission/WiP_3d/run_all.m @@ -1,53 +1,23 @@ -%% Shape only data +%% Shape only data (3d) -% poolobj = gcp('nocreate'); -% delete(poolobj); +verify_adrenal; -% verify_adrenal; +verify_vessel; -% poolobj = gcp('nocreate'); -% delete(poolobj); +%% Volume data (general 3D) -% verify_vessel; -%% Volume data (general 3D) +disp("... fracture ..."); +verify_fracture; + + +disp("... nodule ...") +verify_nodule; + + +disp("... organ ...") +verify_organ; -% poolobj = gcp('nocreate'); -% delete(poolobj); -% -% disp("... fracture ..."); -% try -% verify_fracture; -% catch ME -% warning(ME.message); -% end -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% poolobj = gcp('nocreate'); -% delete(poolobj); -% -% disp("... nodule ...") -% try -% verify_nodule; -% catch ME -% warning(ME.message); -% end -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% poolobj = gcp('nocreate'); -% delete(poolobj); - -% disp("... organ ...") -% try -% verify_organ; -% catch ME -% warning(ME.message); -% end -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% poolobj = gcp('nocreate'); -% delete(poolobj); disp("... synapse ...") -try - verify_synapse; -catch ME - warning(ME.message) -end \ No newline at end of file +verify_synapse; diff --git a/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m b/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m index 8ab9bb66ad..38b8fb4d00 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m @@ -28,6 +28,7 @@ reachOptions = struct; reachOptions.reachMethod = 'relax-star-area'; reachOptions.relaxFactor = 0.95; +reachOptions.lp_solver = "gurobi"; %% Attack 1 diff --git a/code/nnv/examples/Submission/WiP_3d/verify_fracture.m b/code/nnv/examples/Submission/WiP_3d/verify_fracture.m index 04f7fc76a3..c66a8c60f3 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_fracture.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_fracture.m @@ -28,172 +28,43 @@ reachOptions = struct; reachOptions.reachMethod = 'relax-star-area'; reachOptions.relaxFactor = 0.95; - - -%% Attack 1 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 2 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 3 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 4 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 5 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 6 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 7 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 8 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_fracture_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - +reachOptions.lp_solver = "gurobi"; + +% Study variables +advType = ["bright", "dark"]; +maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels +epsilon = [2, 4, 10]; % ep / 255 +threshold = [100; 150]; % bright ; dark + +%% Verification analysis +for a=advType + for mp=maxpixels + for ep=epsilon + + % 1) Initialize results var + results = zeros(N,2); + + % 2) Create adversarial attack + adv_attack = struct; + adv_attack.Name = a; % bright or dark + if strcmp(a, "bright") + adv_attack.threshold = threshold(1); % perturb pixels below this value + else + adv_attack.threshold = threshold(2); % perturb pixels below this value + end + adv_attack.max_pixels = mp; % Max number of pixels to modify from input image + adv_attack.noise_de = ep/255; % disturbance (noise) on pixels + + % 3) Begin verification analysis + for i=1:N + img = squeeze(inputs(:,:,:,:,i)); + target = targets(i); + results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); + end + + % 4) % save results + save("results/verification_fracture_"+ a +"_" + ep +"_" + mp + ".mat", "results"); + + end + end +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/verify_nodule.m b/code/nnv/examples/Submission/WiP_3d/verify_nodule.m index 1798ce746d..d2b09d0778 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_nodule.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_nodule.m @@ -28,172 +28,44 @@ reachOptions = struct; reachOptions.reachMethod = 'relax-star-area'; reachOptions.relaxFactor = 0.95; - - -%% Attack 1 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 2 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 3 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 4 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 5 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 6 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 7 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 8 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_nodule_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - +reachOptions.lp_solver = "gurobi"; + + +% Study variables +advType = ["bright", "dark"]; +maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels +epsilon = [2, 4, 10]; % ep / 255 +threshold = [100; 150]; % bright ; dark + +%% Verification analysis +for a=advType + for mp=maxpixels + for ep=epsilon + + % 1) Initialize results var + results = zeros(N,2); + + % 2) Create adversarial attack + adv_attack = struct; + adv_attack.Name = a; % bright or dark + if strcmp(a, "bright") + adv_attack.threshold = threshold(1); % perturb pixels below this value + else + adv_attack.threshold = threshold(2); % perturb pixels below this value + end + adv_attack.max_pixels = mp; % Max number of pixels to modify from input image + adv_attack.noise_de = ep/255; % disturbance (noise) on pixels + + % 3) Begin verification analysis + for i=1:N + img = squeeze(inputs(:,:,:,:,i)); + target = targets(i); + results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); + end + + % 4) % save results + save("results/verification_nodule_"+ a +"_" + ep +"_" + mp + ".mat", "results"); + + end + end +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/verify_organ.m b/code/nnv/examples/Submission/WiP_3d/verify_organ.m index 29e9c93250..c05164bf48 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_organ.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_organ.m @@ -28,172 +28,43 @@ reachOptions = struct; reachOptions.reachMethod = 'relax-star-area'; reachOptions.relaxFactor = 0.95; - - -%% Attack 1 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 2 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 3 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 4 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 5 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 6 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 7 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 8 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_organ_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - +reachOptions.lp_solver = "gurobi"; + +% Study variables +advType = ["bright", "dark"]; +maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels +epsilon = [2, 4, 10]; % ep / 255 +threshold = [100; 150]; % bright ; dark + +%% Verification analysis +for a=advType + for mp=maxpixels + for ep=epsilon + + % 1) Initialize results var + results = zeros(N,2); + + % 2) Create adversarial attack + adv_attack = struct; + adv_attack.Name = a; % bright or dark + if strcmp(a, "bright") + adv_attack.threshold = threshold(1); % perturb pixels below this value + else + adv_attack.threshold = threshold(2); % perturb pixels below this value + end + adv_attack.max_pixels = mp; % Max number of pixels to modify from input image + adv_attack.noise_de = ep/255; % disturbance (noise) on pixels + + % 3) Begin verification analysis + for i=1:N + img = squeeze(inputs(:,:,:,:,i)); + target = targets(i); + results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); + end + + % 4) % save results + save("results/verification_organ_"+ a +"_" + ep +"_" + mp + ".mat", "results"); + + end + end +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/verify_synapse.m b/code/nnv/examples/Submission/WiP_3d/verify_synapse.m index 390e9ba026..76d0160cdd 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_synapse.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_synapse.m @@ -28,172 +28,43 @@ reachOptions = struct; reachOptions.reachMethod = 'relax-star-area'; reachOptions.relaxFactor = 0.95; - - -%% Attack 1 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 2 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 3 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 4 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 1; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 5 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 6 - -adv_attack = struct; -adv_attack.Name = "bright"; -adv_attack.threshold = 100; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 7 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 50; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 8 - -adv_attack = struct; -adv_attack.Name = "dark"; -adv_attack.threshold = 150; % perturb pixels below this value -adv_attack.max_pixels = 100; % Max number of pixels to modify from input image -adv_attack.noise_de = 2; % disturbance (noise) on pixels - -results = zeros(N,2); % verification result, time - -% verify volumes -parfor i=1:N - img = squeeze(inputs(:,:,:,:,i)); - target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -end - -% save results -save("results/verification_synapse_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - +reachOptions.lp_solver = "gurobi"; + +% Study variables +advType = ["bright", "dark"]; +maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels +epsilon = [2, 4, 10]; % ep / 255 +threshold = [100; 150]; % bright ; dark + +%% Verification analysis +for a=advType + for mp=maxpixels + for ep=epsilon + + % 1) Initialize results var + results = zeros(N,2); + + % 2) Create adversarial attack + adv_attack = struct; + adv_attack.Name = a; % bright or dark + if strcmp(a, "bright") + adv_attack.threshold = threshold(1); % perturb pixels below this value + else + adv_attack.threshold = threshold(2); % perturb pixels below this value + end + adv_attack.max_pixels = mp; % Max number of pixels to modify from input image + adv_attack.noise_de = ep/255; % disturbance (noise) on pixels + + % 3) Begin verification analysis + for i=1:N + img = squeeze(inputs(:,:,:,:,i)); + target = targets(i); + results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); + end + + % 4) % save results + save("results/verification_synapse_"+ a +"_" + ep +"_" + mp + ".mat", "results"); + + end + end +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/verify_vessel.m b/code/nnv/examples/Submission/WiP_3d/verify_vessel.m index 655f9188df..d8775e01e2 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_vessel.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_vessel.m @@ -28,7 +28,7 @@ reachOptions = struct; reachOptions.reachMethod = 'relax-star-area'; reachOptions.relaxFactor = 0.95; - +reachOptions.lp_solver = "gurobi"; %% Attack 1 diff --git a/code/nnv/examples/Submission/WiP_3d/visualize_results.m b/code/nnv/examples/Submission/WiP_3d/visualize_results.m new file mode 100644 index 0000000000..2cef9c40d6 --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/visualize_results.m @@ -0,0 +1,47 @@ +%% Get all results, one dataset at a time + +results = dir('results'); + +% Variable in the study +datasets = ["adrenal"; "fracture"; "nodule"; "organ"; "synapse"; "vessel"]; +attackType = ["bright"; "dark"]; +maxpixels = [50;100;500;1000]; +epsilon = [2/255; 4/255; 10/255]; + +%% Visualize verification results/trends + +% What do we want to show? + +% 1) Trends +% - Time: How does the time increases as the input set gets larger? +% - Epsilon increasing +% - Number of pixels increasing +% - results: How robust are the models as we increase the size of the attack? more unknowns? more sats? +% - Epsilon increasing +% - Number of pixels increasing +% 2) Certified robust accuracy +% - Is it much worse than the accuracy of the model? +% - Do we need more samples? + + + +%% Helper functions + +function summary = summarize_results(resFile) + % Provide details from results file + summary = struct; + data = load(resFile); + results = data.results; + % Total number of samples examined + summary.N = length(results); + % Verified (unsat) + summary.unsat = sum(results(:,1) == 1); + % Falsified (sat) from input set + summary.sat = sum(results(:,1) == 0); + % Unknown (not using exact) + summary.unknown = sum(results(:,1) == 2); + % Misclasified (sat) of original image + summary.missclassified = sum(results(:,1) == -1); + % Also possible -2 (error), which may be out of memory (most common here) + summary.avgTime = sum(results(:,2))/summary.N; +end \ No newline at end of file