diff --git a/code/nnv/engine/nn/layers/MaxPooling2DLayer.m b/code/nnv/engine/nn/layers/MaxPooling2DLayer.m index 1317c17372..b8dd039414 100644 --- a/code/nnv/engine/nn/layers/MaxPooling2DLayer.m +++ b/code/nnv/engine/nn/layers/MaxPooling2DLayer.m @@ -580,7 +580,7 @@ function set_padding(obj, padding) if n > 0 for i=1:n m1 = length(images); - images = obj.stepSplitMultipleInputs(images, pad_image, split_pos(i, :, :), max_index{split_pos(i, 1), split_pos(i, 2), split_pos(i, 3)}, []); + images = obj.stepSplitMultipleInputs(images, pad_image, split_pos(i, :, :), max_index{split_pos(i, 1), split_pos(i, 2), split_pos(i, 3)}, [], lp_solver); m2 = length(images); if strcmp(dis_opt, 'display') fprintf('\nSplit %d images into %d images', m1, m2); diff --git a/code/nnv/engine/utils/lpsolver.m b/code/nnv/engine/utils/lpsolver.m index bf419da5ea..ccde224ead 100644 --- a/code/nnv/engine/utils/lpsolver.m +++ b/code/nnv/engine/utils/lpsolver.m @@ -49,6 +49,8 @@ % Define solver parameters params = struct; % for now, leave default options/params params.OutputFlag = 0; % no display + params.OptimalityTol = 1e-09; + params.FeasibilityTol = 1e-09; result = gurobi(model, params); fval = result.objval; % get fval value from results % get exitflag and match those of linprog for easier parsing diff --git a/code/nnv/examples/Submission/WiP_3d/functions/add_voxels.m b/code/nnv/examples/Submission/WiP_3d/functions/add_voxels.m index 8b97e67dde..2c50ed1c99 100644 --- a/code/nnv/examples/Submission/WiP_3d/functions/add_voxels.m +++ b/code/nnv/examples/Submission/WiP_3d/functions/add_voxels.m @@ -5,32 +5,18 @@ % Return a VolumeStar of a brightening attack on a few pixels % Initialize vars - ct = 0; % keep track of pixels modified - flag = 0; % determine when to stop modifying pixels vol = single(vol); at_vol = vol; - % Create brightening attack - for i=1:size(vol,1) - for j=1:size(vol,2) - for k=1:size(vol,3) - if vol(i,j,k) < threshold - at_vol(i,j,k) = 255; - ct = ct + 1; - if ct >= max_pixels - flag = 1; - break; - end - end - end - if flag == 1 - break - end - end - if flag == 1 - break; - end - end + % we can find the edge of the shape + shape = edge3(vol,'approxcanny',0.6); % this should be okay for this data, but let's test it + + % select a random pixel + idxs = setdiff(find(shape), find(vol)); + voxels = min(voxels,length(idxs)); + + % For now, we can select the first ones + at_vol(idxs(1:voxels)) = 255; % Define input set as VolumeStar dif_vol = -vol + at_vol; diff --git a/code/nnv/examples/Submission/WiP_3d/functions/remove_voxels.m b/code/nnv/examples/Submission/WiP_3d/functions/remove_voxels.m index e69de29bb2..dc5703098a 100644 --- a/code/nnv/examples/Submission/WiP_3d/functions/remove_voxels.m +++ b/code/nnv/examples/Submission/WiP_3d/functions/remove_voxels.m @@ -0,0 +1,31 @@ +function I = remove_voxels(vol, voxels, noise_disturbance) + % noise_disturnamce can be kept fixed here, more interesting on number + % of voxels changed + + % Return a VolumeStar of a brightening attack on a few pixels + + % Initialize vars + vol = single(vol); + at_vol = vol; + + % we can find the edge of the shape + shape = edge3(vol,'approxcanny',0.6); % this should be okay for this data, but let's test it + + % select a random pixel + idxs = intersect(find(shape),find(vol)); + voxels = min(voxels,length(idxs)); + + % For now, we can select the first ones + at_vol(idxs(1:voxels)) = 0; + + % Define input set as VolumeStar + dif_vol = -vol + at_vol; + noise = dif_vol; + V(:,:,:,:,1) = vol; % center of set + V(:,:,:,:,2) = noise; % basis vectors + C = [1; -1]; % constraints + d = [1; noise_disturbance-1]; % constraints + I = VolumeStar(V, C, d, 1-noise_disturbance, 1); % input set + + +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/functions/summarize_results.m b/code/nnv/examples/Submission/WiP_3d/functions/summarize_results.m deleted file mode 100644 index 139597f9cb..0000000000 --- a/code/nnv/examples/Submission/WiP_3d/functions/summarize_results.m +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_shape.m b/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_shape.m index 0b8e0b66e5..784a01c3af 100644 --- a/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_shape.m +++ b/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_shape.m @@ -6,7 +6,6 @@ % Check what type of attack to consider if strcmp(attack.Name, 'add') || strcmp(attack.Name, 'remove') max_pixels = attack.max_pixels; - threshold = attack.threshold; noise_disturbance = attack.noise_de; else error("Adversarial attack not supported."); @@ -14,9 +13,9 @@ % Choose attack if strcmp(attack.Name, 'add') - I = add_voxels(vol, max_pixels, threshold, noise_disturbance); + I = add_voxels(vol, max_pixels, noise_disturbance); elseif strcmp(attack.Name, 'remove') - I = remove_voxels(vol, max_pixels, threshold, noise_disturbance); + I = remove_voxels(vol, max_pixels, noise_disturbance); end % Begin analysis diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/README.md b/code/nnv/examples/Submission/WiP_3d/other/NAV/README.md new file mode 100644 index 0000000000..ce27712094 --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/NAV/README.md @@ -0,0 +1,39 @@ + +# NAV Benchmark + +## Property: +The control goal is to navigate a robot to a goal region while avoiding an obstacle. +Time horizon: `t = 6s`. Control period: `0.2s`. + +Initial states: + + x1 = [2.9, 3.1] + x2 = [2.9, 3.1] + x3 = [0, 0] + x4 = [0, 0] + +Dynamic system: [dynamics.m](./dynamics.m) + +Goal region ( t=6 ): + + x1 = [-0.5, 0.5] + x2 = [-0.5, 0.5] + x3 = [-Inf, Inf] + x4 = [-Inf, Inf] + +Obstacle ( always ): + + x1 = [1, 2] + x2 = [1, 2] + x3 = [-Inf, Inf] + x4 = [-Inf, Inf] + +## Networks: + +We provide two networks: +- The first network is trained with standard (point-based) reinforcement learning: `nn-nav-point.onnx` +- The second network is trained set-based to improve its verifiable robustness by integrating reachability analysis into the training process: `nn-nav-set.onnx` + +Reference set-based training: https://arxiv.org/abs/2401.14961 + + diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/dynamics.m b/code/nnv/examples/Submission/WiP_3d/other/NAV/dynamics.m new file mode 100644 index 0000000000..632c28d9ff --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/NAV/dynamics.m @@ -0,0 +1,11 @@ +function dx = dynamics(x,u) + +dx = [ + x(3)*cos(x(4)); + x(3)*sin(x(4)); + u(1); + u(2) +]; + +end + diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/figs/nav_point.png b/code/nnv/examples/Submission/WiP_3d/other/NAV/figs/nav_point.png new file mode 100644 index 0000000000..3aa4b19bfb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/figs/nav_point.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/figs/nav_set.png b/code/nnv/examples/Submission/WiP_3d/other/NAV/figs/nav_set.png new file mode 100644 index 0000000000..623418b6c5 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/figs/nav_set.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point.mat b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point.mat new file mode 100644 index 0000000000..b33cbd44f2 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point.png b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point.png new file mode 100644 index 0000000000..f7660557ae Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point_21.png b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point_21.png new file mode 100644 index 0000000000..4f0543e91b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_point_21.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_set.mat b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_set.mat new file mode 100644 index 0000000000..da3ba0f19f Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/nav_set.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/networks/nn-nav-point.onnx b/code/nnv/examples/Submission/WiP_3d/other/NAV/networks/nn-nav-point.onnx new file mode 100644 index 0000000000..3b4a1c4dbb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/networks/nn-nav-point.onnx differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/networks/nn-nav-set.onnx b/code/nnv/examples/Submission/WiP_3d/other/NAV/networks/nn-nav-set.onnx new file mode 100644 index 0000000000..7ca29df9db Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/NAV/networks/nn-nav-set.onnx differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/reach_point.m b/code/nnv/examples/Submission/WiP_3d/other/NAV/reach_point.m new file mode 100644 index 0000000000..0abc4e3d0b --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/NAV/reach_point.m @@ -0,0 +1,135 @@ +function rT = reach_point() + + %% Reachability analysis of NAV Benchmark + + %% Load Components + + % Load the controller + %netonnx = importNetworkFromONNX('networks/nn-nav-point.onnx', "InputDataFormats", "BC"); + netonnx = importONNXNetwork('networks/nn-nav-point.onnx', "InputDataFormats", "BC"); + + % Load plant + reachStep = 0.02; + controlPeriod = 0.2; + % plant = NonLinearODE(4, 2, @dynamics, reachStep, controlPeriod, eye(4)); + % plant.set_tensorOrder(2); + % plant.set_taylorTerms(3); + % plant.set_zonotopeOrder(100); + % plant.set_intermediateOrder(50); + + %% Reachability analysis + + % Initial set + lb = [2.9; 2.9; 0; 0]; + ub = [3.1; 3.1; 0; 0]; + init_set = Box(lb,ub); + init = init_set.partition([1 2],[50 50]); + + % Reachability options + num_steps = 20; + reachOptions.reachMethod = 'approx-star'; + + N = length(init); + disp("Verifying "+string(N)+" samples...") + + mkdir('tmp'); + parpool("Processes"); % initialize parallel process + + % Execute reachabilty analysis + t = tic; + parfor j = 1:length(init) + % Get NNV network + net = matlab2nnv(netonnx); + % Create plant + plant = NonLinearODE(4, 2, @dynamics, reachStep, controlPeriod, eye(4)); + plant.set_tensorOrder(2); + plant.set_taylorTerms(3); + plant.set_zonotopeOrder(100); + plant.set_intermediateOrder(50); + % Get initial conditions + init_set = init(j).toStar; + %reachSub = init_set; + for i = 1:num_steps + % Compute controller output set + input_set = net.reach(init_set,reachOptions); + + % Compute plant reachable set + init_set = plantReach(plant, init_set, input_set,'lin'); + end + toc(t); + parsave("tmp/reachSet"+string(j)+".mat",plant); + end + rT = toc(t); % get reach time + disp("Finished reachability...") + + % Shut Down Current Parallel Pool + poolobj = gcp('nocreate'); + delete(poolobj); + + % Save results + if is_codeocean + save('/results/logs/nav_point.mat', 'rT','-v7.3'); + else + save('nav_point.mat', 'rT','-v7.3'); + end + + + %% Visualize results + setFiles = dir('tmp/*.mat'); + + t = tic; + + f = figure; + rectangle('Position',[-0.5,-0.5,1,1],'FaceColor',[0 0.5 0 0.5],'EdgeColor','y', 'LineWidth', 0.1); % goal region + hold on; + rectangle('Position',[1,1,1,1],'FaceColor',[0.7 0 0 0.8], 'EdgeColor','r', 'LineWidth', 0.1); % obstacle + grid; + for K = 1 : length(setFiles) + if ~mod(K,50) + disp("Plotting partition "+string(K)+" ..."); + toc(t) + pause(0.01); % to ensure it prints + end + res = load("tmp/"+setFiles(K).name); + plant = res.plant; + for k=1:length(plant.cora_set) + plot(plant.cora_set{k}, [1,2], 'b', 'Unify', true); + end + end + hold on; + xlabel('x1'); + ylabel('x2'); + + disp("Finished plotting all reach sets"); + + %% Save figure + if is_codeocean + saveas(f,'/results/logs/nav_point.png'); + % exportgraphics(f,'/results/logs/nav-set.pdf', 'ContentType', 'vector'); + else + saveas(f,'nav_point_21.png'); + % exportgraphics(f,'nav-set.pdf','ContentType', 'vector'); + end + + % Save results + if is_codeocean + save('/results/logs/nav_point.mat','rT','-v7.3'); + else + save('nav_point.mat', 'rT','-v7.3'); + end + +end + +%% Helper function +function init_set = plantReach(plant,init_set,input_set,algoC) + nS = length(init_set); % based on approx-star, number of sets should be equal + ss = []; + for k=1:nS + ss =[ss plant.stepReachStar(init_set(k), input_set(k),algoC)]; + end + init_set = ss; +end + +function parsave(fname, plant) % trick to save while on parpool + save(fname, 'plant') +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/other/NAV/reach_set.m b/code/nnv/examples/Submission/WiP_3d/other/NAV/reach_set.m new file mode 100644 index 0000000000..0bc987ae97 --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/NAV/reach_set.m @@ -0,0 +1,129 @@ +function rT = reach_set() + + %% Reachability analysis of NAV Benchmark + + %% Load Components + + % Load the controller + % net = importNetworkFromONNX('networks/nn-nav-set.onnx', "InputDataFormats", "BC"); + netonnx = importONNXNetwork('networks/nn-nav-set.onnx', "InputDataFormats", "BC"); + % Load plant + reachStep = 0.02; + controlPeriod = 0.2; + % plant = NonLinearODE(4, 2, @dynamics, reachStep, controlPeriod, eye(4)); + % plant.set_tensorOrder(2); + % plant.set_taylorTerms(3); + % plant.set_zonotopeOrder(100); + % plant.set_intermediateOrder(50); + + + %% Reachability analysis + + % Initial set + lb = [2.9; 2.9; 0; 0]; + ub = [3.1; 3.1; 0; 0]; + init_set = Box(lb,ub); + init = init_set.partition([1 2],[50 50]); + + % Reachability options + num_steps = 21; + reachOptions.reachMethod = 'approx-star'; + + N = length(init); + disp("Verifying "+string(N)+" samples...") + + mkdir('temp'); + parpool("Processes"); % initialize parallel process + + % Execute reachabilty analysis + t = tic; + parfor j = 1:length(init) + % Get NNV network + net = matlab2nnv(netonnx); + % Create plant + plant = NonLinearODE(4, 2, @dynamics, reachStep, controlPeriod, eye(4)); + plant.set_tensorOrder(2); + plant.set_taylorTerms(3); + plant.set_zonotopeOrder(100); + plant.set_intermediateOrder(50); + % Get initial conditions + init_set = init(j).toStar; + %reachSub = init_set; + for i = 1:num_steps + % Compute controller output set + input_set = net.reach(init_set,reachOptions); + + % Compute plant reachable set + init_set = plantReach(plant, init_set, input_set,'lin'); + end + toc(t); + parsave("temp/reachSet"+string(j)+".mat",plant); + end + rT = toc(t); % get reach time + disp("Finished reachability...") + + % Shut Down Current Parallel Pool + poolobj = gcp('nocreate'); + delete(poolobj); + + %% Visualize results + setFiles = dir('temp/*.mat'); + + f = figure; + rectangle('Position',[-0.5,-0.5,1,1],'FaceColor',[0 0.5 0 0.5],'EdgeColor','y', 'LineWidth', 0.1); % goal region + hold on; + rectangle('Position',[1,1,1,1],'FaceColor',[0.7 0 0 0.8], 'EdgeColor','r', 'LineWidth', 0.1); % obstacle + grid; + t = tic; + for K = 1 : length(setFiles) + if ~mod(K,50) + disp("Plotting partition "+string(K)+" ..."); + toc(t) + pause(0.01); % to ensure it prints + end + res = load("temp/"+setFiles(K).name); + plant = res.plant; + % plant.get_interval_sets; + % Star.plotBoxes_2D_noFill(plant.intermediate_reachSet, 1,2,'b'); + for k=1:(length(plant.cora_set)) + plot(plant.cora_set{k}, [1,2], 'b', 'Unify', true); + end + end + hold on; + xlabel('x1'); + ylabel('x2'); + + disp("Finished plotting all reach sets"); + + + %% Save figure + if is_codeocean + saveas(f,'/results/logs/nav_set.png'); + % exportgraphics(f,'/results/logs/nav-set.pdf', 'ContentType', 'vector'); + else + saveas(f,'nav_set.png'); + % exportgraphics(f,'nav-set.pdf','ContentType', 'vector'); + end + + % Save results + if is_codeocean + save('/results/logs/nav_set.mat','rT','-v7.3'); + else + save('nav_set.mat', 'rT','-v7.3'); + end + +end + +%% Helper function +function init_set = plantReach(plant,init_set,input_set,algoC) + nS = length(init_set); % based on approx-star, number of sets should be equal + ss = []; + for k=1:nS + ss =[ss plant.stepReachStar(init_set(k), input_set(k),algoC)]; + end + init_set = ss; +end + +function parsave(fname, plant) % trick to save while on parpool + save(fname, 'plant') +end diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.h5 b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.h5 new file mode 100644 index 0000000000..6aa242509b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.h5 differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.mat b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.mat new file mode 100644 index 0000000000..1984b024c2 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.nnet b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.nnet new file mode 100644 index 0000000000..5f9e900a87 --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.nnet @@ -0,0 +1,112 @@ +// The contents of this file are licensed under the Creative Commons +// Attribution 4.0 International License: https://creativecommons.org/licenses/by/4.0/ +// Neural Network File Format by Kyle Julian, Stanford 2016 +3, 2, 1, 25 +2, 25, 25,1, 1 +0 +0 +0 +0 +0 +-0.24301, -0.18771 +-0.26350, -0.30369 +0.14823, -0.35473 +-0.32726, 0.23397 +-0.03466, -0.19298 +0.37591, 0.46209 +0.72291, 0.69352 +0.03561, 0.26642 +-0.12026, 0.21470 +-0.16872, -0.30651 +-0.43472, -0.18382 +-0.13991, -0.16939 +-0.40639, 0.06983 +0.27941, -0.20360 +-0.40475, -0.43684 +-0.34817, -0.58015 +0.07824, -0.34545 +-0.36376, -0.39287 +0.65688, 0.18937 +-0.18365, 0.15440 +-0.31044, -0.13218 +-0.07592, -0.43100 +0.42164, -0.02560 +-0.51296, -0.67853 +-0.08458, 0.15043 +-0.00419 +-0.00546 +-0.00949 +-0.00565 +-0.00241 +-0.00529 +0.00291 +-0.00454 +-0.00197 +-0.00673 +-0.00373 +-0.00559 +-0.00445 +-0.00842 +-0.00319 +0.00551 +-0.00701 +-0.00560 +0.00371 +-0.00419 +0.00527 +-0.00635 +-0.00937 +0.00512 +-0.00610 +0.06736, -0.05422, 0.14574, 0.21675, 0.22926, -0.11144, 0.56496, -0.05118, -0.02968, 0.10144, 0.23500, -0.33200, 0.13735, 0.16915, 0.32100, 0.15801, -0.18926, -0.31402, 0.03553, -0.04112, -0.21541, 0.24437, -0.26637, 0.13420, -0.00569 +0.29358, 0.33571, -0.29688, -0.02312, -0.27224, 0.12862, 0.12963, -0.15199, -0.33584, -0.17085, 0.21235, -0.33336, 0.17564, 0.05524, -0.15493, -0.25028, 0.17259, -0.20334, 0.04693, -0.12292, 0.10917, -0.00523, -0.27183, -0.07113, -0.19691 +0.02020, 0.21657, -0.23130, 0.08048, 0.16010, -0.10221, 0.17049, 0.06040, -0.20372, -0.04221, -0.06663, 0.02040, -0.08222, 0.26648, 0.19941, -0.18592, -0.23603, 0.11598, -0.19459, 0.07057, 0.04573, -0.29013, -0.00217, 0.32552, 0.31872 +0.08224, -0.30303, -0.06345, 0.13502, -0.01831, 0.28010, -0.35218, -0.06258, -0.05898, -0.32436, 0.32003, -0.30794, 0.07805, -0.02749, -0.32308, 0.14680, -0.26048, -0.11922, -0.07806, -0.25979, 0.05629, -0.15849, -0.31256, 0.33553, -0.27883 +0.30682, -0.00369, -0.12624, -0.13524, -0.23072, -0.19830, 0.51922, 0.13625, 0.30686, 0.19326, -0.28604, -0.24671, 0.05676, -0.00147, -0.12379, 0.29854, 0.27577, -0.05369, 0.41714, 0.05100, 0.10535, 0.25372, 0.15772, -0.41833, 0.12765 +-0.34396, -0.00131, 0.32711, 0.23731, 0.13850, -0.19803, 0.26637, -0.13583, -0.09831, 0.08030, 0.14047, 0.16551, -0.20448, -0.08249, 0.02232, 0.00606, 0.09334, 0.22577, 0.01190, -0.32693, -0.13444, 0.32171, 0.01188, -0.09272, 0.14908 +-0.28358, -0.23711, -0.00079, -0.20363, 0.26583, -0.25923, -0.01992, 0.30822, 0.19845, -0.02683, 0.02838, 0.07485, 0.06014, 0.13685, -0.18069, 0.31956, -0.29260, 0.01786, -0.24777, -0.13140, -0.13986, -0.16778, 0.29813, -0.27355, 0.13748 +-0.07721, -0.33846, -0.22596, 0.14351, -0.03737, 0.09296, 0.34078, 0.02503, 0.17649, 0.24141, 0.26630, -0.27206, -0.30389, 0.28129, 0.33852, 0.27713, -0.22798, 0.20975, -0.27248, 0.25465, -0.09201, 0.20717, -0.02952, 0.06362, 0.28118 +0.11761, -0.26044, 0.31300, 0.18747, -0.12191, 0.23744, -0.42517, -0.11878, -0.15642, 0.22698, 0.28689, 0.28481, -0.03482, -0.12447, -0.32566, 0.20172, -0.14285, -0.16177, -0.11876, -0.10247, 0.22981, 0.01865, -0.20637, 0.14530, -0.30398 +0.14989, -0.12266, 0.21422, 0.31554, -0.30338, 0.13623, -0.22872, -0.08666, -0.02001, -0.12250, 0.32934, 0.31798, -0.10740, -0.15264, -0.08449, -0.27235, 0.10682, -0.23104, -0.29787, -0.17958, 0.24528, -0.02062, -0.01624, 0.14056, 0.19334 +0.16698, -0.06615, 0.18133, -0.20182, -0.00508, -0.06447, -0.31271, 0.05211, -0.19533, 0.30837, 0.31486, -0.16648, 0.08478, 0.03339, -0.09702, 0.16260, -0.18307, 0.22876, 0.15843, 0.14073, -0.13829, -0.29730, 0.11204, 0.34402, -0.17949 +0.34478, 0.02179, -0.06910, 0.22159, 0.35296, -0.34437, -0.17796, -0.13641, -0.01192, -0.01154, -0.27147, 0.03556, -0.28606, -0.33980, -0.03341, 0.01746, -0.11872, 0.19611, -0.13309, 0.31245, -0.34939, -0.07927, -0.02675, 0.32078, 0.31505 +0.23845, 0.07345, 0.07717, 0.09820, -0.20820, -0.14211, 0.05102, -0.08047, 0.20787, 0.26361, -0.00489, 0.04994, 0.05208, 0.00991, 0.18136, -0.19816, -0.13177, 0.13224, 0.22872, 0.17221, 0.02832, 0.18470, -0.14746, 0.00152, 0.24030 +-0.21038, -0.33311, 0.05829, -0.11248, -0.39031, 0.24018, 0.31533, 0.31591, 0.21842, 0.11106, 0.00419, -0.10125, -0.04118, -0.18811, -0.32428, -0.03926, 0.18926, -0.12291, 0.15375, 0.02808, -0.14635, -0.08132, -0.00497, -0.03953, 0.24568 +0.14025, 0.00668, -0.30074, -0.07440, -0.33066, -0.06600, 0.01166, 0.24954, -0.22035, -0.31518, -0.14028, -0.27341, -0.21291, -0.30773, -0.13274, 0.15092, 0.31991, 0.04984, 0.27492, 0.32448, -0.28637, -0.26806, 0.04600, -0.27472, -0.32435 +-0.34466, -0.28169, 0.16045, -0.29125, -0.30502, 0.21993, 0.00177, 0.00563, -0.15463, 0.06853, -0.13516, 0.02662, 0.18983, -0.22146, -0.14441, 0.06328, -0.05649, 0.30135, -0.23290, -0.21104, 0.01558, -0.07386, 0.31682, -0.33285, -0.10461 +0.26429, 0.11103, -0.28125, 0.20282, 0.09487, 0.00015, 0.17683, -0.19715, 0.17583, 0.14676, -0.26661, -0.21553, -0.23419, 0.12648, 0.02281, -0.34281, -0.14238, -0.11536, -0.29904, 0.24350, -0.06235, 0.04659, -0.08648, 0.21312, 0.23545 +-0.13509, -0.09735, 0.06823, -0.22581, -0.17998, 0.15447, -0.16209, 0.17372, -0.27225, -0.00504, 0.14097, -0.21481, -0.14895, 0.27991, -0.32028, 0.00831, -0.20118, -0.19568, -0.05819, -0.29290, -0.11851, 0.14796, -0.04694, 0.23116, -0.20026 +0.30448, -0.30411, 0.17794, -0.19358, 0.19374, 0.24929, 0.52103, 0.14052, 0.12245, 0.08575, 0.13888, -0.30510, 0.25353, -0.14709, 0.02053, -0.02730, -0.10952, -0.24077, 0.28567, 0.01464, 0.13864, 0.19052, 0.00932, -0.39407, -0.32367 +0.33511, -0.31479, 0.21451, 0.22260, 0.14512, -0.33519, -0.19355, 0.24852, 0.23520, -0.11799, -0.22526, -0.26930, 0.09369, -0.23061, 0.31694, 0.23137, 0.18504, -0.04852, -0.24657, -0.14619, -0.26419, 0.02569, -0.16507, -0.25164, 0.31506 +-0.10569, 0.00200, 0.23895, 0.12868, -0.01618, -0.01120, 0.09951, 0.01289, 0.19420, 0.14726, -0.11491, -0.08230, -0.06727, -0.06584, -0.23567, -0.18129, 0.12943, -0.14550, -0.19815, 0.18831, -0.32903, -0.05185, 0.20706, -0.18634, 0.25237 +0.08708, 0.02897, -0.32331, -0.19755, 0.14713, -0.24031, -0.39770, 0.07247, 0.29314, 0.27825, 0.34283, -0.17500, 0.14072, 0.17569, 0.06979, 0.11654, 0.08414, 0.16877, -0.38219, -0.16694, 0.01645, 0.18350, -0.10728, 0.36223, 0.32663 +0.24512, 0.04184, -0.31656, 0.19614, 0.17455, 0.00337, 0.03893, -0.33625, 0.01757, 0.03453, -0.00278, -0.00553, -0.15259, 0.31461, -0.29216, 0.22329, 0.11153, -0.27728, -0.10089, 0.02872, 0.25985, -0.03283, 0.27549, -0.28317, -0.04284 +0.30643, 0.26365, 0.07507, 0.07706, -0.14626, 0.15628, 0.31865, -0.16474, -0.15439, -0.02804, 0.01072, 0.31345, -0.25433, 0.33175, -0.09756, -0.20805, 0.02763, -0.28118, 0.26005, -0.12078, 0.20447, -0.31157, -0.11742, 0.07141, 0.18978 +-0.34193, -0.20700, 0.07865, -0.01669, 0.04845, -0.06684, -0.49604, -0.02675, 0.07013, -0.15788, 0.13861, -0.15236, -0.06848, -0.11084, 0.33909, 0.27674, -0.29966, 0.22394, 0.20092, -0.04978, 0.17215, 0.07136, -0.24517, 0.19800, -0.30192 +0.00368 +-0.00516 +-0.00375 +0.00510 +0.00306 +-0.00552 +-0.00378 +-0.00580 +0.00648 +-0.00696 +-0.00641 +0.00060 +-0.00563 +-0.00589 +0.00189 +0.00000 +-0.00396 +-0.00536 +0.00366 +-0.00466 +-0.00563 +0.01047 +-0.00733 +0.00197 +0.00208 +-0.39757, -0.47563, -0.13953, 0.32202, -0.25362, 0.32063, 0.01388, -0.12681, 0.43118, -0.33472, 0.09568, 0.00016, 0.45845, -0.01793, -0.00473, -0.30083, -0.16590, 0.32844, -0.36319, -0.33402, 0.45242, 0.34286, 0.05281, -0.12741, 0.40624 +-0.00490 diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.onnx b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.onnx new file mode 100644 index 0000000000..25837086f4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/controller_single_pendulum.onnx differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/dynamics_sp.m b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/dynamics_sp.m new file mode 100644 index 0000000000..073d2b6749 --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/dynamics_sp.m @@ -0,0 +1,12 @@ +function [dx] = dynamics_sp(x,a) +% Ex_single_pendulum +% constants as per the documentation l=0.5, m=0.5; g= 1; c=0; +% description of the pararameters in these equations can be found here +% https://github.com/amaleki2/benchmark_closedloop_verification/blob/master/AINNC_benchmark.pdf + +l=0.5; m=0.5; g= 1; c= 0; + +dx(1,1) = x(2); +dx(2,1) = g/l * sin(x(1)) + (a - c*x(2))/(m*l^2); +% dx(3,1) = 20; +dx(3,1) = 1; \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/figs/singlePendulum.png b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/figs/singlePendulum.png new file mode 100644 index 0000000000..a20c8467f5 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/figs/singlePendulum.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/reach.m b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/reach.m new file mode 100644 index 0000000000..5380da5ed1 --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/reach.m @@ -0,0 +1,117 @@ +% function rT = reach() + + %% Reachability analysis of Single Pendulum Benchmark + + %% Load Components + + % Load the controller + % net = load_NN_from_mat('controller_single_pendulum.mat'); + % Load plant + reachStep = 0.01; + controlPeriod = 0.05; + % plant = NonLinearODE(3,1,@dynamics_sp, reachStep, controlPeriod, eye(3)); + % plant.set_tensorOrder(2); + + + %% Reachability analysis + + % Initial set + lb = [1.0; 0.0; 0]; + ub = [1.175; 0.2; 0]; % original bounds + init_set = Box(lb,ub); + init = init_set.partition([1 2],[35 40]); + % init_set = Star(lb,ub); + + num_steps = 13; + reachOptions.reachMethod = 'approx-star'; + + N = length(init); + disp("Verifying "+string(N)+" samples...") + + mkdir('temp'); + parpool("Processes"); % initialize parallel process + + % Execute reachabilty analysis + t = tic; + parfor j = 1:length(init) + % Create NNCS for each process + net = load_NN_from_mat('controller_single_pendulum.mat'); + plant = NonLinearODE(3,1,@dynamics_sp, reachStep, controlPeriod, eye(3)); + plant.set_tensorOrder(2); + % Get initial conditions + init_set = init(j).toStar; + for i = 1:num_steps + % Compute controller output set + init_set_s = init_set.affineMap([1 0 0;0 1 0],[]); + input_set = net.reach(init_set_s,reachOptions); + + % Compute plant reachable set + init_set = plantReach(plant, init_set, input_set,'lin'); + end + toc(t); + parsave("temp/reachSet"+string(j)+".mat",plant); + end + rT = toc(t); % get reach time + disp("Finished reachability...") + + % Shut Down Current Parallel Pool + poolobj = gcp('nocreate'); + delete(poolobj); + + %% Visualize results + setFiles = dir('temp/*.mat'); + + t = tic; + f = figure; + hold on; + rectangle('Position',[0.5,1,1,1],'FaceColor',[1 0 0 0.5],'EdgeColor','r', 'LineWidth',0.1) + grid; + for K = 1 : length(setFiles) + if ~mod(K,50) + disp("Plotting partition "+string(K)+" ..."); + end + res = load("temp/"+setFiles(K).name); + plant = res.plant; + for k=1:length(plant.cora_set) + plot(plant.cora_set{k}, [3,1], 'b', 'Unify', true); % this seems to be faster + end + end + xlim([0.1 0.6]) + ylim([0.85 1.25]) + xlabel('Time (s)'); + ylabel('\theta'); + toc(t); + + disp("Finished plotting all reach sets"); + + %% Save figure + if is_codeocean + % exportgraphics(f,'/results/logs/singlePendulum.pdf', 'ContentType', 'vector'); % takes too long + saveas(f,'/results/logs/singlePendulum.png'); + else + saveas(f,'singlePendulum.png'); + % exportgraphics(f,'singlePendulum.pdf','ContentType', 'vector'); + end + + % Save results + if is_codeocean + save('/results/logs/single_pendulum.mat', 'rT','-v7.3'); + else + save('single_pendulum.mat', 'rT', '-v7.3'); + end + +% end + +%% Helper function +function init_set = plantReach(plant,init_set,input_set,algoC) + nS = length(init_set); % based on approx-star, number of sets should be equal + ss = []; + for k=1:nS + ss =[ss plant.stepReachStar(init_set(k), input_set(k),algoC)]; + end + init_set = ss; +end + +function parsave(fname, plant) % trick to save while on parpool + save(fname, 'plant') +end diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/singlePendulum.png b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/singlePendulum.png new file mode 100644 index 0000000000..0d60b0ca40 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/singlePendulum.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/single_pendulum.mat b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/single_pendulum.mat new file mode 100644 index 0000000000..cacc0d90f1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/single_pendulum.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/specifications.txt b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/specifications.txt new file mode 100644 index 0000000000..4d962f99ca --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/other/Single_Pendulum/specifications.txt @@ -0,0 +1,12 @@ +Initial states: + +x1 = [1.0, 1.2] +x2 = [0.0, 0.2] + +The controller timestep (control period) = 0.05 (seconds). + +Property: + +x1 in [0.0,1.0] for 0.5 <= t <=1 + +Refer to this for more details: https://github.com/amaleki2/benchmark_closedloop_verification/blob/master/AINNC_benchmark.pdf diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_10.png index df077e7e32..9aa22a8e1b 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_100.png deleted file mode 100644 index 0f6c9bdacf..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_1000.png deleted file mode 100644 index 0b793b9ead..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_2.png index b386d43d6b..8f67920907 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_25.png new file mode 100644 index 0000000000..70c5c7c81b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_4.png deleted file mode 100644 index c03e3540b6..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_5.png new file mode 100644 index 0000000000..e64ee892d1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_50.png deleted file mode 100644 index 89d618c711..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_500.png deleted file mode 100644 index 22b23ccb81..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_10.png index 4cceb477d4..22579a8390 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_100.png deleted file mode 100644 index f526c90337..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_1000.png deleted file mode 100644 index 42bf7e88a6..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_2.png index d18a31a881..a2522099c6 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_25.png new file mode 100644 index 0000000000..2e57745178 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_4.png deleted file mode 100644 index 6eddcba873..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_5.png new file mode 100644 index 0000000000..7a623e26b6 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_50.png deleted file mode 100644 index 9d62612208..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_500.png deleted file mode 100644 index 61ee207fbd..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_10.png index 499501b223..5a94ebed73 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_100.png deleted file mode 100644 index b6dcc81883..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_1000.png deleted file mode 100644 index a5a17369e1..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_2.png index ca9cc01148..a6a6ae0047 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_25.png new file mode 100644 index 0000000000..7e0a1a70e1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_4.png deleted file mode 100644 index c4af97349f..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_5.png new file mode 100644 index 0000000000..9e1fdb9a5e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_50.png deleted file mode 100644 index 1af3bc1b57..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_500.png deleted file mode 100644 index d467bc2f5b..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_10.png index acf2f1c706..e10ece93ba 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_100.png deleted file mode 100644 index a915449c7b..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_1000.png deleted file mode 100644 index 735af31db4..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_2.png index 739b3b6714..01de59df75 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_25.png new file mode 100644 index 0000000000..80219bbfec Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_4.png deleted file mode 100644 index 851548f758..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_5.png new file mode 100644 index 0000000000..92b56b88c1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_50.png deleted file mode 100644 index 4541a6970d..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_500.png deleted file mode 100644 index 0a42e6d828..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_10.png index 271105e06f..c6b23e64bc 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_100.png deleted file mode 100644 index 65a0183cb4..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_1000.png deleted file mode 100644 index 8477355511..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_2.png index 2e72d1145c..02ccb7262f 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_25.png new file mode 100644 index 0000000000..212fa99436 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_4.png deleted file mode 100644 index 322b2caf7e..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_5.png new file mode 100644 index 0000000000..8c5fe3a944 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_50.png deleted file mode 100644 index 9c7011e489..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_500.png deleted file mode 100644 index 19503b28dc..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_10.png index 55602353ae..dfbe88ae00 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_100.png deleted file mode 100644 index c62bbf30e0..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_1000.png deleted file mode 100644 index bc65efe59d..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_2.png index c7fab4ac1b..8b784d0960 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_25.png new file mode 100644 index 0000000000..46984e2493 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_4.png deleted file mode 100644 index 7ce085659c..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_5.png new file mode 100644 index 0000000000..1f1d8aa4b5 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_50.png deleted file mode 100644 index 8999644d63..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_500.png deleted file mode 100644 index 0d802d2f11..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_10.png index ac4de7d928..51a5d20538 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_100.png deleted file mode 100644 index 293b67c821..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_1000.png deleted file mode 100644 index 4274507c24..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_2.png index 09f3430d5e..d4326554bd 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_25.png new file mode 100644 index 0000000000..30827e2756 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_4.png deleted file mode 100644 index ba5b24358b..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_5.png new file mode 100644 index 0000000000..749dfc69c3 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_50.png deleted file mode 100644 index 74a5f82425..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_500.png deleted file mode 100644 index 2214c42dc3..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_10.png index 035baf86fb..986ddd26e9 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_100.png deleted file mode 100644 index a812a42b50..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_1000.png deleted file mode 100644 index 403c0d397d..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_2.png index a5e0e06487..52f548afe6 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_25.png new file mode 100644 index 0000000000..8cfc164ee6 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_4.png deleted file mode 100644 index 35613f269c..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_5.png new file mode 100644 index 0000000000..607e92cee8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_50.png deleted file mode 100644 index 22b30b6904..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_500.png deleted file mode 100644 index b0378e9bfc..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_10.png index 568953bcc1..b835b6e668 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_100.png deleted file mode 100644 index 746dfc9a2a..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_1000.png deleted file mode 100644 index db15d8ca96..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_2.png index af9c970c79..71f941ea2b 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_25.png new file mode 100644 index 0000000000..2780fb9f6c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_4.png deleted file mode 100644 index 8bc0d31b84..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_5.png new file mode 100644 index 0000000000..4ffe8e4082 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_50.png deleted file mode 100644 index 194246c3e3..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_500.png deleted file mode 100644 index cef56d2516..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_10.png index d130628be2..76ece678ee 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_100.png deleted file mode 100644 index 6fdc524d04..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_1000.png deleted file mode 100644 index 6d65c70912..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_2.png index 0da057ca82..afa389dbce 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_25.png new file mode 100644 index 0000000000..06c3011742 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_4.png deleted file mode 100644 index 655ccbe174..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_5.png new file mode 100644 index 0000000000..f06b7e9bd9 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_50.png deleted file mode 100644 index 4a3cca7732..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_500.png deleted file mode 100644 index d0fb7e22fc..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_10.png index aff9e11133..00cdaee715 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_100.png deleted file mode 100644 index af8bf367a2..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_1000.png deleted file mode 100644 index 186064d6d3..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_2.png index fbc46472fb..412fc05cfa 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_25.png new file mode 100644 index 0000000000..f3931b5316 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_4.png deleted file mode 100644 index ee5ff70802..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_5.png new file mode 100644 index 0000000000..00cdaee715 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_50.png deleted file mode 100644 index 5430f60b6c..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_500.png deleted file mode 100644 index e7757b5fbb..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_10.png index 68d0ce9dbf..6aa7e71b95 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_100.png deleted file mode 100644 index 7e2e838879..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_1000.png deleted file mode 100644 index 9253d08f4a..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_2.png index cba4a42beb..f996bd48cd 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_25.png new file mode 100644 index 0000000000..baff249fae Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_4.png deleted file mode 100644 index 34e1c311d8..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_5.png new file mode 100644 index 0000000000..eb0b1e7301 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_50.png deleted file mode 100644 index a05f027ac8..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_500.png deleted file mode 100644 index ab290f4712..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_10.png index c6143b5033..7bd73e358b 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_100.png deleted file mode 100644 index 76e4230df8..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_1000.png deleted file mode 100644 index 2c061bf6e1..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_2.png index c1b0488b5e..ea27ac09dc 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_25.png new file mode 100644 index 0000000000..028a2c3f03 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_4.png deleted file mode 100644 index 0fb6109b40..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_5.png new file mode 100644 index 0000000000..5e651982d4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_50.png deleted file mode 100644 index 183292c5b9..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_500.png deleted file mode 100644 index bcc410bc4a..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_10.png index 4e81591e08..331f650ded 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_100.png deleted file mode 100644 index eb9ecf7749..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_1000.png deleted file mode 100644 index 9f6a343798..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_2.png index eef06f7cf8..dfdd157311 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_25.png new file mode 100644 index 0000000000..ed1360a229 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_4.png deleted file mode 100644 index bdc7d654e7..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_5.png new file mode 100644 index 0000000000..de3797f8cb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_50.png deleted file mode 100644 index 6ba03bbe56..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_500.png deleted file mode 100644 index d1804d018a..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_10.png index d34e16fa25..df396d9cd6 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_100.png deleted file mode 100644 index b0c7849f33..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_1000.png deleted file mode 100644 index 047c5243ad..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_2.png index 83c5d26374..fb8840daeb 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_25.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_25.png new file mode 100644 index 0000000000..de0f4bca2c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_4.png deleted file mode 100644 index a86d90b0a9..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_5.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_5.png new file mode 100644 index 0000000000..acab4eb6dd Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_50.png deleted file mode 100644 index 4e2e92ccd1..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_500.png deleted file mode 100644 index d6cf121774..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_10.png index eb979c7240..3dbf0caa19 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_10.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_100.png deleted file mode 100644 index a8c3bc062e..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_100.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_1000.png deleted file mode 100644 index 92132bce63..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_1000.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_2.png index d08a7be066..de0f4bca2c 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_2.png and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_25.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_25.png new file mode 100644 index 0000000000..aa6f2b99f5 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_25.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_4.png deleted file mode 100644 index 896af079be..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_4.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_5.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_5.png new file mode 100644 index 0000000000..ea4e6c71d4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_5.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_50.png deleted file mode 100644 index 81fc7e7c49..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_50.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_500.png deleted file mode 100644 index a8c3bc062e..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_500.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_100.mat new file mode 100644 index 0000000000..2459cf1eb1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_1000.mat new file mode 100644 index 0000000000..ff8decfa71 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_50.mat new file mode 100644 index 0000000000..cefb924db9 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_500.mat new file mode 100644 index 0000000000..b5f3204632 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_add_1_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_100.mat new file mode 100644 index 0000000000..fbc625d384 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_1000.mat new file mode 100644 index 0000000000..d5f127a3b2 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_50.mat new file mode 100644 index 0000000000..db805a4fb0 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_500.mat new file mode 100644 index 0000000000..60bf807068 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_adrenal_remove_1_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_100.mat index 26e2dcae93..b6fefa4346 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_1000.mat index 8c2eea125c..613c433d31 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_50.mat index b0dab4ee35..4a03c009ce 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_500.mat index cd150cbe1c..24ca3f7117 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_100.mat new file mode 100644 index 0000000000..0229a57188 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_1000.mat new file mode 100644 index 0000000000..1901e9f298 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_50.mat new file mode 100644 index 0000000000..f6ece8cee7 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_500.mat new file mode 100644 index 0000000000..416099dd2b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_100.mat index 893640b7f0..348a52fd79 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_1000.mat index 4e473cf59f..366cc04694 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_50.mat index 33cd671bab..bdb2403368 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_500.mat index 0e75d72ab4..1002093fcd 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_100.mat deleted file mode 100644 index 679916bb1e..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_1000.mat deleted file mode 100644 index 11d24bad5d..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_50.mat deleted file mode 100644 index 112a782a14..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_500.mat deleted file mode 100644 index 9872db73ba..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_100.mat new file mode 100644 index 0000000000..ea71a7812d Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_1000.mat new file mode 100644 index 0000000000..cafb8b8d31 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_50.mat new file mode 100644 index 0000000000..206e632924 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_500.mat new file mode 100644 index 0000000000..4a0b99c968 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_bright_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_100.mat index 7ead3b80d3..1b1ec774d1 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_1000.mat index 32593dbbb1..7ac5fe397c 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_50.mat index 51eff67057..0e42d87614 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_500.mat index 80a6cabd3f..e36f42ca0f 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_100.mat new file mode 100644 index 0000000000..4332a8c7eb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_1000.mat new file mode 100644 index 0000000000..97b6b8fd12 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_50.mat new file mode 100644 index 0000000000..79d7808f58 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_500.mat new file mode 100644 index 0000000000..95da56a1fd Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_100.mat index 6ee1ae6761..87a1123b52 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_1000.mat index a2e76eb4c5..48c25efdc6 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_50.mat index ea9369c995..62059a5987 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_500.mat index 888fa86bcf..dbec3b45c5 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_100.mat deleted file mode 100644 index eecf1d6e3c..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_1000.mat deleted file mode 100644 index 3e1953c4c3..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_50.mat deleted file mode 100644 index 853b18860d..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_500.mat deleted file mode 100644 index ea8d88f9b4..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_100.mat new file mode 100644 index 0000000000..ae55f4a731 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_1000.mat new file mode 100644 index 0000000000..881e2c0712 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_50.mat new file mode 100644 index 0000000000..9aa50f5be8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_500.mat new file mode 100644 index 0000000000..e626c6ba19 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_fracture_dark_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_100.mat index e863b434b7..9fdc388d3f 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_1000.mat index beb09ebccf..7ff284f780 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_50.mat index a9b13f0d7e..b00bf15d32 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_500.mat index f315c9e914..9751f53da9 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_100.mat new file mode 100644 index 0000000000..75f8c82d97 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_1000.mat new file mode 100644 index 0000000000..8594f722f1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_50.mat new file mode 100644 index 0000000000..c3a5df3cf8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_500.mat new file mode 100644 index 0000000000..2048c51e9c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_100.mat index 039cf08e65..1dcb619c29 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_1000.mat index 80f8030bab..a53eb48562 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_50.mat index 87fcc3821e..f8b55d2c4f 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_500.mat index da0dca648e..89e4ec6d24 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_100.mat deleted file mode 100644 index 35ff0b2c0d..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_1000.mat deleted file mode 100644 index 11f503f0bf..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_50.mat deleted file mode 100644 index b82999011a..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_500.mat deleted file mode 100644 index 919ffbd278..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_100.mat new file mode 100644 index 0000000000..2d8fdae8fa Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_1000.mat new file mode 100644 index 0000000000..760cde93eb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_50.mat new file mode 100644 index 0000000000..9e71253493 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_500.mat new file mode 100644 index 0000000000..d3b0a37316 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_bright_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_100.mat index 3ff3f5c430..d6fc28786a 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_1000.mat index 3233b8a9b4..ff097559fb 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_50.mat index 1c16eddd07..c4d4b44f43 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_500.mat index c58d1483ca..85a07cf00f 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_100.mat new file mode 100644 index 0000000000..d8eeac8156 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_1000.mat new file mode 100644 index 0000000000..7a3275bbcd Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_50.mat new file mode 100644 index 0000000000..98be8c681a Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_500.mat new file mode 100644 index 0000000000..38ecd0f2c4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_100.mat index 6f5ba42ca9..b2a52fc1f6 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_1000.mat index 4a4986558c..3f76f72887 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_50.mat index 67632b4f0f..f100e1fe3d 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_500.mat index 8f0e14e382..9736d3b029 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_100.mat deleted file mode 100644 index 387a9b7045..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_1000.mat deleted file mode 100644 index 2246c099cc..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_50.mat deleted file mode 100644 index ffe9838e27..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_500.mat deleted file mode 100644 index 866a09c975..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_100.mat new file mode 100644 index 0000000000..977e7b4306 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_1000.mat new file mode 100644 index 0000000000..90e419a081 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_50.mat new file mode 100644 index 0000000000..7bc94d3f8c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_500.mat new file mode 100644 index 0000000000..64be20d25e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_nodule_dark_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_100.mat index ec5fb690c3..94b5460684 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_1000.mat index 5c122acba8..52fbd2c37b 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_50.mat index 698025a3d7..c8fa4aa19a 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_500.mat index c31b253b5f..0db1569541 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_100.mat new file mode 100644 index 0000000000..34038455ca Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_1000.mat new file mode 100644 index 0000000000..336ca9ca80 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_50.mat new file mode 100644 index 0000000000..b237298403 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_500.mat new file mode 100644 index 0000000000..bff5c0c560 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_100.mat index a63e87ef55..f42a4baba0 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_1000.mat index ae40a94b1b..f7bb6636ce 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_50.mat index 0674ed2e5b..746923be32 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_500.mat index 55b0cb9b39..bdcc3def6a 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_100.mat deleted file mode 100644 index baa488337b..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_1000.mat deleted file mode 100644 index 5468593f38..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_50.mat deleted file mode 100644 index 8977313d6e..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_500.mat deleted file mode 100644 index df8c054341..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_100.mat new file mode 100644 index 0000000000..b77ffe6cba Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_1000.mat new file mode 100644 index 0000000000..09c6bb9d44 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_50.mat new file mode 100644 index 0000000000..57ff0a9b7c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_500.mat new file mode 100644 index 0000000000..a53e7e3962 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_bright_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_100.mat index 5c6ef15057..333856644e 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_1000.mat index ca603fa599..2d0a31c212 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_50.mat index 9ee3033bac..0a56d71c38 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_500.mat index 503a095ac3..7af669d99a 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_100.mat new file mode 100644 index 0000000000..f9e721a51c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_1000.mat new file mode 100644 index 0000000000..88c6914dd0 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_50.mat new file mode 100644 index 0000000000..5a5c7c6a61 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_500.mat new file mode 100644 index 0000000000..e0c08fc8ce Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_100.mat index 5d70e248d2..b00f067a27 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_1000.mat index 1587bfdbf9..557c1abc22 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_50.mat index 531eed5132..dd38e1d2c7 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_500.mat index e16bcc3221..bc12a35010 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_100.mat deleted file mode 100644 index f304bccb0e..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_1000.mat deleted file mode 100644 index 22030210ad..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_50.mat deleted file mode 100644 index 3e26e5dc92..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_500.mat deleted file mode 100644 index b12dfb1135..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_100.mat new file mode 100644 index 0000000000..074360ce6a Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_1000.mat new file mode 100644 index 0000000000..47780685fa Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_50.mat new file mode 100644 index 0000000000..9c731cd9ba Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_500.mat new file mode 100644 index 0000000000..a22c9b1a75 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_organ_dark_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_100.mat index dc7b614069..92293633c3 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_1000.mat index 555c3347b6..c61317d96c 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_50.mat index 3751dab0f6..417ab376ae 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_500.mat index 1973a9c333..92e7f55507 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_100.mat new file mode 100644 index 0000000000..a847acfd05 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_1000.mat new file mode 100644 index 0000000000..b283fa7549 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_50.mat new file mode 100644 index 0000000000..d4ac36856f Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_500.mat new file mode 100644 index 0000000000..7d261f3b40 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_100.mat index 03677ff462..e9d52c5c99 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_1000.mat index 850aff5e62..28ea403874 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_50.mat index 85e4333c8d..d25313ff6c 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_500.mat index 3451c9bd51..597966f13d 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_100.mat deleted file mode 100644 index c37fab5fb3..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_1000.mat deleted file mode 100644 index beb6ef55ab..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_50.mat deleted file mode 100644 index e611315295..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_500.mat deleted file mode 100644 index 9189d04bf0..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_100.mat new file mode 100644 index 0000000000..4240b56453 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_1000.mat new file mode 100644 index 0000000000..cf421bb455 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_50.mat new file mode 100644 index 0000000000..787eee38df Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_500.mat new file mode 100644 index 0000000000..3860c4198e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_bright_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_100.mat index 50be9a5d59..0136418c5d 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_1000.mat index 414f5bb1dd..d2d2c050ae 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_50.mat index d62684f5b4..1c82168b4a 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_500.mat index af014b62fa..61303683d4 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_10_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_100.mat new file mode 100644 index 0000000000..137fe9f523 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_1000.mat new file mode 100644 index 0000000000..9077ffefd1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_50.mat new file mode 100644 index 0000000000..7653aa8c7f Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_500.mat new file mode 100644 index 0000000000..f76fcca1e8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_25_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_100.mat index 39d05f7e6e..c4d3f9d71b 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_100.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_1000.mat index 4737cc0e41..bbdcc2b3ef 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_1000.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_50.mat index 6f456bdc2e..0c7d0d8f71 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_50.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_500.mat index 5bf5037df3..9b884fa6f8 100644 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_500.mat and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_2_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_100.mat deleted file mode 100644 index 92dba55231..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_100.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_1000.mat deleted file mode 100644 index 3b653f7918..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_1000.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_50.mat deleted file mode 100644 index 40c0e9e5a0..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_50.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_500.mat deleted file mode 100644 index 26be2b8dc6..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_4_500.mat and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_100.mat new file mode 100644 index 0000000000..49ee0969e2 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_1000.mat new file mode 100644 index 0000000000..1f83ca12a0 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_50.mat new file mode 100644 index 0000000000..54aad772aa Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_500.mat new file mode 100644 index 0000000000..b8c370a41d Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_synapse_dark_5_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_100.mat new file mode 100644 index 0000000000..43c0b3fed6 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_1000.mat new file mode 100644 index 0000000000..290144fa5e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_50.mat new file mode 100644 index 0000000000..dcdb705749 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_500.mat new file mode 100644 index 0000000000..08a182f683 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_add_1_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_100.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_100.mat new file mode 100644 index 0000000000..d9bd3bdf29 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_100.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_1000.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_1000.mat new file mode 100644 index 0000000000..e4a21a6ac8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_1000.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_50.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_50.mat new file mode 100644 index 0000000000..86c5046108 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_50.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_500.mat b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_500.mat new file mode 100644 index 0000000000..37cf6442c4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/results/verification_vessel_remove_1_500.mat differ diff --git a/code/nnv/examples/Submission/WiP_3d/run_all.m b/code/nnv/examples/Submission/WiP_3d/run_all.m index e87c0011fa..6ff562bd3b 100644 --- a/code/nnv/examples/Submission/WiP_3d/run_all.m +++ b/code/nnv/examples/Submission/WiP_3d/run_all.m @@ -1,9 +1,12 @@ %% Shape only data (3d) +disp("... adrenal ..."); verify_adrenal; +disp("... vessel ..."); verify_vessel; + %% Volume data (general 3D) @@ -21,3 +24,8 @@ disp("... synapse ...") verify_synapse; + + +disp("... Creating plots..."); +visualize_results_gen; +visualize_results_shape; diff --git a/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m b/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m index f5061dd62c..de5e9f0fd5 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m @@ -20,7 +20,7 @@ net = matlab2nnv(net); % select volumes to verify -N = 24; % even for numCores +N = 50; inputs = single(test_images(:,:,:,:,1:N)); targets = single(test_labels(1:N)); @@ -31,6 +31,13 @@ reachOptions.lp_solver = "gurobi"; +% Study variables +advType = ["add", "remove"]; +maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels +% maxpixels = 1000; +epsilon = 1; % ep / 255 + + %% Verification analysis for a=advType for mp=maxpixels @@ -41,12 +48,7 @@ % 2) Create adversarial attack adv_attack = struct; - adv_attack.Name = a; % bright or dark - if strcmp(a, "bright") - adv_attack.threshold = threshold(1); % perturb pixels below this value - else - adv_attack.threshold = threshold(2); % perturb pixels below this value - end + adv_attack.Name = a; % add or remove adv_attack.max_pixels = mp; % Max number of pixels to modify from input image adv_attack.noise_de = ep/255; % disturbance (noise) on pixels @@ -54,7 +56,7 @@ for i=1:N img = squeeze(inputs(:,:,:,:,i)); target = targets(i); - results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); + results(i,:) = verify_instance_shape(net, img, target, adv_attack, reachOptions); end % 4) % save results diff --git a/code/nnv/examples/Submission/WiP_3d/verify_fracture.m b/code/nnv/examples/Submission/WiP_3d/verify_fracture.m index c66a8c60f3..d629615012 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_fracture.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_fracture.m @@ -20,7 +20,7 @@ net = matlab2nnv(net); % select volumes to verify -N = 24; % even for numCores +N = 50; % even for numCores inputs = single(test_images(:,:,:,:,1:N)); targets = single(test_labels(1:N)); @@ -33,7 +33,7 @@ % Study variables advType = ["bright", "dark"]; maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels -epsilon = [2, 4, 10]; % ep / 255 +epsilon = [2, 5, 10, 25]; % ep / 255 threshold = [100; 150]; % bright ; dark %% Verification analysis diff --git a/code/nnv/examples/Submission/WiP_3d/verify_nodule.m b/code/nnv/examples/Submission/WiP_3d/verify_nodule.m index d2b09d0778..0e6e7f6af0 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_nodule.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_nodule.m @@ -20,7 +20,7 @@ net = matlab2nnv(net); % select volumes to verify -N = 24; % even for numCores +N = 50; % even for numCores inputs = single(test_images(:,:,:,:,1:N)); targets = single(test_labels(1:N)); @@ -34,7 +34,7 @@ % Study variables advType = ["bright", "dark"]; maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels -epsilon = [2, 4, 10]; % ep / 255 +epsilon = [2, 5, 10, 25]; % ep / 255 threshold = [100; 150]; % bright ; dark %% Verification analysis diff --git a/code/nnv/examples/Submission/WiP_3d/verify_organ.m b/code/nnv/examples/Submission/WiP_3d/verify_organ.m index c05164bf48..563f5ca2c4 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_organ.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_organ.m @@ -20,7 +20,7 @@ net = matlab2nnv(net); % select volumes to verify -N = 24; +N = 50; inputs = single(test_images(:,:,:,:,1:N)); targets = single(test_labels(1:N)); @@ -33,7 +33,7 @@ % Study variables advType = ["bright", "dark"]; maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels -epsilon = [2, 4, 10]; % ep / 255 +epsilon = [2, 5, 10, 25]; % ep / 255 threshold = [100; 150]; % bright ; dark %% Verification analysis diff --git a/code/nnv/examples/Submission/WiP_3d/verify_synapse.m b/code/nnv/examples/Submission/WiP_3d/verify_synapse.m index 76d0160cdd..e09474ffb5 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_synapse.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_synapse.m @@ -20,7 +20,7 @@ net = matlab2nnv(net); % select volumes to verify -N = 24; % even for numCores +N = 50; % even for numCores inputs = single(test_images(:,:,:,:,1:N)); targets = single(test_labels(1:N)); @@ -33,7 +33,7 @@ % Study variables advType = ["bright", "dark"]; maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels -epsilon = [2, 4, 10]; % ep / 255 +epsilon = [2, 5, 10, 25]; % ep / 255 threshold = [100; 150]; % bright ; dark %% Verification analysis diff --git a/code/nnv/examples/Submission/WiP_3d/verify_vessel.m b/code/nnv/examples/Submission/WiP_3d/verify_vessel.m index d8775e01e2..738fa2cf45 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_vessel.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_vessel.m @@ -20,7 +20,7 @@ net = matlab2nnv(net); % select volumes to verify -N = 24; % even for numCores +N = 50; % even for numCores inputs = single(test_images(:,:,:,:,1:N)); targets = single(test_labels(1:N)); @@ -30,139 +30,37 @@ reachOptions.relaxFactor = 0.95; reachOptions.lp_solver = "gurobi"; -%% Attack 1 - -% adv_attack = struct; -%??????? - -% results = zeros(N,2); % verification result, time - -% verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end - -% save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 2 - -adv_attack = struct; - -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 3 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 4 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 5 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 6 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 7 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 8 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_vessel_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); +% Study variables +advType = ["add", "remove"]; +maxpixels = [50, 100, 500, 1000]; %out of 28x28x28 pixels +epsilon = 1; % ep / 255 + + +%% Verification analysis +for a=advType + for mp=maxpixels + for ep=epsilon + + % 1) Initialize results var + results = zeros(N,2); + + % 2) Create adversarial attack + adv_attack = struct; + adv_attack.Name = a; % add or remove + adv_attack.max_pixels = mp; % Max number of pixels to modify from input image + adv_attack.noise_de = ep/255; % disturbance (noise) on pixels + + % 3) Begin verification analysis + for i=1:N + img = squeeze(inputs(:,:,:,:,i)); + target = targets(i); + results(i,:) = verify_instance_shape(net, img, target, adv_attack, reachOptions); + end + + % 4) % save results + save("results/verification_vessel_"+ a +"_" + ep +"_" + mp + ".mat", "results"); + + end + end +end diff --git a/code/nnv/examples/Submission/WiP_3d/visualize_results_gen.m b/code/nnv/examples/Submission/WiP_3d/visualize_results_gen.m new file mode 100644 index 0000000000..f0e14d02ad --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/visualize_results_gen.m @@ -0,0 +1,196 @@ +%% Get all results, one dataset at a time + +results = dir('results'); + +% Variable in the study +datasets = ["fracture", "nodule", "organ", "synapse"]; +attackType = ["bright", "dark"]; +maxpixels = ["50", "100", "500", "1000"]; +epsilon = ["2", "5", "10", "25"]; % epsilon/255 + +% Initialize directories +% mkdir('plots'); +% for i=datasets +% mkdir("plots/"+i); +% end + +%% Visualize verification results/trends + +% What do we want to show? + +% 1) Trends +% - Time: How does the time increases as the input set gets larger? +% - Epsilon increasing +% - Number of pixels increasing +% - results: How robust are the models as we increase the size of the attack? more unknowns? more sats? +% - Epsilon increasing +% - Number of pixels increasing +% 2) Certified robust accuracy +% - Is it much worse than the accuracy of the model? +% - Do we need more samples? + + +% Time for plots (maxpixels variable) +for ds = datasets + for adv = attackType + for ep = epsilon + + % Initialize vars to plot + sat = []; + unsat = []; + unk = []; + miss = []; + avgVT = []; + avgRT = []; + + % Get data + for mp = maxpixels + resFile = "results/verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; + res = summarize_results(resFile); + sat = [sat, res.sat]; + unsat = [unsat, res.unsat]; + unk = [unk, res.unknown]; + miss = [miss, res.misclassified]; + avgVT = [avgVT, res.avgTime]; % average computation time to verify one instance + avgRT = [avgRT, res.avgRT]; % average computation time to compute the reachable sets (miss and sat do not count here) + end + counts = [unsat; sat; unk; miss]; + + + % Create figure (results) + f = figure('visible','off'); + bar(1:4, counts','stacked') % plot verification results + grid; + % set values for x-axis + xticks([1 2 3 4]); + xticklabels(maxpixels); + % axis labels + xlabel('Max pixels perturbed'); + ylabel('# instances') + % legend + leg = {"unsat", "sat", "unknown", "missclass"}; + legend(leg, 'Location', 'best'); + % save figure + saveas(f, "plots/" + ds + "/verification_" + adv + "_" + ep + ".png"); + + % create figure (time) + f = figure('visible','off'); + % plot computation time results + plot(1:4, avgVT, 'r--o'); + hold on; + plot(1:4, avgRT, 'b--v'); + % set values for x-axis + xticks([1 2 3 4]); + xticklabels(maxpixels) + % axis labels + xlabel('Max pixels perturbed'); + ylabel("Average Time (s)") + % legend + leg = {"all", "unsat & unknown"}; + legend(leg, 'Location', 'best'); + % save figure + saveas(f, "plots/" + ds + "/avgTime_" + adv + "_" + ep + ".png"); + + end + end +end + +% Time for plots (epsilon variable) +for ds = datasets + for adv = attackType + for mp = maxpixels + + % Initialize vars to plot + sat = []; + unsat = []; + unk = []; + miss = []; + avgVT = []; + avgRT = []; + + % Get data + for ep = epsilon + resFile = "results/verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; + res = summarize_results(resFile); + sat = [sat, res.sat]; + unsat = [unsat, res.unsat]; + unk = [unk, res.unknown]; + miss = [miss, res.misclassified]; + avgVT = [avgVT, res.avgTime]; % average computation time to verify one instance + avgRT = [avgRT, res.avgRT]; % average computation time to compute the reachable sets (miss and sat do not count here) + end + counts = [unsat; sat; unk; miss]; + + % Create figure (results) + f = figure('visible','off'); + bar(1:3, counts','stacked'); % plot results + grid; + % set values for x-axis + xticks([1 2 3]); + xticklabels(epsilon) + % axis labels + xlabel('epsilon (\epsilon)'); + ylabel('# instances') + % legend + leg = {"unsat", "sat", "unknown", "missclass"}; + legend(leg, 'Location', 'best'); + % save figure + saveas(f, "plots/" + ds + "/verification_" + adv + "_" + mp + ".png"); + + % create figure (time) + f = figure('visible','off'); + % plot timing results + plot(1:3, avgVT, 'r--o'); + hold on; + plot(1:3, avgRT, 'b--v'); + % set values for x-axis + xticks([1 2 3]); + xticklabels(epsilon) + % axis labels + xlabel('epsilon (\epsilon)'); + ylabel("Average Time (s)") + % legend + leg = {"all", "unsat & unknown"}; + legend(leg, 'Location', 'best'); + % save figure + saveas(f, "plots/" + ds + "/avgTime_" + adv + "_" + mp + ".png"); + + end + end +end + + +%% Helper functions + +function summary = summarize_results(resFile) + + % Provide details from results file + summary = struct; + data = load(resFile); + results = data.results; + + % Total number of samples examined + summary.N = length(results); + + % Verified (unsat) + summary.unsat = sum(results(:,1) == 1); + + % Falsified (sat) from input set + summary.sat = sum(results(:,1) == 0); + + % Unknown (not using exact) + summary.unknown = sum(results(:,1) == 2); + + % Misclasified (sat) of original image + summary.misclassified = sum(results(:,1) == -1); + + % Also possible -2 (error), which may be out of memory (most common here) + summary.avgTime = sum(results(:,2))/summary.N; + + % Find instances where reachability is needed + x = [find(results(:,1)==2); find(results(:,1)==1)]; + nx = length(x); % how many instances reachability is computed for + + % Average time only for unknown and unsat properties (reachsets computed) + summary.avgRT = sum(results(x,2))/nx; +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/visualize_results.m b/code/nnv/examples/Submission/WiP_3d/visualize_results_shape.m similarity index 93% rename from code/nnv/examples/Submission/WiP_3d/visualize_results.m rename to code/nnv/examples/Submission/WiP_3d/visualize_results_shape.m index 4b273482d1..09c8c60370 100644 --- a/code/nnv/examples/Submission/WiP_3d/visualize_results.m +++ b/code/nnv/examples/Submission/WiP_3d/visualize_results_shape.m @@ -3,17 +3,16 @@ results = dir('results'); % Variable in the study -% datasets = ["adrenal", "fracture", "nodule", "organ", "synapse", "vessel"]; -datasets = ["fracture", "nodule", "organ", "synapse"]; +datasets = ["adrenal", "vessel"]; attackType = ["bright", "dark"]; maxpixels = ["50", "100", "500", "1000"]; -epsilon = ["2", "4", "10"]; % epsilon/255 +epsilon = ["1"]; % epsilon/255 % Initialize directories -mkdir('plots'); -for i=datasets - mkdir("plots/"+i); -end +% mkdir('plots'); +% for i=datasets +% mkdir("plots/"+i); +% end %% Visualize verification results/trends @@ -46,7 +45,7 @@ % Get data for mp = maxpixels - resFile = "verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; + resFile = "results/verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; res = summarize_results(resFile); sat = [sat, res.sat]; unsat = [unsat, res.unsat]; @@ -111,7 +110,7 @@ % Get data for ep = epsilon - resFile = "verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; + resFile = "results/verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; res = summarize_results(resFile); sat = [sat, res.sat]; unsat = [unsat, res.unsat];