diff --git a/code/nnv/engine/set/ImageStar.m b/code/nnv/engine/set/ImageStar.m index 44bed518ad..9e005d0bec 100644 --- a/code/nnv/engine/set/ImageStar.m +++ b/code/nnv/engine/set/ImageStar.m @@ -855,7 +855,7 @@ end - % get lowew bound and upper bound images of an imagestar + % get lower bound and upper bound images of an imagestar function [image_lb, image_ub] = getRanges(varargin) % @image_lb: lower bound image % @image_ub: upper bound image diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/ClassifySoundUsingDeepLearningExample.mlx b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/ClassifySoundUsingDeepLearningExample.mlx new file mode 100644 index 0000000000..5f336db38d Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/ClassifySoundUsingDeepLearningExample.mlx differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/adversarialReachability.m b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/adversarialReachability.m new file mode 100644 index 0000000000..e66ec34b91 --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/adversarialReachability.m @@ -0,0 +1,71 @@ +function [robust,T,PR,T_avg,T_sum] = adversarialReachability(varargin) +%% the function adversarialReachability takes the following inputs +% nnvnet : NNV supported neural network model +% dataset : dataset prepared by the 'createSOCDataset' function +% percent : the input perturbations as percentage for different noises +% noiseType : type of Noise +% randFeature : the particular feature we want to introduce the noise in + +% and provides the following outputs as a result of approx-star +% reachability analysis +% robust : local robustness for each of the input audio sequence +% PR : the percentage sample robustness value +% T_avg : the avg time for the reachability calculatiom +% T_sum : total time for the reachability calculation + + reachOptions.reachMethod = 'exact-star'; + switch nargin + case 7 + nnvnet = varargin(1); + dataset = varargin(2); + index = varargin(3); + percent = varargin(4); + noiseType = varargin(5); + classIndex = varargin(6); + randFeature = varargin(7); + case 6 + nnvnet = varargin(1); + dataset = varargin(2); + index = varargin(3); + percent = varargin(4); + noiseType = varargin(5); + classIndex = varargin(6); + otherwise + error('Invalid number of inputs, should be 6 or 7'); + end + percent = percent{1,1}; + nnvnet = nnvnet{1,1}; + randFeature = index{1,1}; + input = dataset{1,1}; + classIndex = classIndex{1,1}; + for j = 1: length(input) + ip = input(:,:,j)'; + lb = ip; + ub = ip; + if strcmp(noiseType{1,1},'SFSI') + XmuNorm = mean(ip(randFeature,:)); + lb(randFeature,end) = lb(randFeature, end)-XmuNorm*percent; + ub(randFeature,end) = ub(randFeature, end)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'SFAI') + XmuNorm = mean(ip(randFeature,:)); + lb(randFeature,:) = lb(randFeature,:)-XmuNorm*percent; + ub(randFeature,:) = ub(randFeature,:)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'MFSI') + XmuNorm = mean(ip(:,:)')'; + lb(:,end) = lb(:, end)-XmuNorm*percent; + ub(:,end) = ub(:, end)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'MFAI') + XmuNorm = mean(ip(:,:)')'; + lb = lb-XmuNorm*percent; + ub = ub+XmuNorm*percent; + end + IM = ImageStar(lb,ub); + start_time = tic; + R(j) = nnvnet.reachSequence(IM, reachOptions); + T(j) = toc(start_time); + robust(j) = nnvnet.checkRobust(R(j),classIndex); + end + PR = sum(robust==1)/100; + T_avg = mean(T); + T_sum = sum(T); +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_results.mat b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_results.mat new file mode 100644 index 0000000000..67552fa137 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_results.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures.fig b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures.fig new file mode 100644 index 0000000000..edfcda8feb Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures.fig differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures.pdf b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures.pdf new file mode 100644 index 0000000000..2598a58a8e Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures.pdf differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures2.fig b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures2.fig new file mode 100644 index 0000000000..7d8ab7c7b6 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures2.fig differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures2.pdf b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures2.pdf new file mode 100644 index 0000000000..78335d28fd Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/audioNoiseClassifier_robustnessMeasures2.pdf differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/calculateRobustnessMeasures.m b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/calculateRobustnessMeasures.m new file mode 100644 index 0000000000..1b5e5c5903 --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/calculateRobustnessMeasures.m @@ -0,0 +1,42 @@ +nnvnet = matlab2nnv(net); +load('data.mat',wNoiseTest_ex); +load('data.mat',bNoiseTest_ex); +load('data.mat',pNoiseTest_ex); + +for i = 1: length(percent) +for j = 1:1 +[robust_SFSI{i,j},T_SFSI{i,j},PR_SFSI{i,j},T_avg_SFSI{i,j},T_sum_SFSI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"SFSI",j); +[robust_SFAI{i,j},T_SFAI{i,j},PR_SFAI{i,j},T_avg_SFAI{i,j},T_sum_SFAI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"SFAI",j); +[robust_MFSI{i,j},T_MFSI{i,j},PR_MFSI{i,j},T_avg_MFSI{i,j},T_sum_MFSI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"MFSI",j); +[robust_MFAI{i,j},T_MFAI{i,j},PR_MFAI{i,j},T_avg_MFAI{i,j},T_sum_MFAI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"MFAI",j); +end +end +for i = 1: length(percent) +for j = 2:2 +[robust_SFSI{i,j},T_SFSI{i,j},PR_SFSI{i,j},T_avg_SFSI{i,j},T_sum_SFSI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"SFSI",j); +[robust_SFAI{i,j},T_SFAI{i,j},PR_SFAI{i,j},T_avg_SFAI{i,j},T_sum_SFAI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"SFAI",j); +[robust_MFSI{i,j},T_MFSI{i,j},PR_MFSI{i,j},T_avg_MFSI{i,j},T_sum_MFSI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"MFSI",j); +[robust_MFAI{i,j},T_MFAI{i,j},PR_MFAI{i,j},T_avg_MFAI{i,j},T_sum_MFAI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"MFAI",j); +end +end +for i = 1: length(percent) +for j = 3:3 +[robust_SFSI{i,j},T_SFSI{i,j},PR_SFSI{i,j},T_avg_SFSI{i,j},T_sum_SFSI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"SFSI",j); +[robust_SFAI{i,j},T_SFAI{i,j},PR_SFAI{i,j},T_avg_SFAI{i,j},T_sum_SFAI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"SFAI",j); +[robust_MFSI{i,j},T_MFSI{i,j},PR_MFSI{i,j},T_avg_MFSI{i,j},T_sum_MFSI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"MFSI",j); +[robust_MFAI{i,j},T_MFAI{i,j},PR_MFAI{i,j},T_avg_MFAI{i,j},T_sum_MFAI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"MFAI",j); +end +end + +GPR_MFAI = sum(cell2mat(PR_MFAI)')/3; +GPR_MFSI = sum(cell2mat(PR_MFSI)')/3; +GPR_SFSI = sum(cell2mat(PR_SFSI)')/3; +GPR_SFAI = sum(cell2mat(PR_SFAI)')/3; +GTavg_MFAI = sum(cell2mat(T_avg_MFAI)')/3; +GTavg_MFSI = sum(cell2mat(T_avg_MFSI)')/3; +GTavg_SFSI = sum(cell2mat(T_avg_SFSI)')/3; +GTavg_SFAI = sum(cell2mat(T_avg_SFAI)')/3; +GTsum_MFAI = sum(cell2mat(T_sum_MFAI)')/3; +GTsum_MFSI = sum(cell2mat(T_sum_MFSI)')/3; +GTsum_SFSI = sum(cell2mat(T_sum_SFSI)')/3; +GTsum_SFAI = sum(cell2mat(T_sum_SFAI)')/3; \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/data.mat b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/data.mat new file mode 100644 index 0000000000..397d6a91b5 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/data.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/generate_fig_robustness_value.m b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/generate_fig_robustness_value.m new file mode 100644 index 0000000000..cc53b4da59 --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/generate_fig_robustness_value.m @@ -0,0 +1,31 @@ +%% this file generates fig. 5 captioned +% "Percentage Robustness and Runtime plots w.r.t increasing noise" + +load audioNoiseClassifier_results.mat +figure; +perturbations = percent; +subplot(1,2,1); +plot(100*[0 perturbations],100*[1, GPR_MFAI]); +hold on +plot(100*[0 perturbations],100*[1, GPR_MFSI]); +hold on +plot(100*[0 perturbations],100*[1, GPR_SFAI]); +hold on +plot(100*[0 perturbations],100*[1, GPR_SFSI]); +xlabel('Pecentage Noise (%) '); +ylabel('Percentage Robustness (%)') +set(gca, 'FontSize', 22); +legend('MFAI','MFSI','SFAI','SFSI'); + +subplot(1,2,2); +plot(100*[0 perturbations],100*[0,GTsum_MFAI]); +hold on +plot(100*[0 perturbations],100*[0,GTsum_MFSI]); +hold on +plot(100*[0 perturbations],100*[0,GTsum_SFAI]); +hold on +plot(100*[0 perturbations],100*[0,GTsum_SFSI]); +xlabel('Pecentage Noise (%) '); +ylabel('Total Runtime (sec)') +set(gca, 'FontSize', 22); +legend('MFAI','MFSI','SFAI','SFSI'); diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/reachabilityData.mat b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/reachabilityData.mat new file mode 100644 index 0000000000..cdc222f2ac Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/reachabilityData.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/testdata.mat b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/testdata.mat new file mode 100644 index 0000000000..6bd66c4867 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/testdata.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/SequenceClassificationUsing1DConvolutionsExample.mlx b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/SequenceClassificationUsing1DConvolutionsExample.mlx new file mode 100644 index 0000000000..77311bbdf5 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/SequenceClassificationUsing1DConvolutionsExample.mlx differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/adversarialReachability.m b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/adversarialReachability.m new file mode 100644 index 0000000000..4324e32d1f --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/adversarialReachability.m @@ -0,0 +1,65 @@ +function [robust,T] = adversarialReachability(varargin) +%% the function adversarialReachability takes the following inputs +% nnvnet : NNV supported neural network model +% dataset : dataset prepared by the 'createSOCDataset' function +% percent : the input perturbations as percentage for different noises +% noiseType : type of Noise +% randFeature : the particular feature we want to introduce the noise in + +% and provides the following outputs as a result of approx-star +% reachability analysis +% robust : local robustness for each of the input audio sequence +% PR : the percentage sample robustness value +% T_avg : the avg time for the reachability calculatiom +% T_sum : total time for the reachability calculation + + reachOptions.reachMethod = 'approx-star'; + switch nargin + case 7 + nnvnet = varargin(1); + input = varargin(2); + index = varargin(3); + percent = varargin(4); + noiseType = varargin(5); + classIndex = varargin(6); + randFeature = varargin(7); + case 6 + nnvnet = varargin(1); + input = varargin(2); + index = varargin(3); + percent = varargin(4); + noiseType = varargin(5); + classIndex = varargin(6); + otherwise + error('Invalid number of inputs, should be 6 or 7'); + end + percent = percent{1,1}; + nnvnet = nnvnet{1,1}; + randFeature = index{1,1}; + ip = input{1,1}; + classIndex = classIndex{1,1}; + lb = ip; + ub = ip; + if strcmp(noiseType{1,1},'SFSI') + XmuNorm = mean(ip(randFeature,:)); + lb(randFeature,end) = lb(randFeature, end)-XmuNorm*percent; + ub(randFeature,end) = ub(randFeature, end)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'SFAI') + XmuNorm = mean(ip(randFeature,:)); + lb(randFeature,:) = lb(randFeature,:)-XmuNorm*percent; + ub(randFeature,:) = ub(randFeature,:)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'MFSI') + XmuNorm = mean(ip(:,:)')'; + lb(:,end) = lb(:, end)-XmuNorm*percent; + ub(:,end) = ub(:, end)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'MFAI') + XmuNorm = mean(ip(:,:)')'; + lb = lb-XmuNorm*percent; + ub = ub+XmuNorm*percent; + end + IM = ImageStar(lb,ub); + start_time = tic; + R = nnvnet.reachSequence(IM, reachOptions); + T = toc(start_time); + robust = nnvnet.checkRobust(R,classIndex); +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.asv b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.asv new file mode 100644 index 0000000000..1231f8dcf9 --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.asv @@ -0,0 +1,22 @@ +percent = [0.01, 0.02, 0.03, 0.04, 0.05]; +nnvnet = matlab2nnv(net); +YTest = double(YTest_reach); +for j = 1: length(percent) + for i = 1:length(XTest_reach) + i + input = XTest_reach{i,1}; + classIndex = YTest(i); + [robust_SFSI(j,i),T_SFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFSI",classIndex); + [robust_SFAI(j,i),T_SFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFAI",classIndex); + [robust_MFSI(j,i),T_MFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFSI",classIndex); + [robust_MFAI(j,i),T_MFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFAI",classIndex); + end + PR_SFSI(1,j) = sum(robust_SFSI(j,:)==1)/length(XTest_reach); + T_sum_SFSI(1,j) = sum(T_SFSI(j,:)); + PR_SFAI(1,j) = sum(robust_SFAI(j,:)==1)/length(XTest_reach); + T_sum_SFAI(1,j) = sum(T_SFAI(j,:)); + PR_MFSI(1,j) = sum(robust_MFSI(j,:)==1)/length(XTest_reach); + T_sum_MFSI(1,j) = sum(T_MFSI(j,:)); + PR_MFAI(1,j) = sum(robust_MFAI(j,:)==1)/length(XTest_correct); + T_sum_MFAI(1,j) = sum(T_MFAI(j,:)); +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.m b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.m new file mode 100644 index 0000000000..029cd72a7b --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.m @@ -0,0 +1,22 @@ +percent = [0.5, 0.6, 0.7, 0.8, 0.9]; +nnvnet = matlab2nnv(net); +YTest = double(YTest_Correct); +for j = 1: length(percent) + for i = 1:length(XTest_Correct) + i + input = XTest_Correct{i,1}; + classIndex = YTest(i); + [robust_SFSI(j,i),T_SFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFSI",classIndex); + [robust_SFAI(j,i),T_SFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFAI",classIndex); + [robust_MFSI(j,i),T_MFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFSI",classIndex); + [robust_MFAI(j,i),T_MFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFAI",classIndex); + end + PR_SFSI(1,j) = sum(robust_SFSI(j,:)==1)/length(XTest_Correct); + T_sum_SFSI(1,j) = sum(T_SFSI(j,:)); + PR_SFAI(1,j) = sum(robust_SFAI(j,:)==1)/length(XTest_Correct); + T_sum_SFAI(1,j) = sum(T_SFAI(j,:)); + PR_MFSI(1,j) = sum(robust_MFSI(j,:)==1)/length(XTest_Correct); + T_sum_MFSI(1,j) = sum(T_MFSI(j,:)); + PR_MFAI(1,j) = sum(robust_MFAI(j,:)==1)/length(XTest_Correct); + T_sum_MFAI(1,j) = sum(T_MFAI(j,:)); +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/data.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/data.mat new file mode 100644 index 0000000000..ad4e88054f Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/data.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/data_cnnlstm.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/data_cnnlstm.mat new file mode 100644 index 0000000000..93aae557c6 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/data_cnnlstm.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/generate_fig_robustness_value.m b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/generate_fig_robustness_value.m new file mode 100644 index 0000000000..d7294a771e --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/generate_fig_robustness_value.m @@ -0,0 +1,31 @@ +%% this file generates fig. 5 captioned +% "Percentage Robustness and Runtime plots w.r.t increasing noise" + +load japanesevowel_cnnlstmClassifier_results2.mat +figure; +perturbations = percent; +subplot(1,2,1); +plot(100*[0 perturbations],100*[1, PR_MFAI]); +hold on +plot(100*[0 perturbations],100*[1, PR_MFSI]); +hold on +plot(100*[0 perturbations],100*[1, PR_SFAI]); +hold on +plot(100*[0 perturbations],100*[1, PR_SFSI]); +xlabel('Pecentage Noise (%) '); +ylabel('Percentage Robustness (%)') +set(gca, 'FontSize', 22); +legend('MFAI','MFSI','SFAI','SFSI'); + +subplot(1,2,2); +plot(100*[0 perturbations],100*[0,T_sum_MFAI]); +hold on +plot(100*[0 perturbations],100*[0,T_sum_MFSI]); +hold on +plot(100*[0 perturbations],100*[0,T_sum_SFAI]); +hold on +plot(100*[0 perturbations],100*[0,T_sum_SFSI]); +xlabel('Pecentage Noise (%) '); +ylabel('Total Runtime (sec)') +set(gca, 'FontSize', 22); +legend('MFAI','MFSI','SFAI','SFSI'); diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/japanesevowel_cnnlstmClassifier_results.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/japanesevowel_cnnlstmClassifier_results.mat new file mode 100644 index 0000000000..2683cd2adb Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/japanesevowel_cnnlstmClassifier_results.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/japanesevowel_cnnlstmClassifier_results2.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/japanesevowel_cnnlstmClassifier_results2.mat new file mode 100644 index 0000000000..2bf334f103 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/japanesevowel_cnnlstmClassifier_results2.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/reachdata.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/reachdata.mat new file mode 100644 index 0000000000..a0dda7ac38 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/reachdata.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures.fig b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures.fig new file mode 100644 index 0000000000..7f599b8400 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures.fig differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures2.fig b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures2.fig new file mode 100644 index 0000000000..5ca7bf6469 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures2.fig differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures2.pdf b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures2.pdf new file mode 100644 index 0000000000..a9d1fff981 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures2.pdf differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures_jv_cnnlstm.pdf b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures_jv_cnnlstm.pdf new file mode 100644 index 0000000000..86de69a872 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/robustnessmeasures_jv_cnnlstm.pdf differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/test.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/test.mat new file mode 100644 index 0000000000..b13a26ecd5 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/test.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/ClassifySequenceDataUsingLSTMNetworksExample.mlx b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/ClassifySequenceDataUsingLSTMNetworksExample.mlx new file mode 100644 index 0000000000..6a4aaf4772 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/ClassifySequenceDataUsingLSTMNetworksExample.mlx differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/adversarialReachability.m b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/adversarialReachability.m new file mode 100644 index 0000000000..fde5f06fb5 --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/adversarialReachability.m @@ -0,0 +1,65 @@ +function [robust,T] = adversarialReachability(varargin) +%% the function adversarialReachability takes the following inputs +% nnvnet : NNV supported neural network model +% dataset : dataset prepared by the 'createSOCDataset' function +% percent : the input perturbations as percentage for different noises +% noiseType : type of Noise +% randFeature : the particular feature we want to introduce the noise in + +% and provides the following outputs as a result of approx-star +% reachability analysis +% robust : local robustness for each of the input audio sequence +% PR : the percentage sample robustness value +% T_avg : the avg time for the reachability calculatiom +% T_sum : total time for the reachability calculation + + reachOptions.reachMethod = 'exact-star'; + switch nargin + case 7 + nnvnet = varargin(1); + input = varargin(2); + index = varargin(3); + percent = varargin(4); + noiseType = varargin(5); + classIndex = varargin(6); + randFeature = varargin(7); + case 6 + nnvnet = varargin(1); + input = varargin(2); + index = varargin(3); + percent = varargin(4); + noiseType = varargin(5); + classIndex = varargin(6); + otherwise + error('Invalid number of inputs, should be 6 or 7'); + end + percent = percent{1,1}; + nnvnet = nnvnet{1,1}; + randFeature = index{1,1}; + ip = input{1,1}; + classIndex = classIndex{1,1}; + lb = ip; + ub = ip; + if strcmp(noiseType{1,1},'SFSI') + XmuNorm = mean(ip(randFeature,:)); + lb(randFeature,end) = lb(randFeature, end)-XmuNorm*percent; + ub(randFeature,end) = ub(randFeature, end)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'SFAI') + XmuNorm = mean(ip(randFeature,:)); + lb(randFeature,:) = lb(randFeature,:)-XmuNorm*percent; + ub(randFeature,:) = ub(randFeature,:)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'MFSI') + XmuNorm = mean(ip(:,:)')'; + lb(:,end) = lb(:, end)-XmuNorm*percent; + ub(:,end) = ub(:, end)+XmuNorm*percent; + elseif strcmp(noiseType{1,1},'MFAI') + XmuNorm = mean(ip(:,:)')'; + lb = lb-XmuNorm*percent; + ub = ub+XmuNorm*percent; + end + IM = ImageStar(lb,ub); + start_time = tic; + R = nnvnet.reachSequence(IM, reachOptions); + T = toc(start_time); + robust = nnvnet.checkRobust(R,classIndex); +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/calculateRobustnessMeasures.m b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/calculateRobustnessMeasures.m new file mode 100644 index 0000000000..2b26b41eed --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/calculateRobustnessMeasures.m @@ -0,0 +1,21 @@ +percent = [0.5, 0.6, 0.7, 0.8, 0.9]; +nnvnet = matlab2nnv(net); +YTest = double(YTest_correct); +for j = 1: length(percent) + for i = 1:length(XTest_correct) + input = XTest_correct{i,1}; + classIndex = YTest(i); + [robust_SFSI(j,i),T_SFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFSI",classIndex); + [robust_SFAI(j,i),T_SFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFAI",classIndex); + [robust_MFSI(j,i),T_MFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFSI",classIndex); + [robust_MFAI(j,i),T_MFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFAI",classIndex); + end + PR_SFSI(1,j) = sum(robust_SFSI(j,:)==1)/length(XTest_correct); + T_sum_SFSI(1,j) = sum(T_SFSI(j,:)); + PR_SFAI(1,j) = sum(robust_SFAI(j,:)==1)/length(XTest_correct); + T_sum_SFAI(1,j) = sum(T_SFAI(j,:)); + PR_MFSI(1,j) = sum(robust_MFSI(j,:)==1)/length(XTest_correct); + T_sum_MFSI(1,j) = sum(T_MFSI(j,:)); + PR_MFAI(1,j) = sum(robust_MFAI(j,:)==1)/length(XTest_correct); + T_sum_MFAI(1,j) = sum(T_MFAI(j,:)); +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/data.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/data.mat new file mode 100644 index 0000000000..d9e218b533 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/data.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/generate_fig_robustness_value.m b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/generate_fig_robustness_value.m new file mode 100644 index 0000000000..74ff5d360b --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/generate_fig_robustness_value.m @@ -0,0 +1,31 @@ +%% this file generates fig. 5 captioned +% "Percentage Robustness and Runtime plots w.r.t increasing noise" + +load japanesevowel_lstmClassifier_results.mat +figure; +perturbations = percent; +subplot(1,2,1); +plot(100*[0 perturbations],100*[1, PR_MFAI]); +hold on +plot(100*[0 perturbations],100*[1, PR_MFSI]); +hold on +plot(100*[0 perturbations],100*[1, PR_SFAI]); +hold on +plot(100*[0 perturbations],100*[1, PR_SFSI]); +xlabel('Pecentage Noise (%) '); +ylabel('Percentage Robustness (%)') +set(gca, 'FontSize', 22); +legend('MFAI','MFSI','SFAI','SFSI'); + +subplot(1,2,2); +plot(100*[0 perturbations],100*[0,T_sum_MFAI]); +hold on +plot(100*[0 perturbations],100*[0,T_sum_MFSI]); +hold on +plot(100*[0 perturbations],100*[0,T_sum_SFAI]); +hold on +plot(100*[0 perturbations],100*[0,T_sum_SFSI]); +xlabel('Pecentage Noise (%) '); +ylabel('Total Runtime (sec)') +set(gca, 'FontSize', 22); +legend('MFAI','MFSI','SFAI','SFSI'); diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/japanesevowel_lstmClassifier_results.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/japanesevowel_lstmClassifier_results.mat new file mode 100644 index 0000000000..e73092c4a7 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/japanesevowel_lstmClassifier_results.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/reachability.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/reachability.mat new file mode 100644 index 0000000000..e457ce847a Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/reachability.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures.fig b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures.fig new file mode 100644 index 0000000000..741a039a95 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures.fig differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures2.fig b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures2.fig new file mode 100644 index 0000000000..f8c44e81f4 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures2.fig differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures_jv_lstm.pdf b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures_jv_lstm.pdf new file mode 100644 index 0000000000..baae32be06 Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/robustnessmeasures_jv_lstm.pdf differ diff --git a/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/test.mat b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/test.mat new file mode 100644 index 0000000000..ae6c07b19b Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/test.mat differ diff --git a/code/nnv/examples/Submission/FMAS2023/networks.fig b/code/nnv/examples/Submission/FMAS2023/networks.fig new file mode 100644 index 0000000000..709db83e0a Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/networks.fig differ diff --git a/code/nnv/examples/Submission/FMAS2023/networks.pdf b/code/nnv/examples/Submission/FMAS2023/networks.pdf new file mode 100644 index 0000000000..be1f97fede Binary files /dev/null and b/code/nnv/examples/Submission/FMAS2023/networks.pdf differ diff --git a/code/nnv/examples/Submission/FMAS2023/plotnetworks.m b/code/nnv/examples/Submission/FMAS2023/plotnetworks.m new file mode 100644 index 0000000000..6e46675645 --- /dev/null +++ b/code/nnv/examples/Submission/FMAS2023/plotnetworks.m @@ -0,0 +1,15 @@ +figure +subplot(1,3,1); +plot(audionoise.layerGraph); +title("Audio Noise LSTM Classifier") +set(gca, 'FontSize', 22); + +subplot(1,3,2); +plot(audionoise.layerGraph); +title("Japanese Vowel LSTM Classifier") +set(gca, 'FontSize', 22); + +subplot(1,3,3); +plot(cnnlstm.layerGraph); +title("Japanese Vowel CNN+LSTM Classifier") +set(gca, 'FontSize', 22); \ No newline at end of file diff --git a/code/nnv/examples/Submission/FMICS2023/PredictBatterySOCUsingDeepLearningExample/createSOCDataset.m b/code/nnv/examples/Submission/FMICS2023/PredictBatterySOCUsingDeepLearningExample/createSOCDataset.m index b5f92a484e..484d5057f6 100644 --- a/code/nnv/examples/Submission/FMICS2023/PredictBatterySOCUsingDeepLearningExample/createSOCDataset.m +++ b/code/nnv/examples/Submission/FMICS2023/PredictBatterySOCUsingDeepLearningExample/createSOCDataset.m @@ -3,7 +3,7 @@ % and stores in a struct variable 'dataset' % input : single-featured or multi-featured time series % output : desired output for the dataset -% preictedOutput : output predicted by the NN model +% predictedOutput : output predicted by the NN model % sqeL : desired sequence length of each segment of time-series % delta : extent of deviation allowed in output (scale of 0-100)% % type : the dataset type, "cons"->consecutive, "rand"->random diff --git a/code/nnv/examples/Submission/FMICS2023/RULEstimationUsingCNNExample/createTEDSDataset.m b/code/nnv/examples/Submission/FMICS2023/RULEstimationUsingCNNExample/createTEDSDataset.m index 95dfeb48c4..2e2be128fd 100644 --- a/code/nnv/examples/Submission/FMICS2023/RULEstimationUsingCNNExample/createTEDSDataset.m +++ b/code/nnv/examples/Submission/FMICS2023/RULEstimationUsingCNNExample/createTEDSDataset.m @@ -5,7 +5,7 @@ % output : desired output for the dataset % preictedOutput : output predicted by the NN model % sqeL : desired sequence length of each segment of time-series -% delta : extent of deviation allowed in output (scale of 0-100)% +% epsilon : extent of deviation allowed in output (scale of 0-100)% % type : the noise type % dataset : a struct, containing the allowable bounds (i.e., desired diff --git a/code/nnv/examples/Submission/FMICS2023/RULEstimationUsingCNNExample/testing code for RUL.m b/code/nnv/examples/Submission/FMICS2023/RULEstimationUsingCNNExample/testing code for RUL.m new file mode 100644 index 0000000000..3deaaa0e8c --- /dev/null +++ b/code/nnv/examples/Submission/FMICS2023/RULEstimationUsingCNNExample/testing code for RUL.m @@ -0,0 +1,25 @@ +load('TEDS_results.mat', 'X') +load('TEDS_results.mat', 'Y') +load('TEDS_results.mat', 'Y_pred_52') +load('TEDS_results.mat', 'dataset_TEDS') +load('TEDS_results.mat', 'net_TEDS') +load('TEDS_results.mat', 'perturbations') +load('TEDS_results.mat', 'feature_to_attack') + +nnvnet_TEDS = matlab2nnv(net_TEDS); + +%% i represenrts the dataset for different sequence lengths +% i == 1 -> seq length 10 +% i == 2 -> seq length 20 +% i == 3 -> seq length 30 + +i = 1; + +for j = 1: length(perturbations) + + [LB_n1{i,j},UB_n1{i,j},T_n1{i,j},PR_n1{i,j},POR_n1{i,j},T_avg_n1{i,j},T_sum_n1{i,j}] = reachabilityNoise(nnvnet,dataset_cons,i,perturbations(j),'SFSI',feature_to_attack); + % [LB_n2{i,j},UB_n2{i,j},T_n2{i,j},PR_n2{i,j},POR_n2{i,j},T_avg_n2{i,j},T_sum_n2{i,j}] = reachabilityNoise(nnvnet,dataset_cons,i,perturbations(j),'SFAI',feature_to_attack); + % [LB_n3{i,j},UB_n3{i,j},T_n3{i,j},PR_n3{i,j},POR_n3{i,j},T_avg_n3{i,j},T_sum_n3{i,j}] = reachabilityNoise(nnvnet,dataset_cons,i,perturbations(j),'MFSI',feature_to_attack); + +end + \ No newline at end of file