Merge pull request #204 from Neelanjana314/master

Support for FMAS2023

code/nnv/engine/set/ImageStar.m

@@ -855,7 +855,7 @@
 
         end
 
-        % get lowew bound and upper bound images of an imagestar
+        % get lower bound and upper bound images of an imagestar
         function [image_lb, image_ub] = getRanges(varargin)
             % @image_lb: lower bound image
             % @image_ub: upper bound image

code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/adversarialReachability.m

@@ -0,0 +1,71 @@
+function [robust,T,PR,T_avg,T_sum] = adversarialReachability(varargin)
+%% the function adversarialReachability takes the following inputs
+%   nnvnet  : NNV supported neural network model
+%   dataset : dataset prepared by the 'createSOCDataset' function
+%   percent : the input perturbations as percentage for different noises
+%   noiseType   : type of Noise
+%   randFeature : the particular feature we want to introduce the noise in
+
+%   and provides the following outputs as a result of approx-star
+%   reachability analysis
+%   robust  :   local robustness for each of the input audio sequence
+%   PR      :   the percentage sample robustness value
+%   T_avg   :   the avg time for the reachability calculatiom
+%   T_sum   : total time for the reachability calculation
+
+    reachOptions.reachMethod = 'exact-star';
+    switch nargin
+        case 7
+            nnvnet = varargin(1);
+            dataset = varargin(2);
+            index = varargin(3);
+            percent = varargin(4);
+            noiseType = varargin(5);
+            classIndex = varargin(6);
+            randFeature = varargin(7);
+        case 6
+            nnvnet = varargin(1);
+            dataset = varargin(2);
+            index = varargin(3);
+            percent = varargin(4);
+            noiseType = varargin(5);
+            classIndex = varargin(6);
+        otherwise
+            error('Invalid number of inputs, should be 6 or 7');
+    end
+    percent = percent{1,1};
+    nnvnet = nnvnet{1,1};
+    randFeature = index{1,1};
+    input = dataset{1,1};
+    classIndex = classIndex{1,1};
+    for j = 1: length(input)
+        ip = input(:,:,j)';
+        lb = ip;
+        ub = ip;
+        if strcmp(noiseType{1,1},'SFSI')
+            XmuNorm = mean(ip(randFeature,:));
+            lb(randFeature,end) = lb(randFeature, end)-XmuNorm*percent;
+            ub(randFeature,end) = ub(randFeature, end)+XmuNorm*percent;
+        elseif strcmp(noiseType{1,1},'SFAI')
+            XmuNorm = mean(ip(randFeature,:));
+            lb(randFeature,:) = lb(randFeature,:)-XmuNorm*percent;
+            ub(randFeature,:) = ub(randFeature,:)+XmuNorm*percent;
+        elseif strcmp(noiseType{1,1},'MFSI')
+            XmuNorm = mean(ip(:,:)')';
+            lb(:,end) = lb(:, end)-XmuNorm*percent;
+            ub(:,end) = ub(:, end)+XmuNorm*percent;
+        elseif strcmp(noiseType{1,1},'MFAI')
+            XmuNorm = mean(ip(:,:)')';
+            lb = lb-XmuNorm*percent;
+            ub = ub+XmuNorm*percent;
+        end
+        IM = ImageStar(lb,ub);
+        start_time = tic;
+        R(j) = nnvnet.reachSequence(IM, reachOptions);
+        T(j) = toc(start_time);
+        robust(j) = nnvnet.checkRobust(R(j),classIndex);
+    end
+    PR = sum(robust==1)/100;
+    T_avg = mean(T);
+    T_sum = sum(T);   
+end

code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/calculateRobustnessMeasures.m

@@ -0,0 +1,42 @@
+nnvnet = matlab2nnv(net);
+load('data.mat',wNoiseTest_ex);
+load('data.mat',bNoiseTest_ex);
+load('data.mat',pNoiseTest_ex);
+
+for i = 1: length(percent)
+for j = 1:1
+[robust_SFSI{i,j},T_SFSI{i,j},PR_SFSI{i,j},T_avg_SFSI{i,j},T_sum_SFSI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"SFSI",j);
+[robust_SFAI{i,j},T_SFAI{i,j},PR_SFAI{i,j},T_avg_SFAI{i,j},T_sum_SFAI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"SFAI",j);
+[robust_MFSI{i,j},T_MFSI{i,j},PR_MFSI{i,j},T_avg_MFSI{i,j},T_sum_MFSI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"MFSI",j);
+[robust_MFAI{i,j},T_MFAI{i,j},PR_MFAI{i,j},T_avg_MFAI{i,j},T_sum_MFAI{i,j}] = adversarialReachability(nnvnet,wNoiseTest_ex,1,percent(i),"MFAI",j);
+end
+end
+for i = 1: length(percent)
+for j = 2:2
+[robust_SFSI{i,j},T_SFSI{i,j},PR_SFSI{i,j},T_avg_SFSI{i,j},T_sum_SFSI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"SFSI",j);
+[robust_SFAI{i,j},T_SFAI{i,j},PR_SFAI{i,j},T_avg_SFAI{i,j},T_sum_SFAI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"SFAI",j);
+[robust_MFSI{i,j},T_MFSI{i,j},PR_MFSI{i,j},T_avg_MFSI{i,j},T_sum_MFSI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"MFSI",j);
+[robust_MFAI{i,j},T_MFAI{i,j},PR_MFAI{i,j},T_avg_MFAI{i,j},T_sum_MFAI{i,j}] = adversarialReachability(nnvnet,bNoiseTest_ex,1,percent(i),"MFAI",j);
+end
+end
+for i = 1: length(percent)
+for j = 3:3
+[robust_SFSI{i,j},T_SFSI{i,j},PR_SFSI{i,j},T_avg_SFSI{i,j},T_sum_SFSI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"SFSI",j);
+[robust_SFAI{i,j},T_SFAI{i,j},PR_SFAI{i,j},T_avg_SFAI{i,j},T_sum_SFAI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"SFAI",j);
+[robust_MFSI{i,j},T_MFSI{i,j},PR_MFSI{i,j},T_avg_MFSI{i,j},T_sum_MFSI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"MFSI",j);
+[robust_MFAI{i,j},T_MFAI{i,j},PR_MFAI{i,j},T_avg_MFAI{i,j},T_sum_MFAI{i,j}] = adversarialReachability(nnvnet,pNoiseTest_ex,1,percent(i),"MFAI",j);
+end
+end
+
+GPR_MFAI = sum(cell2mat(PR_MFAI)')/3;
+GPR_MFSI = sum(cell2mat(PR_MFSI)')/3;
+GPR_SFSI = sum(cell2mat(PR_SFSI)')/3;
+GPR_SFAI = sum(cell2mat(PR_SFAI)')/3;
+GTavg_MFAI = sum(cell2mat(T_avg_MFAI)')/3;
+GTavg_MFSI = sum(cell2mat(T_avg_MFSI)')/3;
+GTavg_SFSI = sum(cell2mat(T_avg_SFSI)')/3;
+GTavg_SFAI = sum(cell2mat(T_avg_SFAI)')/3;
+GTsum_MFAI = sum(cell2mat(T_sum_MFAI)')/3;
+GTsum_MFSI = sum(cell2mat(T_sum_MFSI)')/3;
+GTsum_SFSI = sum(cell2mat(T_sum_SFSI)')/3;
+GTsum_SFAI = sum(cell2mat(T_sum_SFAI)')/3;

code/nnv/examples/Submission/FMAS2023/audioNoiseDataset/generate_fig_robustness_value.m

@@ -0,0 +1,31 @@
+%% this file generates fig. 5 captioned
+% "Percentage Robustness and Runtime plots w.r.t increasing noise"
+
+load audioNoiseClassifier_results.mat
+figure;
+perturbations = percent;
+subplot(1,2,1);
+plot(100*[0 perturbations],100*[1, GPR_MFAI]);
+hold on
+plot(100*[0 perturbations],100*[1, GPR_MFSI]);
+hold on
+plot(100*[0 perturbations],100*[1, GPR_SFAI]);
+hold on
+plot(100*[0 perturbations],100*[1, GPR_SFSI]);
+xlabel('Pecentage Noise (%) ');
+ylabel('Percentage Robustness (%)')
+set(gca, 'FontSize', 22);
+legend('MFAI','MFSI','SFAI','SFSI');
+
+subplot(1,2,2);
+plot(100*[0 perturbations],100*[0,GTsum_MFAI]);
+hold on
+plot(100*[0 perturbations],100*[0,GTsum_MFSI]);
+hold on
+plot(100*[0 perturbations],100*[0,GTsum_SFAI]);
+hold on
+plot(100*[0 perturbations],100*[0,GTsum_SFSI]);
+xlabel('Pecentage Noise (%) ');
+ylabel('Total Runtime (sec)')
+set(gca, 'FontSize', 22);
+legend('MFAI','MFSI','SFAI','SFSI');

code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/adversarialReachability.m

@@ -0,0 +1,65 @@
+function [robust,T] = adversarialReachability(varargin)
+%% the function adversarialReachability takes the following inputs
+%   nnvnet  : NNV supported neural network model
+%   dataset : dataset prepared by the 'createSOCDataset' function
+%   percent : the input perturbations as percentage for different noises
+%   noiseType   : type of Noise
+%   randFeature : the particular feature we want to introduce the noise in
+
+%   and provides the following outputs as a result of approx-star
+%   reachability analysis
+%   robust  :   local robustness for each of the input audio sequence
+%   PR      :   the percentage sample robustness value
+%   T_avg   :   the avg time for the reachability calculatiom
+%   T_sum   : total time for the reachability calculation
+
+    reachOptions.reachMethod = 'approx-star';
+    switch nargin
+        case 7
+            nnvnet = varargin(1);
+            input = varargin(2);
+            index = varargin(3);
+            percent = varargin(4);
+            noiseType = varargin(5);
+            classIndex = varargin(6);
+            randFeature = varargin(7);
+        case 6
+            nnvnet = varargin(1);
+            input = varargin(2);
+            index = varargin(3);
+            percent = varargin(4);
+            noiseType = varargin(5);
+            classIndex = varargin(6);
+        otherwise
+            error('Invalid number of inputs, should be 6 or 7');
+    end
+    percent = percent{1,1};
+    nnvnet = nnvnet{1,1};
+    randFeature = index{1,1};
+    ip = input{1,1};
+    classIndex = classIndex{1,1};
+    lb = ip;
+    ub = ip;
+    if strcmp(noiseType{1,1},'SFSI')
+        XmuNorm = mean(ip(randFeature,:));
+        lb(randFeature,end) = lb(randFeature, end)-XmuNorm*percent;
+        ub(randFeature,end) = ub(randFeature, end)+XmuNorm*percent;
+    elseif strcmp(noiseType{1,1},'SFAI')
+        XmuNorm = mean(ip(randFeature,:));
+        lb(randFeature,:) = lb(randFeature,:)-XmuNorm*percent;
+        ub(randFeature,:) = ub(randFeature,:)+XmuNorm*percent;
+    elseif strcmp(noiseType{1,1},'MFSI')
+        XmuNorm = mean(ip(:,:)')';
+        lb(:,end) = lb(:, end)-XmuNorm*percent;
+        ub(:,end) = ub(:, end)+XmuNorm*percent;
+    elseif strcmp(noiseType{1,1},'MFAI')
+        XmuNorm = mean(ip(:,:)')';
+        lb = lb-XmuNorm*percent;
+        ub = ub+XmuNorm*percent;
+    end
+    IM = ImageStar(lb,ub);
+    start_time = tic;
+    R = nnvnet.reachSequence(IM, reachOptions);
+    T = toc(start_time);
+    robust = nnvnet.checkRobust(R,classIndex);  
+end

code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.asv

@@ -0,0 +1,22 @@
+percent = [0.01, 0.02, 0.03, 0.04, 0.05];
+nnvnet = matlab2nnv(net);
+YTest = double(YTest_reach);
+for j = 1: length(percent)
+    for i = 1:length(XTest_reach)
+        i
+        input = XTest_reach{i,1};
+        classIndex = YTest(i);
+        [robust_SFSI(j,i),T_SFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFSI",classIndex);
+        [robust_SFAI(j,i),T_SFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFAI",classIndex);
+        [robust_MFSI(j,i),T_MFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFSI",classIndex);
+        [robust_MFAI(j,i),T_MFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFAI",classIndex);
+    end
+    PR_SFSI(1,j) = sum(robust_SFSI(j,:)==1)/length(XTest_reach);
+    T_sum_SFSI(1,j) = sum(T_SFSI(j,:));
+    PR_SFAI(1,j) = sum(robust_SFAI(j,:)==1)/length(XTest_reach);
+    T_sum_SFAI(1,j) = sum(T_SFAI(j,:));
+    PR_MFSI(1,j) = sum(robust_MFSI(j,:)==1)/length(XTest_reach);
+    T_sum_MFSI(1,j) = sum(T_MFSI(j,:));
+    PR_MFAI(1,j) = sum(robust_MFAI(j,:)==1)/length(XTest_correct);
+    T_sum_MFAI(1,j) = sum(T_MFAI(j,:));
+end

code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/calculateRobustnessMeasures.m

@@ -0,0 +1,22 @@
+percent = [0.5, 0.6, 0.7, 0.8, 0.9];
+nnvnet = matlab2nnv(net);
+YTest = double(YTest_Correct);
+for j = 1: length(percent)
+    for i = 1:length(XTest_Correct)
+        i
+        input = XTest_Correct{i,1};
+        classIndex = YTest(i);
+        [robust_SFSI(j,i),T_SFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFSI",classIndex);
+        [robust_SFAI(j,i),T_SFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFAI",classIndex);
+        [robust_MFSI(j,i),T_MFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFSI",classIndex);
+        [robust_MFAI(j,i),T_MFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFAI",classIndex);
+    end
+    PR_SFSI(1,j) = sum(robust_SFSI(j,:)==1)/length(XTest_Correct);
+    T_sum_SFSI(1,j) = sum(T_SFSI(j,:));
+    PR_SFAI(1,j) = sum(robust_SFAI(j,:)==1)/length(XTest_Correct);
+    T_sum_SFAI(1,j) = sum(T_SFAI(j,:));
+    PR_MFSI(1,j) = sum(robust_MFSI(j,:)==1)/length(XTest_Correct);
+    T_sum_MFSI(1,j) = sum(T_MFSI(j,:));
+    PR_MFAI(1,j) = sum(robust_MFAI(j,:)==1)/length(XTest_Correct);
+    T_sum_MFAI(1,j) = sum(T_MFAI(j,:));
+end

code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/cnnlstm model/generate_fig_robustness_value.m

@@ -0,0 +1,31 @@
+%% this file generates fig. 5 captioned
+% "Percentage Robustness and Runtime plots w.r.t increasing noise"
+
+load japanesevowel_cnnlstmClassifier_results2.mat
+figure;
+perturbations = percent;
+subplot(1,2,1);
+plot(100*[0 perturbations],100*[1, PR_MFAI]);
+hold on
+plot(100*[0 perturbations],100*[1, PR_MFSI]);
+hold on
+plot(100*[0 perturbations],100*[1, PR_SFAI]);
+hold on
+plot(100*[0 perturbations],100*[1, PR_SFSI]);
+xlabel('Pecentage Noise (%) ');
+ylabel('Percentage Robustness (%)')
+set(gca, 'FontSize', 22);
+legend('MFAI','MFSI','SFAI','SFSI');
+
+subplot(1,2,2);
+plot(100*[0 perturbations],100*[0,T_sum_MFAI]);
+hold on
+plot(100*[0 perturbations],100*[0,T_sum_MFSI]);
+hold on
+plot(100*[0 perturbations],100*[0,T_sum_SFAI]);
+hold on
+plot(100*[0 perturbations],100*[0,T_sum_SFSI]);
+xlabel('Pecentage Noise (%) ');
+ylabel('Total Runtime (sec)')
+set(gca, 'FontSize', 22);
+legend('MFAI','MFSI','SFAI','SFSI');

code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/adversarialReachability.m

@@ -0,0 +1,65 @@
+function [robust,T] = adversarialReachability(varargin)
+%% the function adversarialReachability takes the following inputs
+%   nnvnet  : NNV supported neural network model
+%   dataset : dataset prepared by the 'createSOCDataset' function
+%   percent : the input perturbations as percentage for different noises
+%   noiseType   : type of Noise
+%   randFeature : the particular feature we want to introduce the noise in
+
+%   and provides the following outputs as a result of approx-star
+%   reachability analysis
+%   robust  :   local robustness for each of the input audio sequence
+%   PR      :   the percentage sample robustness value
+%   T_avg   :   the avg time for the reachability calculatiom
+%   T_sum   : total time for the reachability calculation
+
+    reachOptions.reachMethod = 'exact-star';
+    switch nargin
+        case 7
+            nnvnet = varargin(1);
+            input = varargin(2);
+            index = varargin(3);
+            percent = varargin(4);
+            noiseType = varargin(5);
+            classIndex = varargin(6);
+            randFeature = varargin(7);
+        case 6
+            nnvnet = varargin(1);
+            input = varargin(2);
+            index = varargin(3);
+            percent = varargin(4);
+            noiseType = varargin(5);
+            classIndex = varargin(6);
+        otherwise
+            error('Invalid number of inputs, should be 6 or 7');
+    end
+    percent = percent{1,1};
+    nnvnet = nnvnet{1,1};
+    randFeature = index{1,1};
+    ip = input{1,1};
+    classIndex = classIndex{1,1};
+    lb = ip;
+    ub = ip;
+    if strcmp(noiseType{1,1},'SFSI')
+        XmuNorm = mean(ip(randFeature,:));
+        lb(randFeature,end) = lb(randFeature, end)-XmuNorm*percent;
+        ub(randFeature,end) = ub(randFeature, end)+XmuNorm*percent;
+    elseif strcmp(noiseType{1,1},'SFAI')
+        XmuNorm = mean(ip(randFeature,:));
+        lb(randFeature,:) = lb(randFeature,:)-XmuNorm*percent;
+        ub(randFeature,:) = ub(randFeature,:)+XmuNorm*percent;
+    elseif strcmp(noiseType{1,1},'MFSI')
+        XmuNorm = mean(ip(:,:)')';
+        lb(:,end) = lb(:, end)-XmuNorm*percent;
+        ub(:,end) = ub(:, end)+XmuNorm*percent;
+    elseif strcmp(noiseType{1,1},'MFAI')
+        XmuNorm = mean(ip(:,:)')';
+        lb = lb-XmuNorm*percent;
+        ub = ub+XmuNorm*percent;
+    end
+    IM = ImageStar(lb,ub);
+    start_time = tic;
+    R = nnvnet.reachSequence(IM, reachOptions);
+    T = toc(start_time);
+    robust = nnvnet.checkRobust(R,classIndex);  
+end

code/nnv/examples/Submission/FMAS2023/japanesevowelDataset/lstm model/calculateRobustnessMeasures.m

@@ -0,0 +1,21 @@
+percent = [0.5, 0.6, 0.7, 0.8, 0.9];
+nnvnet = matlab2nnv(net);
+YTest = double(YTest_correct);
+for j = 1: length(percent)
+    for i = 1:length(XTest_correct)
+        input = XTest_correct{i,1};
+        classIndex = YTest(i);
+        [robust_SFSI(j,i),T_SFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFSI",classIndex);
+        [robust_SFAI(j,i),T_SFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"SFAI",classIndex);
+        [robust_MFSI(j,i),T_MFSI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFSI",classIndex);
+        [robust_MFAI(j,i),T_MFAI(j,i)] = adversarialReachability(nnvnet,input,1,percent(j),"MFAI",classIndex);
+    end
+    PR_SFSI(1,j) = sum(robust_SFSI(j,:)==1)/length(XTest_correct);
+    T_sum_SFSI(1,j) = sum(T_SFSI(j,:));
+    PR_SFAI(1,j) = sum(robust_SFAI(j,:)==1)/length(XTest_correct);
+    T_sum_SFAI(1,j) = sum(T_SFAI(j,:));
+    PR_MFSI(1,j) = sum(robust_MFSI(j,:)==1)/length(XTest_correct);
+    T_sum_MFSI(1,j) = sum(T_MFSI(j,:));
+    PR_MFAI(1,j) = sum(robust_MFAI(j,:)==1)/length(XTest_correct);
+    T_sum_MFAI(1,j) = sum(T_MFAI(j,:));
+end