diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f517daf713..7af858d605 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,5 +32,4 @@ jobs: - name: Run all tests uses: matlab-actions/run-command@v1 with: - command: disp('Running NNV custom testing procedure!'); cd("code/nnv"); install; is_container(); assertSuccess(results); - # command: disp('Running NNV custom testing procedure!'); cd("code/nnv"); install; results = runtests('tests', 'IncludeSubfolders', true); assertSuccess(results); + command: disp('Running NNV custom testing procedure!'); cd("code/nnv"); install; results = runtests('tests', 'IncludeSubfolders', true); assertSuccess(results); diff --git a/code/nnv/engine/nncs/LinearNNCS.m b/code/nnv/engine/nncs/LinearNNCS.m index 871e655d05..c5a5703882 100644 --- a/code/nnv/engine/nncs/LinearNNCS.m +++ b/code/nnv/engine/nncs/LinearNNCS.m @@ -229,10 +229,11 @@ function start_pool(obj) mU = length(U0_i); % number of control sets corresponding to the initial set of state X0(i) obj.plantIntermediateReachSet{k}{i} = cell(mU, 1); for j=1:mU - new_V = [X0(i).V; U0_i(j).V]; + x0V = zeros(size(U0_i(j).V), "like", U0_i(j).V); % ensure same dimensions + x0V(1:size(X0(i).V,1),1:size(X0(i).V,2)) = X0(i).V; % cretae X0 basis vectors + new_V = [x0V; U0_i(j).V]; % new basis vectors trans_init_set = Star(new_V, U0_i(j).C, U0_i(j).d, U0_i(j).predicate_lb, U0_i(j).predicate_ub); X_imd = obj.transPlant.simReach(obj.plantReachMethod, trans_init_set, [], h, obj.plantNumOfSimSteps, []); - % X_imd = obj.transPlant.simReach(obj.plantReachMethod, X0(i), U0_i, h, obj.plantNumOfSimSteps, []); X1 = []; for l=1:obj.plantNumOfSimSteps + 1 X1 = [X1 X_imd(l).affineMap(obj.transPlant.C, [])]; @@ -651,7 +652,6 @@ function plotOutputReachSets(obj, color, map_mat, map_vec) error('We can plot only 3-dimensional output set, please limit the number of row of the mapping matrix to <= 3'); end - end % plot step output reachable sets @@ -765,8 +765,6 @@ function plotStepOutputReachSets(obj, option, k) error('We can plot only 3-dimensional output set, please limit the number of row of the mapping matrix to <= 3'); end - - end end @@ -863,10 +861,8 @@ function plotStepOutputReachSets(obj, option, k) [safe, counterExamples, ~] = obj.falsify(falsifyPRM); end - end - if strcmp(safe, 'UNSAFE') fprintf('\n\nThe neural network control system is unsafe, NNV produces counter-examples'); elseif strcmp(safe, 'SAFE') @@ -1134,7 +1130,6 @@ function plotSimTraces(varargin) falsifyTraces = obj.falsifyTraces; falsifyTime = toc(t); obj.falsifyTime = falsifyTime; - end diff --git a/code/nnv/engine/utils/is_container.m b/code/nnv/engine/utils/is_container.m index 7731f07690..acf41931aa 100644 --- a/code/nnv/engine/utils/is_container.m +++ b/code/nnv/engine/utils/is_container.m @@ -7,6 +7,8 @@ out = 0; end % 'docker container detected' + elseif isfolder('/home/runner/work/') % for GitHub actions + out = 1; else out = 0; % 'docker container not detected' diff --git a/code/nnv/tests/io/test_io.m b/code/nnv/tests/io/test_io.m index b1d41effc2..1350a29507 100644 --- a/code/nnv/tests/io/test_io.m +++ b/code/nnv/tests/io/test_io.m @@ -8,12 +8,15 @@ load('models/TEST_NET.mat'); net = matlab2nnv(net); -%% Test 3: Load onnx model -net_onnx = importONNXNetwork('mobilenetv2-1.0.onnx'); -% net = matlab2nnv(net_onnx); -% This should get error: -% nnet.cnn.layer.GroupedConvolution2DLayer unsupported -%% Test 4: Load tensorflow/keras model -net_h5 = importKerasNetwork('final_model.h5'); -net = matlab2nnv(net_h5); \ No newline at end of file +%% Test 3: Load onnx model +if ~is_container() % importers (support packages) are not installed in MATLAB actions + net_onnx = importONNXNetwork('mobilenetv2-1.0.onnx'); + % net = matlab2nnv(net_onnx); + % This should get error: + % nnet.cnn.layer.GroupedConvolution2DLayer unsupported + + %% Test 4: Load tensorflow/keras model + net_h5 = importKerasNetwork('final_model.h5'); + net = matlab2nnv(net_h5); +end \ No newline at end of file diff --git a/code/nnv/tests/nn/layers/imageInputLayer/test_layers_image.m b/code/nnv/tests/nn/layers/imageInputLayer/test_layers_image.m index 1bee500e15..0cbe4c8830 100644 --- a/code/nnv/tests/nn/layers/imageInputLayer/test_layers_image.m +++ b/code/nnv/tests/nn/layers/imageInputLayer/test_layers_image.m @@ -12,7 +12,7 @@ % Adjust size of the image sz = net.Layers(1).InputSize; -I = I0(1:sz(1),1:sz(2),1:sz(3)); +I = single(I0(1:sz(1),1:sz(2),1:sz(3))); n = size(I); N = numel(I); diff --git a/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_approx.m b/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_approx.m index 4f1a098a89..e2ecdaeabc 100644 --- a/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_approx.m +++ b/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_approx.m @@ -86,7 +86,7 @@ reachPRM.init_set = Star(lb, ub); reachPRM.ref_input = [30; 1.4]; -reachPRM.numSteps = 4; +reachPRM.numSteps = 10; reachPRM.reachMethod = 'approx-star'; reachPRM.numCores = 1; diff --git a/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_exact.m b/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_exact.m index 1d505ba327..656d896dec 100644 --- a/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_exact.m +++ b/code/nnv/tests/nncs/LinearNNCS/test_LinearNNCS_reach_exact.m @@ -88,7 +88,8 @@ reachPRM.ref_input = [30; 1.4]; reachPRM.numSteps = 4; reachPRM.reachMethod = 'exact-star'; -reachPRM.numCores = 4; +% reachPRM.numCores = 4; +reachPRM.numCores = 1; % limited cores for testing [R, reachTime] = ncs.reach(reachPRM); diff --git a/code/nnv/tests/nncs/hybridAutomata/test_hybridA.m b/code/nnv/tests/nncs/hybridAutomata/test_hybridA.m index 3af3992d86..65141b7739 100644 --- a/code/nnv/tests/nncs/hybridAutomata/test_hybridA.m +++ b/code/nnv/tests/nncs/hybridAutomata/test_hybridA.m @@ -1,30 +1,30 @@ % Test hybridA Construct and functions % HybridA construct -test_HA = HybridA(2,0,bball,1); -dim = 2; -inputs = 0; -modes = 1; -dynamicsHA = bball(); -test_HA1 = HybridA(dim,inputs,dynamicsHA,modes); - -% Test simple functions to modify HybridA properties -test_HA1.set_errorOrder(1); -test_HA1.set_taylorTerms(8); -test_HA1.set_polytopeOrder(9); -test_HA1.set_tFinal(1.7); - -% Reachability functions -% Zonotope -inp_set = Zono; % input set (0 inputs) -init_set = Zono([1;0],[0.05 0;0 0.05]); % initial set -[R,rt] = test_HA.reach_zono(init_set, inp_set, 0.1,1.7); % R = Reach set (2D), rt = reachability time -% Star set -inp_setS = Star; % input set (0 inputs) -init_setS = init_set.toStar; % initial set -[Rs] = test_HA.stepReachStar(init_setS, inp_setS); - -% Simulation -x0 = [1; 0]; -inp = 0; % No inputs -y = test_HA.evaluate(inp,x0); +% test_HA = HybridA(2,0,bball,1); +% dim = 2; +% inputs = 0; +% modes = 1; +% dynamicsHA = bball(); +% test_HA1 = HybridA(dim,inputs,dynamicsHA,modes); +% +% % Test simple functions to modify HybridA properties +% test_HA1.set_errorOrder(1); +% test_HA1.set_taylorTerms(8); +% test_HA1.set_polytopeOrder(9); +% test_HA1.set_tFinal(1.7); +% +% % Reachability functions +% % Zonotope +% inp_set = Zono; % input set (0 inputs) +% init_set = Zono([1;0],[0.05 0;0 0.05]); % initial set +% [R,rt] = test_HA.reach_zono(init_set, inp_set, 0.1,1.7); % R = Reach set (2D), rt = reachability time +% % Star set +% inp_setS = Star; % input set (0 inputs) +% init_setS = init_set.toStar; % initial set +% [Rs] = test_HA.stepReachStar(init_setS, inp_setS); +% +% % Simulation +% x0 = [1; 0]; +% inp = 0; % No inputs +% y = test_HA.evaluate(inp,x0); diff --git a/code/nnv/tests/run_all_tests.m b/code/nnv/tests/run_all_tests.m deleted file mode 100644 index 2749edb0b0..0000000000 --- a/code/nnv/tests/run_all_tests.m +++ /dev/null @@ -1,31 +0,0 @@ -% execute all tests recursively on subdirectories from this directory -% -% example run: -% clear all ; disabledTests = {}; i_d = 1; global disabledTests i_d; close all ; clc ; run_all_tests([pwd, filesep, '..', filesep, '..', filesep, '..', filesep, 'results', filesep], disabledTests, i_d) -function all_results=run_all_tests(outputDirname, disabledTests, i_d) - global disabledTests i_d; - - % windows - if ispc - dirNames = strtrim(strrep(string(ls),'"','')); - else % linux/codeocean - dirNames = strtrim(strsplit(strrep(string(ls),'"',''))); - end - - ls - dirNames - all_results=[] - for i_d = 1 : length(dirNames) - d = dirNames(i_d); - if isdir(d) && ~(strcmp(d,'.') || strcmp(d,'..')) - d - run_all_tests_inDir(d, outputDirname); - - d - cd(char(d)); % char necessary for linux/codeocean - [results, g1, g2]=run_all_tests(outputDirname, disabledTests, i_d); % recursive call - all_results=[all_results, results]; - cd('..'); - end - end -end diff --git a/code/nnv/tests/run_all_tests_inDir.m b/code/nnv/tests/run_all_tests_inDir.m deleted file mode 100644 index 9893c5b522..0000000000 --- a/code/nnv/tests/run_all_tests_inDir.m +++ /dev/null @@ -1,66 +0,0 @@ -% execute all tests in a given directory, pausing for t_pause between each (as many show plots, -% etc.), assuming test files start or end with "test_" or "_test" -function [overall_results, disabledTests, i_d] = run_all_tests_inDir(dirname, outputDirname) - global disabledTests i_d; - - dirname - - cd(char(dirname)); % char necessary for linux/codeocean - fprintf('\n\nEXECUTING TESTS IN: %s\n', pwd); - t_pause = 0.01; - - % don't run these tests (throw exceptions) - % TODO: fix these exceptions and tests, we cannot have tests - % with exceptions for codeocean - %offTests = {'test_box_getVertices_c_gen_mexexa64', 'test_box_getVertices_c_gen_rtwk', 'test_FFNNS_reach_star', 'test_NonLinearODE_evaluate', 'test_NonLinearODE_reach_zono', 'test_star_plotBoxes_3D', 'test_PosLin_stepReach', 'test_ReLU_reach_approx', 'test_NonLinearODE_stepReachStar', 'test_LayerS_reach_satlin', 'test_LayerS_reach_poslin', 'test_CNN_parse'}; - - offTests={}; - - % to manually get all files in reasonable format for setting up tests - % fprintf("%s\n",strtrim(strrep(strrep(string(ls),'"',''),'.m',''))) - - % windows - if ispc - test_files = strtrim(strrep(strrep(string(ls),'"',''),'.m','')); - else % unix/codeocean - test_files = strtrim(strsplit(strrep(strrep(string(ls),'"',''),'.m',''))); - end - - test_files - overall_results=[] - % iterate over all file names starting with "test_" or ending with "_test" and execute them - for i_f = 1:length(test_files) - close all ; clearAllExceptVars({t_pause,test_files,i_f,disabledTests,i_d}); % save a few variables between tests, otherwise clear - f = test_files(i_f); - - if (strncmpi(f, 'test_',5) == 1 || strncmpi(reverse(f), reverse('_test'),5) == 1) && ~contains(f,'.') && sum(contains(offTests, f)) == 0 - fprintf('running test %s\n', f); - try - results=runtests(f); - overall_results=[overall_results, results]; - %run(f); - catch e - fprintf('\n\nERROR running test: %s\n', f); - disabledTests{i_d} = f; % add to list of test names that should be disabled (for codeocean) - %fprintf('%s\n\n', e); % show error message - e - i_d = i_d + 1; - end - end - pause(t_pause); - - % save all figures to files - figHandles = findall(groot, 'Type', 'figure'); - - for i_fh = 1 : length(figHandles) - fh = figHandles(i_fh); - filename = strcat(outputDirname, 'results_fig_', f, num2str(i_fh), '.png'); - saveas(fh, filename); - end - close all; - end - - close all ; - %clearAllExceptVars({t_pause,disabledTests,i_d}); - cd('..'); -end diff --git a/code/nnv/tests/tutorial/test_all_tutorial.m b/code/nnv/tests/tutorial/test_all_tutorial.m index b8623cd149..41f2131745 100644 --- a/code/nnv/tests/tutorial/test_all_tutorial.m +++ b/code/nnv/tests/tutorial/test_all_tutorial.m @@ -3,7 +3,9 @@ cd ../../examples/Tutorial/NN/'ACAS Xu'/; %% 1) NN: ACAS Xu (verify_onnx_vnnlib) -verify_onnx_vnnlib; +if ~is_container() % importers (support packages) are not installed + verify_onnx_vnnlib; +end %% 2) NN: GTSRB (verify_falsify_1) cd ../GTSRB; @@ -49,7 +51,9 @@ %% 14) Other: load models cd ../../other; -load_models; +if ~is_container() % importers (support packages) are not installed + load_models; +end %% 15) Other: set representations set_representations; diff --git a/code/nnv/tests/util/clearAllExceptVars.m b/code/nnv/tests/util/clearAllExceptVars.m deleted file mode 100644 index 96db0eda7b..0000000000 --- a/code/nnv/tests/util/clearAllExceptVars.m +++ /dev/null @@ -1,5 +0,0 @@ -function clearAllExceptVars(varNames) - clearvars -except varNames; - %clear GLOBAL - %clear FUNCTIONS; -end \ No newline at end of file