From 9754a402a1461a244110dc515c81fdb8f01271be Mon Sep 17 00:00:00 2001 From: Diego Manzanas Date: Thu, 11 Jul 2024 17:04:38 -0500 Subject: [PATCH] Stuborn --- .../Submission/VNN_COMP2024/execute.asv | 128 ++++++++++++++++++ .../Submission/VNN_COMP2024/execute.py | 26 ++-- .../VNN_COMP2024/prepare_instance.sh | 4 +- .../instance_1.txt | 1 + .../VNN_COMP2024/run_all_instances_from_csv.m | 4 +- .../Submission/VNN_COMP2024/run_instance.sh | 2 +- .../VNN_COMP2024/run_vnncomp2024_instance.m | 9 +- 7 files changed, 159 insertions(+), 15 deletions(-) create mode 100644 code/nnv/examples/Submission/VNN_COMP2024/execute.asv create mode 100644 code/nnv/examples/Submission/VNN_COMP2024/results_approx_tinyimagenet/instance_1.txt diff --git a/code/nnv/examples/Submission/VNN_COMP2024/execute.asv b/code/nnv/examples/Submission/VNN_COMP2024/execute.asv new file mode 100644 index 000000000..413443adb --- /dev/null +++ b/code/nnv/examples/Submission/VNN_COMP2024/execute.asv @@ -0,0 +1,128 @@ +"""Launch MATLAB Engines with Python + +This module was created for local testing of reachability analysis (https://github.com/verivital/nnv.git) ahead of VNN-COMP 2023. +To perform testing, the following functionality is enabled. + +- Start a MATLAB engine. +- Run an instance of reachability analysis given some arguments through the open MATLAB engine. + +Arguments can be passed to the call to the script and will be parsed accordingly. +""" + + +import sys +import matlab.engine +import time +import os + + +def prepare_instance(category: str, onnx: str, vnnlib: str) -> None: + """Set up the MATLAB engine for running an instance. + + Parameters: + onnx (str): the path to the .onnx file + vnnlib (str): the path to the .vnnlib file + """ + + print("We should not be here...") + + +def run_instance(category, onnx, vnnlib, timeout, outputlocation) -> None: + """Run an instance based on parameters defined in .csv file. + + Parameters: + onnx (str): the path to the .onnx file + vnnlib (str): the path to the .vnnlib file + timeout (int): the time (in ms) to wait before proceeding to the next instance + """ + + print("Looking for connections") + eng_name = matlab.engine.find_matlab() + print(eng_name) + try: + eng = matlab.engine.connect_matlab(eng_name[0]) + print(eng) + except: + print("Connect to anythin + eng = matlab.engine + + # print(eng) + + # eng = matlab.engine.start_matlab() + + # eng.addpath(os.getcwd()) + # eng.addpath(eng.genpath('/home/ubuntu/toolkit/code/nnv/')) + # print("Paths added"); + # eng.addpath(eng.genpath('/root/Documents/MATLAB/SupportPackages/R2024a')) # This is where the support packages get installed from mpm + + status = 2 #initialize with an 'Unknown' status + future = eng.run_vnncomp2024_instance(category, onnx, vnnlib, outputlocation, nargout = 2, background=True) + + print("future initiated") + + timeout = float(timeout) + print('Trying to get the results without specified timeout') + + # time.sleep(timeout) # wait until timeout everytime? + + # [status, total_time] = future.result() + + try: + [status, total_time] = future.result(timeout) + #print('extra time = ',int(toc-tic)) + except matlab.engine.TimeoutError: + print("timeout") + #print('extra time = ',int(toc-tic)) + total_time = timeout + status = 3 + + future.cancel() + eng.quit() + + if status == 3: + resultfile = outputlocation + with open(resultfile, 'w') as f: + f.write('timeout') + # All the other results are written from matlab + + +def _get_args() -> None: + """Get the arguments passed to the script from the command line. + + Expected usage is : [ACTION, PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION] + """ + args = sys.argv[1:] + ACTION = args[0] + + # prepare_instance expects: benchmark_category, onnx, vnnlib + if (ACTION == 'prepare_instance'): + if len(args) != 4: + raise ValueError(f'Incorrect number of arguments, expected 4 got {len(args)}.') + args.append(None) # timeout + args.append(None) # outputlocation + + # run_instance expects: benchmark_category, onnx, vnnlib, timeout, outputlocation + if (ACTION == 'run_instance'): + if len(args) != 6: + raise ValueError(f'Incorrect number of arguments, expected 6 got {len(args)}.') + + print(args) + + return args + + +if __name__=="__main__": + # parse the arguments. + ACTION, CATEGORY, PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION = _get_args() + + # implement logic for each action we might want to take. + switcher = { + 'prepare_instance': lambda: prepare_instance(CATEGORY, PATH_TO_ONNX, PATH_TO_VNNLIB), # prepare_instance(PATH_TO_ONNX, PATH_TO_VNNLIB), + 'run_instance': lambda: run_instance(CATEGORY, PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION) # run_instance(PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION), + } + + # retrieve the correct function call based on the input action. + func = switcher.get(ACTION, 'Invalid')() + + if func == 'Invalid': + raise ValueError(f'Incorrect ACTION. Expected one of {list(switcher.keys())}; instead got {ACTION}.') \ No newline at end of file diff --git a/code/nnv/examples/Submission/VNN_COMP2024/execute.py b/code/nnv/examples/Submission/VNN_COMP2024/execute.py index ed9db70ac..6182c87c7 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/execute.py +++ b/code/nnv/examples/Submission/VNN_COMP2024/execute.py @@ -36,16 +36,24 @@ def run_instance(category, onnx, vnnlib, timeout, outputlocation) -> None: timeout (int): the time (in ms) to wait before proceeding to the next instance """ - # print("Looking for connections") - # print(matlab.engine.find_matlab()) - # eng = matlab.engine.connect_matlab() + print("Looking for connections") + eng_name = matlab.engine.find_matlab() + print(eng_name) + try: + eng = matlab.engine.connect_matlab(eng_name[0]) + print(eng) + except: + print("Connect to anything") + eng = matlab.engine.connect_matlab() + print(eng) + # print(eng) - eng = matlab.engine.start_matlab() + # eng = matlab.engine.start_matlab() - # eng.addpath(os.getcwd()) - # eng.addpath(eng.genpath('/home/ubuntu/toolkit/code/nnv/')) - # print("Paths added"); + eng.addpath(os.getcwd()) + eng.addpath(eng.genpath('/home/ubuntu/toolkit/code/nnv/')) + print("Paths added"); # eng.addpath(eng.genpath('/root/Documents/MATLAB/SupportPackages/R2024a')) # This is where the support packages get installed from mpm status = 2 #initialize with an 'Unknown' status @@ -56,12 +64,12 @@ def run_instance(category, onnx, vnnlib, timeout, outputlocation) -> None: timeout = float(timeout) print('Trying to get the results without specified timeout') - time.sleep(10) # wait until timeout everytime? + # time.sleep(timeout) # wait until timeout everytime? # [status, total_time] = future.result() try: - [status, total_time] = future.result(timeout=float(timeout)) + [status, total_time] = future.result(timeout) #print('extra time = ',int(toc-tic)) except matlab.engine.TimeoutError: print("timeout") diff --git a/code/nnv/examples/Submission/VNN_COMP2024/prepare_instance.sh b/code/nnv/examples/Submission/VNN_COMP2024/prepare_instance.sh index 2d71c89b7..f396dd725 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/prepare_instance.sh +++ b/code/nnv/examples/Submission/VNN_COMP2024/prepare_instance.sh @@ -25,8 +25,10 @@ pkill -f matlab WAIT_FOR_CONNECTION_TO_OPEN='import matlab.engine\nimport time\nwhile not matlab.engine.find_matlab(): time.sleep(1)' -matlab -batch "p = parpool; p.IdleTimeout = 120; matlab.engine.shareEngine;" & +matlab -batch "p = parpool; p.IdleTimeout = 12000; matlab.engine.shareEngine;" & python3 -c "exec('$WAIT_FOR_CONNECTION_TO_OPEN')" +wait(20) # it takes about this long to start the parpool + exit 0 diff --git a/code/nnv/examples/Submission/VNN_COMP2024/results_approx_tinyimagenet/instance_1.txt b/code/nnv/examples/Submission/VNN_COMP2024/results_approx_tinyimagenet/instance_1.txt new file mode 100644 index 000000000..c0804f6cc --- /dev/null +++ b/code/nnv/examples/Submission/VNN_COMP2024/results_approx_tinyimagenet/instance_1.txt @@ -0,0 +1 @@ +Operands to the logical AND (&&) and OR (||) operators must be convertible to logical scalar values. Use the ANY or ALL functions to reduce operands to logical scalar values. diff --git a/code/nnv/examples/Submission/VNN_COMP2024/run_all_instances_from_csv.m b/code/nnv/examples/Submission/VNN_COMP2024/run_all_instances_from_csv.m index 58f84d7d9..4828281b8 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/run_all_instances_from_csv.m +++ b/code/nnv/examples/Submission/VNN_COMP2024/run_all_instances_from_csv.m @@ -17,7 +17,7 @@ }; % we don't really care much about this track, focus on the other one % for i=3:length(benchmarks) -for i=17 % only do acasxu +for i=19 % only do acasxu name_noyear = split(benchmarks(i).name, "_"); if length(name_noyear) > 1 @@ -39,7 +39,7 @@ results_dir = "results_approx_" + benchmarks(i).name; mkdir(results_dir); - for k=1:size(instances, 1) + for k=3:size(instances, 1) onnx = benchpath + filesep + instances(k,1); vnnlib = benchpath + filesep + instances(k,2); diff --git a/code/nnv/examples/Submission/VNN_COMP2024/run_instance.sh b/code/nnv/examples/Submission/VNN_COMP2024/run_instance.sh index 5e4a2229b..7a4d86b9f 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/run_instance.sh +++ b/code/nnv/examples/Submission/VNN_COMP2024/run_instance.sh @@ -30,4 +30,4 @@ echo $PYTHONPATH python3 /home/ubuntu/toolkit/code/nnv/examples/Submission/VNN_COMP2024/execute.py 'run_instance' "$CATEGORY" "$ONNX_FILE" "$VNNLIB_FILE" "$TIMEOUT" "$RESULTS_FILE" # sudo python3 execute.py 'run_instance' "$CATEGORY" "$ONNX_FILE" "$VNNLIB_FILE" "$TIMEOUT" "$RESULTS_FILE" -echo "" +# echo "" diff --git a/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m b/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m index 1c335365f..96167bb92 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m +++ b/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m @@ -58,7 +58,7 @@ warning("Working on adding support to other vnnlib properties"); end -cEX_time = toc(t); +cEX_time = toc(t) %% 3) UNSAT? @@ -470,8 +470,13 @@ net = importNetworkFromONNX(onnx, "InputDataFormats","BCSS", "OutputDataFormats","BC"); nnvnet = matlab2nnv(net); reachOptions = struct; - reachOptions.reachMethod = 'approx-star'; % default parameters + reachOptions.reachMethod = 'relax-star-area'; % default parameters + reachOptions.relaxFactor = 1; reachOptionsList{1} = reachOptions; + reachOptions = struct; + reachOptions.reachMethod = 'relax-star-area'; % default parameters + reachOptions.relaxFactor = 0.8; + reachOptionsList{2} = reachOptions; needReshape = 1; % elseif contains(category, "linearizenn")% we do not support the current version