Stuborn

code/nnv/examples/Submission/VNN_COMP2024/execute.asv

@@ -0,0 +1,128 @@
+"""Launch MATLAB Engines with Python
+
+This module was created for local testing of reachability analysis (https://github.com/verivital/nnv.git) ahead of VNN-COMP 2023.
+To perform testing, the following functionality is enabled.
+
+- Start a MATLAB engine.
+- Run an instance of reachability analysis given some arguments through the open MATLAB engine.
+
+Arguments can be passed to the call to the script and will be parsed accordingly.
+"""
+
+
+import sys
+import matlab.engine
+import time
+import os
+
+
+def prepare_instance(category: str, onnx: str, vnnlib: str) -> None:
+    """Set up the MATLAB engine for running an instance.
+
+    Parameters:
+        onnx   (str): the path to the .onnx file
+        vnnlib (str): the path to the .vnnlib file
+    """
+
+    print("We should not be here...")
+
+
+def run_instance(category, onnx, vnnlib, timeout, outputlocation) -> None:
+    """Run an instance based on parameters defined in .csv file.
+
+    Parameters:
+        onnx    (str): the path to the .onnx file
+        vnnlib  (str): the path to the .vnnlib file
+        timeout (int): the time (in ms) to wait before proceeding to the next instance
+    """
+
+    print("Looking for connections")
+    eng_name = matlab.engine.find_matlab()
+    print(eng_name)
+    try:
+        eng = matlab.engine.connect_matlab(eng_name[0])
+        print(eng)
+    except:
+        print("Connect to anythin
+        eng = matlab.engine
+
+    # print(eng)
+
+    # eng = matlab.engine.start_matlab()
+
+    # eng.addpath(os.getcwd())
+    # eng.addpath(eng.genpath('/home/ubuntu/toolkit/code/nnv/'))
+    # print("Paths added");
+    # eng.addpath(eng.genpath('/root/Documents/MATLAB/SupportPackages/R2024a')) # This is where the support packages get installed from mpm
+
+    status = 2 #initialize with an 'Unknown' status
+    future = eng.run_vnncomp2024_instance(category, onnx, vnnlib, outputlocation, nargout = 2, background=True)
+
+    print("future initiated")
+
+    timeout = float(timeout)
+    print('Trying to get the results without specified timeout')
+
+    # time.sleep(timeout) # wait until timeout everytime?
+
+    # [status, total_time] = future.result()
+
+    try: 
+        [status, total_time] = future.result(timeout)
+        #print('extra time = ',int(toc-tic))
+    except matlab.engine.TimeoutError:
+        print("timeout")
+        #print('extra time = ',int(toc-tic))
+        total_time = timeout
+        status = 3
+
+    future.cancel()
+    eng.quit() 
+
+    if status == 3:
+        resultfile = outputlocation
+        with open(resultfile, 'w') as f:
+            f.write('timeout')
+    # All the other results are written from matlab
+
+
+def _get_args() -> None:
+    """Get the arguments passed to the script from the command line.
+
+    Expected usage is : [ACTION, PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION]
+    """
+    args = sys.argv[1:]
+    ACTION = args[0]
+
+    # prepare_instance expects: benchmark_category, onnx, vnnlib
+    if (ACTION == 'prepare_instance'):
+        if len(args) != 4:
+            raise ValueError(f'Incorrect number of arguments, expected 4 got {len(args)}.')
+        args.append(None) # timeout
+        args.append(None) # outputlocation
+
+    # run_instance expects: benchmark_category, onnx, vnnlib, timeout, outputlocation
+    if (ACTION == 'run_instance'):
+        if len(args) != 6:
+            raise ValueError(f'Incorrect number of arguments, expected 6 got {len(args)}.')
+
+    print(args)
+
+    return args
+
+
+if __name__=="__main__":
+    # parse the arguments.
+    ACTION, CATEGORY, PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION = _get_args()
+
+    # implement logic for each action we might want to take.
+    switcher = {
+        'prepare_instance': lambda: prepare_instance(CATEGORY, PATH_TO_ONNX, PATH_TO_VNNLIB), # prepare_instance(PATH_TO_ONNX, PATH_TO_VNNLIB),
+        'run_instance': lambda: run_instance(CATEGORY, PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION) # run_instance(PATH_TO_ONNX, PATH_TO_VNNLIB, TIMEOUT, OUTPUTLOCATION),
+    }
+
+    # retrieve the correct function call based on the input action.
+    func = switcher.get(ACTION, 'Invalid')()
+
+    if func == 'Invalid':
+        raise ValueError(f'Incorrect ACTION. Expected one of {list(switcher.keys())}; instead got {ACTION}.')

code/nnv/examples/Submission/VNN_COMP2024/execute.py

@@ -36,16 +36,24 @@ def run_instance(category, onnx, vnnlib, timeout, outputlocation) -> None:
         timeout (int): the time (in ms) to wait before proceeding to the next instance
     """
 
-    # print("Looking for connections")
-    # print(matlab.engine.find_matlab())
-    # eng = matlab.engine.connect_matlab()
+    print("Looking for connections")
+    eng_name = matlab.engine.find_matlab()
+    print(eng_name)
+    try:
+        eng = matlab.engine.connect_matlab(eng_name[0])
+        print(eng)
+    except:
+        print("Connect to anything")
+        eng = matlab.engine.connect_matlab()
+        print(eng)
+
     # print(eng)
 
-    eng = matlab.engine.start_matlab()
+    # eng = matlab.engine.start_matlab()
 
-    # eng.addpath(os.getcwd())
-    # eng.addpath(eng.genpath('/home/ubuntu/toolkit/code/nnv/'))
-    # print("Paths added");
+    eng.addpath(os.getcwd())
+    eng.addpath(eng.genpath('/home/ubuntu/toolkit/code/nnv/'))
+    print("Paths added");
     # eng.addpath(eng.genpath('/root/Documents/MATLAB/SupportPackages/R2024a')) # This is where the support packages get installed from mpm
 
     status = 2 #initialize with an 'Unknown' status
@@ -56,12 +64,12 @@ def run_instance(category, onnx, vnnlib, timeout, outputlocation) -> None:
     timeout = float(timeout)
     print('Trying to get the results without specified timeout')
 
-    time.sleep(10) # wait until timeout everytime?
+    # time.sleep(timeout) # wait until timeout everytime?
 
     # [status, total_time] = future.result()
 
     try: 
-        [status, total_time] = future.result(timeout=float(timeout))
+        [status, total_time] = future.result(timeout)
         #print('extra time = ',int(toc-tic))
     except matlab.engine.TimeoutError:
         print("timeout")

code/nnv/examples/Submission/VNN_COMP2024/prepare_instance.sh

@@ -25,8 +25,10 @@ pkill -f matlab
 
 WAIT_FOR_CONNECTION_TO_OPEN='import matlab.engine\nimport time\nwhile not matlab.engine.find_matlab(): time.sleep(1)'
 
-matlab -batch "p = parpool; p.IdleTimeout = 120; matlab.engine.shareEngine;" &
+matlab -batch "p = parpool; p.IdleTimeout = 12000; matlab.engine.shareEngine;" &
 
 python3 -c "exec('$WAIT_FOR_CONNECTION_TO_OPEN')"
 
+wait(20) # it takes about this long to start the parpool
+
 exit 0

code/nnv/examples/Submission/VNN_COMP2024/results_approx_tinyimagenet/instance_1.txt

@@ -0,0 +1 @@
+Operands to the logical AND (&&) and OR (||) operators must be convertible to logical scalar values. Use the ANY or ALL functions to reduce operands to logical scalar values.

code/nnv/examples/Submission/VNN_COMP2024/run_all_instances_from_csv.m

@@ -17,7 +17,7 @@
     }; % we don't really care much about this track, focus on the other one
 
 % for i=3:length(benchmarks)
-for i=17 % only do acasxu
+for i=19 % only do acasxu
 
     name_noyear = split(benchmarks(i).name, "_");
     if length(name_noyear) > 1
@@ -39,7 +39,7 @@
             results_dir = "results_approx_" + benchmarks(i).name;
             mkdir(results_dir);
 
-            for k=1:size(instances, 1)
+            for k=3:size(instances, 1)
 
                 onnx = benchpath + filesep + instances(k,1);
                 vnnlib = benchpath + filesep + instances(k,2);

code/nnv/examples/Submission/VNN_COMP2024/run_instance.sh

@@ -30,4 +30,4 @@ echo $PYTHONPATH
 python3 /home/ubuntu/toolkit/code/nnv/examples/Submission/VNN_COMP2024/execute.py 'run_instance' "$CATEGORY" "$ONNX_FILE" "$VNNLIB_FILE" "$TIMEOUT" "$RESULTS_FILE"
 # sudo python3 execute.py 'run_instance' "$CATEGORY" "$ONNX_FILE" "$VNNLIB_FILE" "$TIMEOUT" "$RESULTS_FILE"
 
-echo ""
+# echo ""

code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m

@@ -58,7 +58,7 @@
     warning("Working on adding support to other vnnlib properties");
 end
 
-cEX_time = toc(t);
+cEX_time = toc(t)
 
 %% 3) UNSAT?
 
@@ -470,8 +470,13 @@
         net = importNetworkFromONNX(onnx, "InputDataFormats","BCSS", "OutputDataFormats","BC");
         nnvnet = matlab2nnv(net);
         reachOptions = struct;
-        reachOptions.reachMethod = 'approx-star'; % default parameters
+        reachOptions.reachMethod = 'relax-star-area'; % default parameters
+        reachOptions.relaxFactor = 1;
         reachOptionsList{1} = reachOptions;
+        reachOptions = struct;
+        reachOptions.reachMethod = 'relax-star-area'; % default parameters
+        reachOptions.relaxFactor = 0.8;
+        reachOptionsList{2} = reachOptions;
         needReshape = 1;
 
     % elseif contains(category, "linearizenn")%  we do not support the current version