Merge pull request #281 from yash-ni/testing-suite

Resolved an issue where the system-installed Python was mistakenly utilized instead of the venv Python

tests/New_ATS/CreatePythonVirtualEnv.bat

@@ -11,6 +11,7 @@ IF NOT exist %script_dir%\venv (
     echo Creating Virtual Environment at \venv
     call python -m venv %script_dir%\venv
     echo Installing grpcio-tools into Virtual Environment
+    call %script_dir%\venv\Scripts\python.exe -m pip install --upgrade pip
     call %script_dir%\venv\Scripts\python.exe -m pip install grpcio-tools pytest
     echo Successfully Installed Virtual Environment
 ) ELSE (

tests/New_ATS/RunPythonClient.bat

@@ -5,5 +5,4 @@ set script_dir=%CD%
 popd
 
 echo Running Python Client
-echo %script_dir%\venv\Scripts\python.exe -m pytest %python_client_path% -vv
 call %script_dir%\venv\Scripts\python.exe -m pytest %python_client_path% -vv

tests/New_ATS/pylib/run_tests.py

@@ -3,12 +3,32 @@
 import shutil
 import subprocess
 import os
+import re
+
+FAILED = 0
+
+def count_failed_testcases(test_summary):
+    pattern = r'(\d+) failed(?=, \d+ passed)'
+    matches = re.findall(pattern, test_summary)
+    count = 0
+    if matches:
+        count = int(matches[-1])
+    return count
+
+
+def run_command(command):
+    output = subprocess.run(command, capture_output=True, text=True)
+    if output.stderr:
+        raise Exception(output.stderr)
+    return output.stdout
+
 
 def check_for_pre_requisites(test_config):
     # check if LabVIEW CLI is installed
     if not test_config["lvcli_path"].exists():
         raise Exception(f'LabVIEW CLI is not installed at {test_config["lvcli_path"]}')
 
+
 def generate_server(test_config):
     # 1. Delete the Generated_server folder. TODO: Take a boolean in the config to say whether the build should be a clean build
     if test_config['generated_server'].exists():
@@ -26,10 +46,12 @@ def generate_server(test_config):
         f"{test_config['proto_path']}",
         f"{test_config['project_path']}",
         f"{test_config['gen_type']}"])
-    subprocess.run(CLI_command)
+    run_command(CLI_command)
 
 
 def run_test(test_config):
+    global FAILED
+
     # 1. Check for pre_requisites
     check_for_pre_requisites(test_config)
 
@@ -47,7 +69,7 @@ def run_test(test_config):
     shutil.copyfile(start_sync_impl_path, start_sync_gen_path)
 
     # 5. Quit LabVIEW if it is running
-    subprocess.run(['taskkill', '/f', '/im', 'labview.exe'])
+    run_command(['taskkill', '/f', '/im', 'labview.exe'])
 
     # 6. Start Run Service.vi from command prompt by launching labview.exe form lv_folder with the following arguments:
     # this must be non-blocking
@@ -62,39 +84,36 @@ def run_test(test_config):
         f"{test_config['test_folder']}"])
 
     # TODO Check whether labviewCLI is installed or not before running the command
-    subprocess.run(CLI_command)
+    run_command(CLI_command)
 
     # 7. Create python virtual environment
-    run_command = ' '.join([
+    CLI_command = ' '.join([
         str(test_config['test_suite_folder'] / 'CreatePythonVirtualEnv.bat')
     ])
-    subprocess.run(run_command)
+    run_command(CLI_command)
 
     # 7. Generate python grpc classes
     generate_command = ' '.join([
-        "python -m grpc_tools.protoc",
+        f"{test_config['python_path']} -m grpc_tools.protoc",
         f"--proto_path={test_config['test_folder']}",
         f"--python_out={test_config['test_folder']}",
         f"--pyi_out={test_config['test_folder']}",
         f"--grpc_python_out={test_config['test_folder']}",
         f"{test_config['test_name']}.proto"
     ])
-    subprocess.run(generate_command)
+    run_command(generate_command)
 
-    # 8. Call the TestServer() from test_folder/test_name_client.py and get the return value
+    # 8. Call the TestServer() from test_folder/test_name_client.py
     client_py_path = test_config['test_folder'] / str(test_config['test_name'] + '_client.py')
-    run_command = ' '.join([
+    run_client_command = ' '.join([
         str(test_config['test_suite_folder'] / 'RunPythonClient.bat'),
         str(client_py_path)])
-    return_value = subprocess.run(run_command)
-
-    if return_value.returncode == 0:
-        print(f'{test_config["test_name"]} passed')
-    else:
-        print(f'{test_config["test_name"]} failed')
+    output = run_command(run_client_command)
+    print(output)
+    FAILED += count_failed_testcases(output)
 
     # 8. Quit LabVIEW if it is running
-    subprocess.run(['taskkill', '/f', '/im', 'labview.exe'])
+    run_command(['taskkill', '/f', '/im', 'labview.exe'])
 
     # 9. Delete python grpc generated files
     # for filename in os.listdir(test_config['test_folder']):
@@ -104,6 +123,7 @@ def run_test(test_config):
 
 # Desc: Run the tests in the testlist.json file
 def main():
+    global FAILED
     # read the list of tests from testlist.json
     test_list_json_path = pathlib.Path(__file__).parent.absolute() / 'testlist.json'
     with open(test_list_json_path) as f:
@@ -123,6 +143,7 @@ def main():
             test_config['test_suite_pylib_folder'] = pathlib.Path(__file__).parent.absolute()
             test_config['test_suite_folder'] = test_config['test_suite_pylib_folder'].parent.absolute()
             test_config['tests_folder'] = test_config['test_suite_folder'] / 'Tests'
+            test_config['python_path'] = test_config['test_suite_folder'] / 'venv' / "Scripts" / "python.exe"
             tests_folder = test_config['tests_folder']
             # locals
             for test_name in test['name']:
@@ -136,7 +157,8 @@ def main():
                 test_config['impl'] = test_config['test_folder'] / 'Impl'
                 test_config['gen_type'] = gen_type
                 run_test(test_config)
-
+            if FAILED:
+                raise Exception(f"{FAILED} test cases have failed. Please review the above results")            
 
 if __name__ == '__main__':
     main()