Implement a first basic version of end-to-end tests

.github/workflows/test.yml

@@ -0,0 +1,36 @@
+name: Unit Testing
+
+on:
+  push:
+    branches: '**'
+  pull_request:
+    branches: [ 'main', 'dev' ]
+
+jobs:
+  lint:
+    name: Run Unit Tests
+    runs-on: [ 'ubuntu-latest' ]
+    env:
+      AGENT_TOOLSDIRECTORY: /home/runner/actions-runner/_work/_tool/
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9.16'
+          cache: 'pip' # caching pip dependencies
+          cache-dependency-path: '**/requirements_full.txt'
+          token: ${{ secrets.QUARK_GH_GITHUB_COM_TOKEN }}
+
+      - name: Install Dependencies
+        run: pip install -r .settings/requirements_full.txt
+
+      - name: Run tests with unittest
+        run: python -m unittest discover .
+
+

tests/__init__.py

@@ -0,0 +1,20 @@
+#  Copyright 2021 The QUARK Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import sys
+
+PROJECT_PATH = os.getcwd()
+SOURCE_PATH = os.path.join(PROJECT_PATH, "src")
+sys.path.append(SOURCE_PATH)

tests/configs/invalid/TSP.yml

@@ -0,0 +1,26 @@
+application:
+  config:
+    nodes:
+    - 4
+    - 6
+  name: TSP
+  submodules:
+  - config: {}
+    name: GreedyClassicalTSPInvalid
+    submodules:
+    - config: {}
+      name: Local
+      submodules: []
+  - config: {}
+    name: ReverseGreedyClassicalTSP
+    submodules:
+    - config: {}
+      name: Invalid
+      submodules: []
+  - config: {}
+    name: RandomTSP
+    submodules:
+    - config: {}
+      name: Local
+      submodules: []
+repetitions: 2

tests/configs/valid/GenerativeModeling.yml

@@ -0,0 +1,124 @@
+application:
+  config:
+    n_qubits:
+    - 8
+  name: GenerativeModeling
+  submodules:
+  - config:
+      data_set:
+      - Stocks_2D
+    name: Continuous Data
+    submodules:
+    - config: {}
+      name: PIT
+      submodules:
+      - config:
+          depth:
+          - 1
+        name: CircuitCopula
+        submodules:
+        - config:
+            backend:
+            - aer_simulator_cpu
+            n_shots:
+            - 100
+          name: LibraryQiskit
+          submodules:
+          - config:
+              loss:
+              - KL
+              max_evaluations:
+              - 10
+              population_size:
+              - 5
+              pretrained:
+              - 'False'
+              sigma:
+              - 0.01
+            name: QCBM
+            submodules: []
+    - config: {}
+      name: MinMax
+      submodules:
+      - config:
+          depth:
+          - 1
+        name: CircuitStandard
+        submodules:
+        - config:
+            backend:
+            - aer_statevector_simulator_cpu
+            n_shots:
+            - 100
+          name: LibraryQiskit
+          submodules:
+          - config:
+              loss:
+              - NLL
+              max_evaluations:
+              - 10
+              population_size:
+              - 5
+              pretrained:
+              - 'False'
+              sigma:
+              - 0.01
+            name: QCBM
+            submodules: []
+      - config:
+          depth:
+          - 2
+        name: CircuitCardinality
+        submodules:
+        - config:
+            backend:
+            - aer_statevector_simulator_cpu
+            n_shots:
+            - 100
+          name: LibraryQiskit
+          submodules:
+          - config:
+              loss:
+              - KL
+              max_evaluations:
+              - 10
+              population_size:
+              - 5
+              pretrained:
+              - 'False'
+              sigma:
+              - 0.01
+            name: QCBM
+            submodules: []
+  - config:
+      train_size:
+      - 0.1
+      - 1.0
+    name: Discrete Data
+    submodules:
+    - config:
+        depth:
+        - 2
+      name: CircuitCardinality
+      submodules:
+      - config:
+          backend:
+          - aer_statevector_simulator_cpu
+          n_shots:
+          - 100
+        name: LibraryQiskit
+        submodules:
+        - config:
+            loss:
+            - KL
+            max_evaluations:
+            - 10
+            population_size:
+            - 5
+            pretrained:
+            - 'False'
+            sigma:
+            - 0.01
+          name: QCBM
+          submodules: []
+repetitions: 1

tests/configs/valid/PVC.yml

@@ -0,0 +1,35 @@
+application:
+  config:
+    seams:
+    - 2
+    - 3
+  name: PVC
+  submodules:
+  - config:
+      lagrange_factor:
+      - 1.0
+      - 1.25
+    name: QUBO
+    submodules:
+    - config:
+        number_of_reads:
+        - 250
+        - 500
+      name: Annealer
+      submodules:
+      - config: {}
+        name: Simulated Annealer
+        submodules: []
+  - config: {}
+    name: GreedyClassicalPVC
+    submodules:
+    - config: {}
+      name: Local
+      submodules: []
+  - config: {}
+    name: ReverseGreedyClassicalPVC
+    submodules:
+    - config: {}
+      name: Local
+      submodules: []
+repetitions: 2

tests/configs/valid/SAT.yml

@@ -0,0 +1,24 @@
+application:
+  config:
+    clvar_ratio_cons:
+    - 2
+    clvar_ratio_test:
+    - 2
+    max_tries:
+    - 100
+    problem_set:
+    - 0
+    variables:
+    - 10
+  name: SAT
+  submodules:
+  - config: {}
+    name: Direct
+    submodules:
+    - config: {}
+      name: ClassicalSAT
+      submodules:
+      - config: {}
+        name: Local
+        submodules: []
+repetitions: 2

tests/configs/valid/TSP.yml

@@ -0,0 +1,25 @@
+application:
+  config:
+    nodes:
+    - 6
+  name: TSP
+  submodules:
+  - config: {}
+    name: GreedyClassicalTSP
+    submodules:
+    - config: {}
+      name: Local
+      submodules: []
+  - config: {}
+    name: ReverseGreedyClassicalTSP
+    submodules:
+    - config: {}
+      name: Local
+      submodules: []
+  - config: {}
+    name: RandomTSP
+    submodules:
+    - config: {}
+      name: Local
+      submodules: []
+repetitions: 2

tests/testMain.py

@@ -0,0 +1,50 @@
+#  Copyright 2021 The QUARK Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import glob
+import argparse
+import unittest
+from main import handle_benchmark_run, create_benchmark_parser
+
+
+class TestMain(unittest.TestCase):
+
+    def test_handle_valid_benchmark_run(self) -> None:
+        """
+        Test a couple of valid QUARK configs
+        :return:
+        :rtype: None
+        """
+        parser = argparse.ArgumentParser()
+        create_benchmark_parser(parser)
+
+        for filename in glob.glob("tests/configs/valid/**.yml"):
+            with self.subTest(msg=f"Running Benchmark Test with valid config {filename}", filename=filename):
+                args = parser.parse_args(["--config", filename, "--failfast"])
+                self.assertEqual(handle_benchmark_run(args), None)
+
+    def test_handle_invalid_benchmark_run(self) -> None:
+        """
+        Test a couple of invalid QUARK configs
+        :return:
+        :rtype: None
+        """
+        parser = argparse.ArgumentParser()
+        create_benchmark_parser(parser)
+
+        for filename in glob.glob("tests/configs/invalid/**.yml"):
+            with self.subTest(msg=f"Running Benchmark Test with invalid config {filename}", filename=filename):
+                args = parser.parse_args(["--config", filename, "--failfast"])
+                with self.assertRaises(Exception):
+                    handle_benchmark_run(args)