Merge pull request #22 from PixelgenTechnologies/fix/ci-integration-t…

…ests

Fix workflow tests on external action runners

.github/workflows/integration-tests.yml → .github/workflows/workflow-tests.yml

@@ -13,18 +13,13 @@ on:
       container-tag:
         description: 'Container tag'
         required: true
-        default: 'latest-dev'
+        default: 'dev'
       keep-workdirs:
         description: 'Keep working directories for debugging'
         required: false
         default: 'false'
-  push:
-    branches:
-      - main
-    tags:
-      - 'v*'
   schedule:
-    # 01:00 every day
+    # 01:00 every day, using last commit on default branch
     - cron: "0 1 * * *"
 
 jobs:
@@ -51,6 +46,7 @@ jobs:
                 - 'poetry.lock'
               paths_ignore:
                 - 'tests/**/*.md'
+
   setup:
     runs-on: ubuntu-latest
     needs: pre_job
@@ -74,19 +70,20 @@ jobs:
           mask-password: 'false'
 
   test:
-    runs-on: [ "self-hosted", "linux", "x64", "ubuntu-latest" ]
+    runs-on: [ "self-hosted", "Linux", "X64" ]
     needs: [setup]
     container:
-      image: 890888997283.dkr.ecr.eu-north-1.amazonaws.com/pixelator:${{ github.event.inputs.container-tag || 'latest-dev' }}
+      image: 890888997283.dkr.ecr.eu-north-1.amazonaws.com/pixelator:${{ github.event.inputs.container-tag || 'dev' }}
       credentials:
         username: ${{ needs.setup.outputs.docker_username }}
         password: ${{ needs.setup.outputs.docker_password }}
       volumes:
-        - /mnt/github-actions-data:/data
+        - /mnt/pixelgen-github-actions-data:/mnt/pixelgen-github-actions-data
+        - /mnt/pixelgen-technologies-datasets:/mnt/pixelgen-technologies-datasets
     strategy:
       fail-fast: false
       matrix:
-        test: [ "small", "single-cell", "tissue" ]
+        test: [ "test_small_D21", "test_full_D21" ]
 
     steps:
       - name: Install poetry
@@ -96,25 +93,18 @@ jobs:
           pipx install poetry
           poetry self add poetry-plugin-export
 
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Install dev dependencies
-        working-directory: /pixelator
         run: |
           poetry export -f requirements.txt --output requirements_only_dev.txt --only dev
           pip3 install -r requirements_only_dev.txt
 
-      - name: Prepare test case string
-        uses: actions/github-script@v6
-        id: test_case_file
-        with:
-          script: |
-            const test_name = "${{ matrix.test }}"
-            const test_file_name = `test_${test_name.replace(/-/g, '_')}.py`
-            core.setOutput('filename', test_file_name)
-
       - name: Test pixelator
         run: |
-          CMD="pytest /pixelator/tests/integration/${{ steps.test_case_file.outputs.filename }} --log-cli-level=DEBUG"
+          CMD="pytest --log-cli-level=DEBUG"
           if [[ "${{ github.event.inputs.keep-workdirs }}" == "true" ]]; then
             CMD="$CMD --keep-workdirs"
           fi
-          $CMD -m "workflow_test"
+          $CMD -m "workflow_test" -k "${{ matrix.test }}"

src/pixelator/test_utils/collector.py

@@ -1,30 +1,45 @@
-"""
+"""Custom pytest collector utilities to generate tests from yaml files.
+
 Copyright (c) 2023 Pixelgen Technologies AB.
 """
-
 import pytest
 
 from pixelator.test_utils import PixelatorWorkflowTest, WorkflowConfig
 
 
+class NoModuleClass(pytest.Class):  # noqa
+    """Mock pytest.Class that does not require a module as a parent."""
+
+    class MockModule:
+        """Fake module with a None obj."""
+
+        def __init__(self):  # noqa
+            self.obj = None
+
+    def getparent(self, cls):  # noqa
+        parent = super().getparent(cls)
+        if parent is None:
+            return self.MockModule()
+        return parent
+
+
 class YamlIntegrationTestsCollector(pytest.File):
-    """
-    Custom pytest collector generating tests from yaml files.
-    """
+    """Custom pytest collector generating tests from yaml files."""
 
-    def collect(self, parent_module=None):
-        # We need a yaml parser, e.g. PyYAML.
+    def collect(self):
+        """Convert yaml files to test cases."""
         config = WorkflowConfig(self.path)
 
         for case in config.keys():
             name = case.split("-")[1]
+            case_config = config.get_test_config(case)
             wf = type(
-                f"TestSmallWorkflow{name}",
+                f"TestWorkflow{name}",
                 (PixelatorWorkflowTest,),
-                {"test_id": case, **config.get_test_config(case)},
+                {"test_id": case, **case_config},
             )
 
-            collector = pytest.Class.from_parent(parent=self, name=name)
+            collector = NoModuleClass.from_parent(parent=self, name=self.path.stem)
             # Need to add this explicitly since the constructor does not set obj
             collector.obj = wf
 

tests/integration/external_tests/readme.md

@@ -0,0 +1 @@
+External workflow test YAML files can be mounted or symlinked here and will be run as part of the workflow test suite.

tests/integration/test_full_D21.yaml

@@ -0,0 +1,25 @@
+full-D21:
+  design: D21
+  panel: "human-sc-immunology-spatial-proteomics"
+  panel_file: null
+  input_files:
+    - /mnt/pixelgen-technologies-datasets/mpx-datasets/scsp/1.0/1k-human-pbmcs-v1.0-immunology-I/Sample01_human_pbmcs_unstimulated_R1_001.fastq.gz
+    - /mnt/pixelgen-technologies-datasets/mpx-datasets/scsp/1.0/1k-human-pbmcs-v1.0-immunology-I/Sample01_human_pbmcs_unstimulated_R2_001.fastq.gz
+  options:
+    common:
+      verbose: true
+    graph:
+      params: [
+        "--multiplet-recovery"
+      ]
+    annotate:
+      params: [
+        "--dynamic-filter", "min",
+        "--aggregate-calling"
+      ]
+
+    analysis:
+      params: [
+         "--compute-polarization",
+         "--compute-colocalization",
+      ]