Github workflow - integration with job to protect against malicious users

.github/workflows/integration.yaml

@@ -0,0 +1,224 @@
+name: integration
+
+concurrency:
+  group: community-aws-integration-${{ github.sha }}
+  cancel-in-progress: true
+
+on:
+  pull_request_target:
+
+env:
+  AWS_INTEGRATION_TOTAL_JOBS: 22
+
+jobs:
+  safe-to-test:
+    if: ${{ github.event.label.name == 'safe to test' }} || ${{ github.event.action != 'labeled' }}
+    uses: abikouo/github_actions/.github/workflows/safe-to-test.yml@safe_to_test_v2
+  splitter:
+    runs-on: ubuntu-latest
+    needs:
+      - safe-to-test
+    env:
+      amazon_dir: "./amazon"
+      community_dir: "./community"
+    outputs:
+      test_targets: ${{ steps.display.outputs.test_targets }}
+    steps:
+      - name: Determines number of jobs to run integration tests
+        id: determine-workflow
+        run: |
+          import os, re
+          total_jobs = os.environ.get('AWS_INTEGRATION_TOTAL_JOBS')
+          for line in os.environ.get('PR_BODY').split('\n'):
+              if m := re.match(r"^GHA_SPLITTER_TOTAL_JOBS=(\d+)$", line):
+                  total_jobs = m.group(1)
+                  break
+          output_path = os.environ.get('GITHUB_OUTPUT')
+          with open(output_path, "a", encoding="utf-8") as fw:
+              fw.write(f'total_jobs={total_jobs}\n')
+        shell: python
+        env:
+          PR_BODY: "${{ github.event.pull_request.body }}"
+
+      - name: Checkout repository
+        uses: ansible-network/github_actions/.github/actions/checkout_dependency@main
+        with:
+          path: ${{ env.community_dir }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: "0"
+
+      - name: Checkout dependency (amazon.aws)
+        uses: ansible-network/github_actions/.github/actions/checkout_dependency@main
+        with:
+          repository: ansible-collections/amazon.aws
+          path: ${{ env.amazon_dir }}
+          ref: ${{ github.event.pull_request.base.ref }}
+
+      - name: list changes for pull request
+        id: splitter
+        uses: ansible-network/github_actions/.github/actions/ansible_test_splitter@main
+        with:
+          total_jobs: ${{ steps.determine-workflow.outputs.total_jobs }}
+          collections_to_test: |
+            ${{ env.community_dir }}
+            ${{ env.amazon_dir }}
+
+      - name: display targets
+        id: display
+        run: echo "test_targets=${{ steps.splitter.outputs.test_targets }}" >> $GITHUB_OUTPUT
+        shell: bash
+  build_jobs:
+    runs-on: ubuntu-latest
+    if: ${{ needs.splitter.outputs.test_targets != '' }}
+    needs:
+      - splitter
+    outputs:
+      test_matrix: ${{ steps.generate-workflow.outputs.targets }}
+      test_targets: ${{ needs.splitter.outputs.test_targets }}
+    steps:
+      - name: Generate workflow id
+        id: generate-workflow
+        run: |
+          import os, json
+          targets=[d.split(':', maxsplit=1)[0] for d in os.environ.get('TEST_TARGETS').split(";")]
+          output_path = os.environ.get('GITHUB_OUTPUT')
+          with open(output_path, "a", encoding="utf-8") as fw:
+              fw.write(f'targets={json.dumps(targets)}\n')
+        shell: python
+        env:
+          TEST_TARGETS: ${{ needs.splitter.outputs.test_targets }}
+
+      - name: Display matrix elements
+        run: echo "Workflow list => ${{ steps.generate-workflow.outputs.targets }}"
+        shell: bash
+  integration:
+    needs:
+      - build_jobs
+    env:
+      test_targets: ${{ needs.build_jobs.outputs.test_targets }}
+      source: "./source"
+      dependency: "./dependency"
+    if: ${{ needs.build_jobs.outputs.test_matrix != '' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ansible-version:
+          - milestone
+        python-version:
+          - "3.9"
+        workflow-id: ${{ fromJson(needs.build_jobs.outputs.test_matrix) }}
+    name: "integration-${{ matrix.workflow-id }}-py${{ matrix.python-version }}-${{ matrix.ansible-version }}"
+    steps:
+      - name: Read ansible-test targets
+        id: read-targets
+        run: |
+          import os
+          workflow_id = os.environ.get('TEST_WORKFLOW_ID')
+          all_targets = os.environ.get('TEST_TARGETS')
+          workflow_targets={d.split(':', maxsplit=1)[0]: d.split(':', maxsplit=1)[1] for d in all_targets.split(";")}[workflow_id]
+          targets = " ".join(workflow_targets.split(","))
+          output_path = os.environ.get('GITHUB_OUTPUT')
+          with open(output_path, "a", encoding="utf-8") as fw:
+              fw.write(f"ansible_test_targets={targets}\n")
+              fw.write(f"ansible_collection_to_test={workflow_id.split('-', maxsplit=1)[0]}\n")
+        shell: python
+        env:
+          TEST_WORKFLOW_ID: "${{ matrix.workflow-id }}"
+          TEST_TARGETS: ${{ env.test_targets }}
+
+      # Install ansible
+      - name: Install ansible-core (${{ matrix.ansible-version }})
+        run: python3 -m pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible-version }}.tar.gz --disable-pip-version-check
+
+      # checkout collections
+      - name: Checkout repository
+        uses: ansible-network/github_actions/.github/actions/checkout_dependency@main
+        with:
+          path: ${{ env.source }}
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Checkout dependency (amazon.aws)
+        uses: ansible-network/github_actions/.github/actions/checkout_dependency@main
+        with:
+          repository: ansible-collections/amazon.aws
+          path: ${{ env.dependency }}
+          ref: ${{ github.event.pull_request.base.ref }}
+
+      # install collection requirements
+      - name: Install test requirements for amazon.aws
+        run: |
+          [[ "$COLLECTION_TO_TEST" == "community.aws" ]] && collection_path=$COMMUNITY_PATH || collection_path=$AMAZON_PATH
+          ansible-galaxy collection install -r ${collection_path}/tests/integration/requirements.yml -p /home/runner/collections/
+        shell: bash
+        env:
+          COMMUNITY_PATH: ${{ env.source }}
+          AMAZON_PATH: ${{ env.dependency }}
+          COLLECTION_TO_TEST: ${{ steps.read-targets.outputs.ansible_collection_to_test }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # Install checkout collection (community.aws)
+      - name: Install community.aws - Read collection metadata from galaxy.yml
+        id: identify
+        uses: ansible-network/github_actions/.github/actions/identify_collection@main
+        with:
+          source_path: ${{ env.source }}
+
+      - name: Install community.aws - Build and install the collection
+        uses: ansible-network/github_actions/.github/actions/build_install_collection@main
+        with:
+          install_python_dependencies: true
+          source_path: ${{ env.source }}
+          collection_path: ${{ steps.identify.outputs.collection_path }}
+          tar_file: ${{ steps.identify.outputs.tar_file }}
+          ansible_version: ${{ matrix.ansible-version }}
+
+      # Install checkout collection (amazon.aws)
+      - name: Install amazon.aws - Read collection metadata from galaxy.yml
+        id: identify-amazon
+        uses: ansible-network/github_actions/.github/actions/identify_collection@main
+        with:
+          source_path: ${{ env.dependency }}
+
+      - name: Install amazon.aws - Build and install the collection
+        uses: ansible-network/github_actions/.github/actions/build_install_collection@main
+        with:
+          install_python_dependencies: true
+          source_path: ${{ env.dependency }}
+          collection_path: ${{ steps.identify-amazon.outputs.collection_path }}
+          tar_file: ${{ steps.identify-amazon.outputs.tar_file }}
+          ansible_version: ${{ matrix.ansible-version }}
+
+      # Determines path to the collection being tested
+      - name: Determines path to the collection being tested
+        id: collection-path
+        run: |
+          [[ "$COLLECTION_TO_TEST" == "amazon.aws" ]] && test_path=$AMAZON_PATH || test_path=$COMMUNITY_PATH
+          echo "test_path=${test_path}" >> $GITHUB_OUTPUT
+        env:
+          COLLECTION_TO_TEST: ${{ steps.read-targets.outputs.ansible_collection_to_test }}
+          AMAZON_PATH: ${{ steps.identify-amazon.outputs.collection_path }}
+          COMMUNITY_PATH: ${{ steps.identify.outputs.collection_path }}
+
+      - name: Create AWS/sts session credentials
+        uses: ansible-network/github_actions/.github/actions/ansible_aws_test_provider@main
+        with:
+          collection_path: ${{ steps.collection-path.outputs.test_path }}
+          ansible_core_ci_key: ${{ secrets.ANSIBLE_CORE_CI_KEY }}
+
+      - name: Run integration tests
+        uses: ansible-network/github_actions/.github/actions/ansible_test_integration@main
+        with:
+          collection_path: ${{ steps.collection-path.outputs.test_path }}
+          python_version: ${{ matrix.python-version }}
+          ansible_version: ${{ matrix.ansible-version }}
+          ansible_test_requirement_files: |
+            requirements.txt
+            test-requirements.txt
+            tests/integration/requirements.txt
+          ansible_test_constraint_files: tests/integration/constraints.txt
+          ansible_test_targets: ${{ steps.read-targets.outputs.ansible_test_targets }}

.gitignore

@@ -1,10 +1,9 @@
-
 # Created by https://www.gitignore.io/api/git,linux,pydev,python,windows,pycharm+all,jupyternotebook,vim,webstorm,emacs,dotenv
 # Edit at https://www.gitignore.io/?templates=git,linux,pydev,python,windows,pycharm+all,jupyternotebook,vim,webstorm,emacs,dotenv
 
 ### dotenv ###
 .env
-
+tests/integration/inventory
 ### Emacs ###
 # -*- mode: gitignore; -*-
 *~

tests/integration/requirements.yml

@@ -6,3 +6,4 @@ collections:
   - ansible.windows
   - community.crypto
   - community.general
+  - ansible.utils # ipaddr