chore(general): Add a job to ensure evaluated keys are added to new resource checks #15450
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: PR Test | |
on: pull_request | |
permissions: | |
contents: read | |
jobs: | |
lint: | |
uses: bridgecrewio/gha-reusable-workflows/.github/workflows/pre-commit.yaml@main | |
with: | |
python-version: "3.9" | |
danger-check: | |
runs-on: [ self-hosted, public, linux, x64 ] | |
permissions: | |
contents: read | |
pull-requests: read | |
steps: | |
- name: Checkout code | |
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- name: Install Node.js | |
uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
with: | |
node-version: "16" | |
- name: Install and run DangerJS | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
run: | | |
npm install -g danger | |
danger ci --verbose --failOnErrors | |
cfn-lint: | |
runs-on: ubuntu-latest | |
env: | |
PYTHON_VERSION: "3.8" | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ env.PYTHON_VERSION }} | |
- name: Get changed CFN test files | |
id: changed-files-specific | |
uses: tj-actions/changed-files@6b2903bdce6310cfbddd87c418f253cf29b2dec9 # v44 | |
with: | |
files: tests/cloudformation/checks/resource/aws/**/* | |
- name: Filter YAML and JSON files | |
if: steps.changed-files-specific.outputs.any_changed == 'true' | |
id: filter-files | |
run: | | |
YAML_JSON_FILES=$(echo ${{ steps.changed-files-specific.outputs.all_changed_files }} | tr ' ' '\n' | grep -E '\.ya?ml$|\.json$' | tr '\n' ' ') | |
if [ -n "$YAML_JSON_FILES" ]; then | |
echo "YAML_JSON_FILES=$YAML_JSON_FILES" >> "$GITHUB_ENV" | |
fi | |
- name: Install cfn-lint & Lint Cloudformation templates | |
if: env.YAML_JSON_FILES != '' | |
run: | | |
pip install -U cfn-lint | |
for file in $YAML_JSON_FILES; do | |
cfn-lint "$file" -i W | |
done | |
mypy: | |
uses: bridgecrewio/gha-reusable-workflows/.github/workflows/mypy.yaml@main | |
with: | |
python-version: "3.8" | |
unit-tests: | |
strategy: | |
fail-fast: true | |
matrix: | |
python: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
runs-on: ubuntu-latest | |
timeout-minutes: 30 | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- name: Set up Python ${{ matrix.python }} | |
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ matrix.python }} | |
allow-prereleases: true | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
with: | |
github-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Install pipenv | |
run: | | |
if [ '${{ matrix.python }}' == '3.12' ]; then | |
# needed for numpy | |
python -m pip install --no-cache-dir --upgrade pipenv==2024.0.3 | |
else | |
python -m pip install --no-cache-dir --upgrade pipenv | |
fi | |
- name: Install dependencies | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ matrix.python }} | |
if [ '${{ matrix.python }}' == '3.12' ]; then | |
# needed for numpy | |
pipenv install --skip-lock --dev -v | |
else | |
pipenv install --dev -v | |
fi | |
# list all dependencies to get a better view about installed package versions | |
pipenv run pip list | |
- name: Unit tests | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
run: pipenv run python -m pytest tests | |
integration-tests: | |
strategy: | |
fail-fast: true | |
matrix: | |
python: ["3.10", "3.11", "3.12"] | |
os: [ubuntu-latest, macos-latest, windows-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ matrix.python }} | |
allow-prereleases: true | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
- uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
if: ${{ runner.os != 'windows' }} | |
with: | |
github-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
shell: bash | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ matrix.python }} | |
pipenv run pip install pytest pytest-xdist setuptools wheel | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Clone Terragoat - vulnerable terraform | |
run: git clone https://github.com/bridgecrewio/terragoat | |
- name: Clone Cfngoat - vulnerable cloudformation | |
run: git clone https://github.com/bridgecrewio/cfngoat | |
- name: Clone Kubernetes-goat - vulnerable kubernetes | |
run: git clone https://github.com/madhuakula/kubernetes-goat | |
- name: Clone kustomize-goat - vulnerable kustomize | |
run: git clone https://github.com/bridgecrewio/kustomizegoat | |
- name: Create checkov reports | |
env: | |
LOG_LEVEL: INFO | |
BC_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
run: | | |
# Just making sure the API key tests don't run on PRs | |
bash -c './integration_tests/prepare_data.sh ${{ matrix.os }} 3.8' | |
- name: Run integration tests | |
run: | | |
pipenv run pytest integration_tests -k 'not api_key' | |
integration-tests-old-python: | |
strategy: | |
fail-fast: true | |
matrix: | |
python: ["3.8", "3.9"] | |
os: [ubuntu-latest, windows-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ matrix.python }} | |
allow-prereleases: true | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
- uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
if: ${{ runner.os != 'windows' }} | |
with: | |
github-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
shell: bash | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ matrix.python }} | |
pipenv run pip install pytest pytest-xdist setuptools wheel | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Clone Terragoat - vulnerable terraform | |
run: git clone https://github.com/bridgecrewio/terragoat | |
- name: Clone Cfngoat - vulnerable cloudformation | |
run: git clone https://github.com/bridgecrewio/cfngoat | |
- name: Clone Kubernetes-goat - vulnerable kubernetes | |
run: git clone https://github.com/madhuakula/kubernetes-goat | |
- name: Clone kustomize-goat - vulnerable kustomize | |
run: git clone https://github.com/bridgecrewio/kustomizegoat | |
- name: Create checkov reports | |
env: | |
LOG_LEVEL: INFO | |
BC_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
run: | | |
# Just making sure the API key tests don't run on PRs | |
bash -c './integration_tests/prepare_data.sh ${{ matrix.os }} 3.8' | |
- name: Run integration tests | |
run: | | |
pipenv run pytest integration_tests -k 'not api_key' | |
sast-integration-tests: | |
strategy: | |
fail-fast: true | |
matrix: | |
python: ["3.12"] | |
os: [ubuntu-latest, macos-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ matrix.python }} | |
allow-prereleases: true | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ matrix.python }} | |
pipenv run pip install pytest pytest-xdist setuptools wheel | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Clone flask - Python repo for SAST | |
run: git clone https://github.com/pallets/flask | |
- name: Clone WebGoat - Java repo for SAST | |
run: git clone https://github.com/WebGoat/WebGoat | |
- name: Clone axios - JavaScript repo for SAST | |
run: git clone https://github.com/axios/axios | |
- name: Create checkov reports | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: bash -c './sast_integration_tests/prepare_data.sh' | |
- name: Run integration tests | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: | | |
pipenv run pytest sast_integration_tests | |
sast-integration-tests-old-python: | |
strategy: | |
fail-fast: true | |
matrix: | |
python: ["3.8"] | |
os: [ubuntu-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ matrix.python }} | |
allow-prereleases: true | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ matrix.python }} | |
pipenv run pip install pytest pytest-xdist setuptools wheel | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Clone flask - Python repo for SAST | |
run: git clone https://github.com/pallets/flask | |
- name: Clone WebGoat - Java repo for SAST | |
run: git clone https://github.com/WebGoat/WebGoat | |
- name: Clone axios - JavaScript repo for SAST | |
run: git clone https://github.com/axios/axios | |
- name: Create checkov reports | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: bash -c './sast_integration_tests/prepare_data.sh' | |
- name: Run integration tests | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: | | |
pipenv run pytest sast_integration_tests | |
cdk-integration-tests: | |
strategy: | |
fail-fast: true | |
matrix: | |
python: ["3.12"] | |
os: [ubuntu-latest, macos-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ matrix.python }} | |
allow-prereleases: true | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ matrix.python }} | |
pipenv run pip install pytest pytest-xdist setuptools wheel | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Create checkov reports | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: bash -c './cdk_integration_tests/prepare_data.sh' | |
- name: Run integration tests | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: | | |
pipenv run pytest cdk_integration_tests | |
cdk-integration-tests-old-python: | |
strategy: | |
fail-fast: true | |
matrix: | |
python: ["3.8"] | |
os: [ubuntu-latest] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ matrix.python }} | |
allow-prereleases: true | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ matrix.python }} | |
pipenv run pip install pytest pytest-xdist setuptools wheel | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Create checkov reports | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: bash -c './cdk_integration_tests/prepare_data.sh' | |
- name: Run integration tests | |
env: | |
LOG_LEVEL: INFO | |
BC_API_KEY: ${{ secrets.PRISMA_KEY_API2 }} | |
PRISMA_API_URL: ${{ secrets.PRISMA_API_URL_2 }} | |
if: env.BC_API_KEY != null | |
run: | | |
pipenv run pytest cdk_integration_tests | |
performance-tests: | |
env: | |
PYTHON_VERSION: "3.8" | |
working-directory: ./performance_tests | |
runs-on: [self-hosted, public, linux, x64] | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ env.PYTHON_VERSION }} | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 | |
- uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
with: | |
github-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ env.PYTHON_VERSION }} | |
# 'py' package is used in 'pytest-benchmark', but 'pytest' removed it in their latest version | |
pipenv run pip install pytest pytest-benchmark py | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Clone terraform-aws-components | |
run: git clone --branch 0.182.0 https://github.com/cloudposse/terraform-aws-components.git | |
working-directory: ${{ env.working-directory }} | |
- name: Clone aws-cloudformation-templates | |
run: git clone --branch 0.0.1 https://github.com/awslabs/aws-cloudformation-templates.git | |
working-directory: ${{ env.working-directory }} | |
- name: Clone kubernetes-yaml-templates | |
run: git clone https://github.com/dennyzhang/kubernetes-yaml-templates.git | |
working-directory: ${{ env.working-directory }} | |
# TODO: migrate to separate performance tests | |
# - name: Clone Python-Mini-Projects | |
# run: git clone https://github.com/alimoustafa2000/Python-Mini-Projects.git | |
# working-directory: ${{ env.working-directory }} | |
# - name: Clone NodeJs | |
# run: git clone https://github.com/harshitbansal373/NodeJs.git | |
# working-directory: ${{ env.working-directory }} | |
# - name: Clone Mini-Project-using-Java | |
# run: git clone https://github.com/ikanurfitriani/Mini-Project-using-Java.git | |
# working-directory: ${{ env.working-directory }} | |
- name: Run performance tests | |
run: | | |
pipenv run pytest | |
working-directory: ${{ env.working-directory }} | |
dogfood-tests: | |
runs-on: ubuntu-latest | |
env: | |
PYTHON_VERSION: "3.8" | |
WORKING_DIRECTORY: ./dogfood_tests | |
steps: | |
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v4 | |
with: | |
python-version: ${{ env.PYTHON_VERSION }} | |
cache: "pipenv" | |
cache-dependency-path: "Pipfile.lock" | |
- uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3 | |
with: | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- uses: imranismail/setup-kustomize@a76db1c6419124d51470b1e388c4b29476f495f1 # v2 | |
with: | |
github-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Install pipenv | |
run: | | |
python -m pip install --no-cache-dir --upgrade pipenv | |
- name: Build & install checkov package | |
run: | | |
# remove venv, if exists | |
pipenv --rm || true | |
pipenv --python ${{ env.PYTHON_VERSION }} | |
pipenv run pip install pytest pytest-xdist | |
pipenv run python setup.py sdist bdist_wheel | |
bash -c 'pipenv run pip install dist/checkov-*.whl' | |
- name: Run dogfood tests | |
run: | | |
pipenv run pytest | |
working-directory: ${{ env.WORKING_DIRECTORY }} | |
eval-keys-test: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Check out repository | |
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3 | |
- name: Get changed Python files | |
id: changed-files | |
uses: tj-actions/changed-files@6b2903bdce6310cfbddd87c418f253cf29b2dec9 # v44 | |
with: | |
files: checkov/**/*.py | |
- name: Validate 'BaseResourceCheck' use contains eval keys | |
if: steps.changed-files.outputs.any_changed == 'true' | |
run: | | |
# Define an array of exceptions (files to skip) | |
EXCEPTIONS=( | |
"base_resource_check.py" | |
"VPCDefaultNetwork.py" | |
"IAMUserNotUsedForAccess.py" # Whole Resource type check | |
) | |
echo "Changed files:" | |
echo "${{ steps.changed-files.outputs.all_changed_files }}" | |
EXIT_CODE=0 | |
IFS=$'\n' # Change Internal Field Separator to handle spaces in filenames too | |
for file in $(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ',' '\n'); do | |
# Check if the file is in the list of exceptions | |
SKIP_FILE="false" | |
for exception in "${EXCEPTIONS[@]}"; do | |
# If the file ends with one of the exception file names, skip it | |
if [[ "$file" == *"$exception" ]]; then | |
echo "Skipping $file (allowed exception)" | |
SKIP_FILE="true" | |
break | |
fi | |
done | |
# Only run checks if not in exceptions list | |
if [[ "$SKIP_FILE" == "false" ]]; then | |
# If file contains 'BaseResourceCheck', check for 'get_inspected_key' or 'evaluated_keys' | |
if grep -q "BaseResourceCheck" "$file"; then | |
if ! grep -q "get_inspected_key" "$file" && ! grep -q "evaluated_keys" "$file"; then | |
echo "ERROR: $file has BaseResourceCheck but does NOT contain 'get_inspected_key' or 'evaluated_keys'" | |
EXIT_CODE=1 | |
fi | |
fi | |
fi | |
done | |
unset IFS # Restore IFS to default | |
# Fail the job if any file violated the rule | |
if [ "$EXIT_CODE" -ne 0 ]; then | |
echo "One or more files did not satisfy the requirement." | |
exit 1 | |
fi |