diff --git a/.github/workflows/transformations_aws_asset_inventory_free.yml b/.github/workflows/transformations_aws_asset_inventory_free.yml new file mode 100644 index 000000000..ae1acc137 --- /dev/null +++ b/.github/workflows/transformations_aws_asset_inventory_free.yml @@ -0,0 +1,126 @@ +name: "Test AWS Asset Inventory Free Policies" + +on: + pull_request: + paths: + - "transformations/aws/asset-inventory-free/**" + - ".github/workflows/transformations_aws_asset_inventory_free.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/aws/asset-inventory-free/**" + - ".github/workflows/transformations_aws_asset_inventory_free.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/aws/asset-inventory-free + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + console.log(JSON.stringify({ transformation_dir, postgres, snowflake, bigquery })); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-aws-asset-inventory-free: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml b/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml deleted file mode 100644 index f739b9153..000000000 --- a/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test AWS Asset Inventory Free Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/aws/asset-inventory-free/**" - - ".github/workflows/transformations_aws_asset_inventory_free_postgres.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/aws/asset-inventory-free/**" - - ".github/workflows/transformations_aws_asset_inventory_free_postgres.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - -jobs: - transformations-aws-asset-inventory-free: - name: transformations/aws/asset-inventory-free - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/aws/asset-inventory-free - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/aws/asset-inventory-free/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests diff --git a/.github/workflows/transformations_aws_compliance_free.yml b/.github/workflows/transformations_aws_compliance_free.yml new file mode 100644 index 000000000..1c6dcdab9 --- /dev/null +++ b/.github/workflows/transformations_aws_compliance_free.yml @@ -0,0 +1,125 @@ +name: "Test AWS Compliance Free Policies" + +on: + pull_request: + paths: + - "transformations/aws/compliance-free/**" + - ".github/workflows/transformations_aws_compliance_free.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/aws/compliance-free/**" + - ".github/workflows/transformations_aws_compliance_free.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/aws/compliance-free + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-aws-compliance-free: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests --select aws_compliance__foundational_security_free + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests --select aws_compliance__foundational_security_free + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests --select aws_compliance__foundational_security_free diff --git a/.github/workflows/transformations_aws_compliance_free_postgres.yml b/.github/workflows/transformations_aws_compliance_free_postgres.yml deleted file mode 100644 index 784dc3db9..000000000 --- a/.github/workflows/transformations_aws_compliance_free_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test AWS Compliance Free Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/aws/compliance-free/**" - - ".github/workflows/transformations_aws_compliance_free_postgres.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/aws/compliance-free/**" - - ".github/workflows/transformations_aws_compliance_free_postgres.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - -jobs: - transformations-aws-compliance-free: - name: transformations/aws/compliance-free - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/aws/compliance-free - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/aws/compliance-free/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/.github/workflows/transformations_aws_compliance_free_snowflake.yml b/.github/workflows/transformations_aws_compliance_free_snowflake.yml deleted file mode 100644 index 05d0704b1..000000000 --- a/.github/workflows/transformations_aws_compliance_free_snowflake.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: "Test AWS Compliance Free Policies: Snowflake" - -on: - pull_request: - paths: - - "transformations/aws/compliance-free/**" - - ".github/workflows/transformations_aws_compliance_free_snowflake.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/aws/compliance-free/**" - - ".github/workflows/transformations_aws_compliance_free_snowflake.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - -jobs: - transformations-aws-compliance-free: - name: transformations/aws/compliance-free - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/aws/compliance-free - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/aws/compliance-free/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/snowflake.yml - env: - SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" - - name: Run Policies - run: | - dbt run --target dev-snowflake --profiles-dir ./tests --select --select aws_compliance__security_group_ingress_rules, aws_compliance__api_gateway_method_settingse_selected_only, aws_compliance__foundational_security - env: - SNOW_USER: ${{ secrets.SNOW_USER }} - SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} - # DBT assumes the account is in the form of . - SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" - SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} - SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} - SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} - SNOW_REGION: ${{ secrets.SNOW_REGION }} \ No newline at end of file diff --git a/.github/workflows/transformations_aws_compliance_premium.yml b/.github/workflows/transformations_aws_compliance_premium.yml new file mode 100644 index 000000000..1592116e2 --- /dev/null +++ b/.github/workflows/transformations_aws_compliance_premium.yml @@ -0,0 +1,125 @@ +name: "Test AWS Compliance Premium Policies" + +on: + pull_request: + paths: + - "transformations/aws/compliance-premium/**" + - ".github/workflows/transformations_aws_compliance_premium.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/aws/compliance-premium/**" + - ".github/workflows/transformations_aws_compliance_premium.yml" + - "transformations/aws/macros/**" + - "transformations/aws/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/aws/compliance-premium + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-aws-compliance-premium: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests --select aws_compliance__foundational_security + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests --select aws_compliance__foundational_security + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests --select aws_compliance__foundational_security diff --git a/.github/workflows/transformations_aws_compliance_premium_postgres.yml b/.github/workflows/transformations_aws_compliance_premium_postgres.yml deleted file mode 100644 index 7c5fb91a3..000000000 --- a/.github/workflows/transformations_aws_compliance_premium_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test AWS Compliance Premium Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/aws/compliance-premium/**" - - ".github/workflows/transformations_aws_compliance_premium_postgres.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/aws/compliance-premium/**" - - ".github/workflows/transformations_aws_compliance_premium_postgres.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - -jobs: - transformations-aws-compliance-premium: - name: transformations/aws/compliance-premium - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/aws/compliance-premium - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/aws/compliance-premium/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests --exclude aws_compliance__foundational_security \ No newline at end of file diff --git a/.github/workflows/transformations_aws_compliance_premium_snowflake.yml b/.github/workflows/transformations_aws_compliance_premium_snowflake.yml deleted file mode 100644 index 9f0ee4a4b..000000000 --- a/.github/workflows/transformations_aws_compliance_premium_snowflake.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: "Test AWS Compliance Premium Policies: Snowflake" - -on: - pull_request: - paths: - - "transformations/aws/compliance-premium/**" - - ".github/workflows/transformations_aws_compliance_premium_snowflake.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/aws/compliance-premium/**" - - ".github/workflows/transformations_aws_compliance_premium_snowflake.yml" - - "transformations/aws/macros/**" - - "transformations/aws/models/**" - - "transformations/macros/**" - -jobs: - transformations-aws-compliance-premium: - name: transformations/aws/compliance-premium - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/aws/compliance-premium - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/aws/compliance-premium/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/snowflake.yml - env: - SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" - - name: Run Policies - run: | - dbt run --target dev-snowflake --profiles-dir ./tests --select --select aws_compliance__security_group_ingress_rules, aws_compliance__api_gateway_method_settingse_selected_only, aws_compliance__foundational_security - env: - SNOW_USER: ${{ secrets.SNOW_USER }} - SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} - # DBT assumes the account is in the form of . - SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" - SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} - SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} - SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} - SNOW_REGION: ${{ secrets.SNOW_REGION }} \ No newline at end of file diff --git a/.github/workflows/transformations_aws_cost.yml b/.github/workflows/transformations_aws_cost.yml index d67cebf82..cfc03070e 100644 --- a/.github/workflows/transformations_aws_cost.yml +++ b/.github/workflows/transformations_aws_cost.yml @@ -1,10 +1,10 @@ -name: "Test AWS Cost Policies: Postgres" +name: "Test AWS Cost Policies" on: pull_request: paths: - "transformations/aws/cost/**" - - ".github/workflows/transformations_aws_cost_postgres.yml" + - ".github/workflows/transformations_aws_cost.yml" - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" @@ -13,19 +13,62 @@ on: - main paths: - "transformations/aws/cost/**" - - ".github/workflows/transformations_aws_cost_postgres.yml" + - ".github/workflows/transformations_aws_cost.yml" - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/aws/cost + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; transformations-aws-cost: - name: transformations/aws/cost + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare timeout-minutes: 30 runs-on: ubuntu-latest defaults: run: - working-directory: ./transformations/aws/cost + working-directory: ${{ needs.prepare.outputs.transformation_dir }} services: postgres: image: postgres:11 @@ -44,22 +87,42 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' - uses: actions/setup-python@v5 with: python-version: "3.9" cache: "pip" - cache-dependency-path: "./transformations/aws/cost/requirements.txt" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" - name: Install dependencies run: pip install -r requirements.txt - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt seed --target dev-pg --profiles-dir ./tests + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt seed --target dev-pg --profiles-dir ./tests + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' run: | + cloudquery migrate tests/bigquery.yml dbt seed --target dev-pg --profiles-dir ./tests - dbt run --target dev-pg --profiles-dir ./tests --vars 'cost_usage_table: cost_check' + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_aws_data_resilience.yml b/.github/workflows/transformations_aws_data_resilience.yml index 1c2e9c57f..8c72683a2 100644 --- a/.github/workflows/transformations_aws_data_resilience.yml +++ b/.github/workflows/transformations_aws_data_resilience.yml @@ -1,10 +1,10 @@ -name: "Test AWS Data Resilience Policies: Postgres" +name: "Test AWS Data Resilience Policies" on: pull_request: paths: - "transformations/aws/data-resilience/**" - - ".github/workflows/transformations_aws_data_resilience_postgres.yml" + - ".github/workflows/transformations_aws_data_resilience.yml" - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" @@ -13,19 +13,62 @@ on: - main paths: - "transformations/aws/data-resilience/**" - - ".github/workflows/transformations_aws_data_resilience_postgres.yml" + - ".github/workflows/transformations_aws_data_resilience.yml" - "transformations/aws/macros/**" - "transformations/aws/models/**" - "transformations/macros/**" +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/aws/data-resilience + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; transformations-aws-data-resilience: - name: transformations/aws/data-resilience + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare timeout-minutes: 30 runs-on: ubuntu-latest defaults: run: - working-directory: ./transformations/aws/data-resilience + working-directory: ${{ needs.prepare.outputs.transformation_dir }} services: postgres: image: postgres:11 @@ -44,21 +87,39 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' - uses: actions/setup-python@v5 with: python-version: "3.9" cache: "pip" - cache-dependency-path: "./transformations/aws/data-resilience/requirements.txt" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" - name: Install dependencies run: pip install -r requirements.txt - name: Setup CloudQuery uses: cloudquery/setup-cloudquery@v3 with: version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' run: | - dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_azure_compliance_free.yml b/.github/workflows/transformations_azure_compliance_free.yml new file mode 100644 index 000000000..cb58a32e7 --- /dev/null +++ b/.github/workflows/transformations_azure_compliance_free.yml @@ -0,0 +1,125 @@ +name: "Test Azure Compliance Free Policies" + +on: + pull_request: + paths: + - "transformations/azure/compliance-free/**" + - ".github/workflows/transformations_azure_compliance_free.yml" + - "transformations/azure/macros/**" + - "transformations/azure/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/azure/compliance-free/**" + - ".github/workflows/transformations_azure_compliance_free.yml" + - "transformations/azure/macros/**" + - "transformations/azure/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/azure/compliance-free + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-azure-compliance-free: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_azure_compliance_free_postgres.yml b/.github/workflows/transformations_azure_compliance_free_postgres.yml deleted file mode 100644 index cb04e22e8..000000000 --- a/.github/workflows/transformations_azure_compliance_free_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test Azure Compliance Free Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/azure/compliance-free/**" - - ".github/workflows/transformations_azure_compliance_free_postgres.yml" - - "transformations/azure/macros/**" - - "transformations/azure/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/azure/compliance-free/**" - - ".github/workflows/transformations_azure_compliance_free_postgres.yml" - - "transformations/azure/macros/**" - - "transformations/azure/models/**" - - "transformations/macros/**" - -jobs: - transformations-azure-compliance-free: - name: transformations/azure/compliance-free - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/azure/compliance-free - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/azure/compliance-free/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/.github/workflows/transformations_azure_compliance_premium.yml b/.github/workflows/transformations_azure_compliance_premium.yml new file mode 100644 index 000000000..ecf46dacc --- /dev/null +++ b/.github/workflows/transformations_azure_compliance_premium.yml @@ -0,0 +1,125 @@ +name: "Test Azure Compliance Premium Policies" + +on: + pull_request: + paths: + - "transformations/azure/compliance-premium/**" + - ".github/workflows/transformations_azure_compliance_premium.yml" + - "transformations/azure/macros/**" + - "transformations/azure/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/azure/compliance-premium/**" + - ".github/workflows/transformations_azure_compliance_premium.yml" + - "transformations/azure/macros/**" + - "transformations/azure/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/azure/compliance-premium + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-azure-compliance-premium: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_azure_compliance_premium_postgres.yml b/.github/workflows/transformations_azure_compliance_premium_postgres.yml deleted file mode 100644 index 68f290a11..000000000 --- a/.github/workflows/transformations_azure_compliance_premium_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test Azure Compliance Premium Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/azure/compliance-premium/**" - - ".github/workflows/transformations_azure_compliance_premium_postgres.yml" - - "transformations/azure/macros/**" - - "transformations/azure/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/azure/compliance-premium/**" - - ".github/workflows/transformations_azure_compliance_premium_postgres.yml" - - "transformations/azure/macros/**" - - "transformations/azure/models/**" - - "transformations/macros/**" - -jobs: - transformations-azure-compliance-premium: - name: transformations/azure/compliance-premium - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/azure/compliance-premium - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/azure/compliance-premium/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests diff --git a/.github/workflows/transformations_gcp_compliance_free.yml b/.github/workflows/transformations_gcp_compliance_free.yml new file mode 100644 index 000000000..8210d00be --- /dev/null +++ b/.github/workflows/transformations_gcp_compliance_free.yml @@ -0,0 +1,125 @@ +name: "Test GCP Compliance Free Policies" + +on: + pull_request: + paths: + - "transformations/gcp/compliance-free/**" + - ".github/workflows/transformations_gcp_compliance_free.yml" + - "transformations/gcp/macros/**" + - "transformations/gcp/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/gcp/compliance-free/**" + - ".github/workflows/transformations_gcp_compliance_free.yml" + - "transformations/gcp/macros/**" + - "transformations/gcp/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/gcp/compliance-free + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-gcp-compliance-free: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_gcp_compliance_free_postgres.yml b/.github/workflows/transformations_gcp_compliance_free_postgres.yml deleted file mode 100644 index 16e3395b1..000000000 --- a/.github/workflows/transformations_gcp_compliance_free_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test GCP Compliance free Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/gcp/compliance-free/**" - - ".github/workflows/transformations_gcp_compliance_free_postgres.yml" - - "transformations/gcp/macros/**" - - "transformations/gcp/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/gcp/compliance-free/**" - - ".github/workflows/transformations_gcp_compliance_free_postgres.yml" - - "transformations/gcp/macros/**" - - "transformations/gcp/models/**" - - "transformations/macros/**" - -jobs: - transformations-gcp-compliance-free: - name: transformations/gcp/compliance-free - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/gcp/compliance-free - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/gcp/compliance-free/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests diff --git a/.github/workflows/transformations_gcp_compliance_premium.yml b/.github/workflows/transformations_gcp_compliance_premium.yml new file mode 100644 index 000000000..c008cd3b0 --- /dev/null +++ b/.github/workflows/transformations_gcp_compliance_premium.yml @@ -0,0 +1,125 @@ +name: "Test GCP Compliance Premium Policies" + +on: + pull_request: + paths: + - "transformations/gcp/compliance-premium/**" + - ".github/workflows/transformations_gcp_compliance_premium.yml" + - "transformations/gcp/macros/**" + - "transformations/gcp/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/gcp/compliance-premium/**" + - ".github/workflows/transformations_gcp_compliance_premium.yml" + - "transformations/gcp/macros/**" + - "transformations/gcp/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/gcp/compliance-premium + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-gcp-compliance-premium: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_gcp_compliance_premium_postgres.yml b/.github/workflows/transformations_gcp_compliance_premium_postgres.yml deleted file mode 100644 index 1dcc8e4ee..000000000 --- a/.github/workflows/transformations_gcp_compliance_premium_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test GCP Compliance Premium Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/gcp/compliance-premium/**" - - ".github/workflows/transformations_gcp_compliance_premium_postgres.yml" - - "transformations/gcp/macros/**" - - "transformations/gcp/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/gcp/compliance-premium/**" - - ".github/workflows/transformations_gcp_compliance_premium_postgres.yml" - - "transformations/gcp/macros/**" - - "transformations/gcp/models/**" - - "transformations/macros/**" - -jobs: - transformations-gcp-compliance-premium: - name: transformations/gcp/compliance-premium - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/gcp/compliance-premium - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/gcp/compliance-premium/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests diff --git a/.github/workflows/transformations_k8s_compliance_free.yml b/.github/workflows/transformations_k8s_compliance_free.yml new file mode 100644 index 000000000..82df72944 --- /dev/null +++ b/.github/workflows/transformations_k8s_compliance_free.yml @@ -0,0 +1,125 @@ +name: "Test K8S Compliance Free Policies" + +on: + pull_request: + paths: + - "transformations/k8s/compliance-free/**" + - ".github/workflows/transformations_k8s_compliance_free.yml" + - "transformations/k8s/macros/**" + - "transformations/k8s/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/k8s/compliance-free/**" + - ".github/workflows/transformations_k8s_compliance_free.yml" + - "transformations/k8s/macros/**" + - "transformations/k8s/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/k8s/compliance-free + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-k8s-compliance-free: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_k8s_compliance_free_postgres.yml b/.github/workflows/transformations_k8s_compliance_free_postgres.yml deleted file mode 100644 index f12f18053..000000000 --- a/.github/workflows/transformations_k8s_compliance_free_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test K8S Compliance Free Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/k8s/compliance-free/**" - - ".github/workflows/transformations_k8s_compliance_free_postgres.yml" - - "transformations/k8s/macros/**" - - "transformations/k8s/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/k8s/compliance-free/**" - - ".github/workflows/transformations_k8s_compliance_free_postgres.yml" - - "transformations/k8s/macros/**" - - "transformations/k8s/models/**" - - "transformations/macros/**" - -jobs: - transformations-k8s-compliance-free: - name: transformations/k8s/compliance-free - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/k8s/compliance-free - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/k8s/compliance-free/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/.github/workflows/transformations_k8s_compliance_premium.yml b/.github/workflows/transformations_k8s_compliance_premium.yml new file mode 100644 index 000000000..3b095538d --- /dev/null +++ b/.github/workflows/transformations_k8s_compliance_premium.yml @@ -0,0 +1,125 @@ +name: "Test K8S Compliance Premium Policies" + +on: + pull_request: + paths: + - "transformations/k8s/compliance-premium/**" + - ".github/workflows/transformations_k8s_compliance_premium.yml" + - "transformations/k8s/macros/**" + - "transformations/k8s/models/**" + - "transformations/macros/**" + push: + branches: + - main + paths: + - "transformations/k8s/compliance-premium/**" + - ".github/workflows/transformations_k8s_compliance_premium.yml" + - "transformations/k8s/macros/**" + - "transformations/k8s/models/**" + - "transformations/macros/**" + +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + transformation_dir: ${{ fromJson(steps.set-result.outputs.result).transformation_dir }} + postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }} + snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }} + bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/github-script@v7 + id: set-result + env: + TRANSFORMATION_DIR: transformations/k8s/compliance-premium + with: + script: | + const fs = require('fs/promises'); + const { TRANSFORMATION_DIR: transformation_dir } = process.env; + const [postgres, snowflake, bigquery] = await Promise.all([ + fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false), + ]); + return { + transformation_dir, + postgres, + snowflake, + bigquery, + }; + transformations-k8s-compliance-premium: + permissions: + id-token: 'write' + contents: 'read' + name: ${{ needs.prepare.outputs.transformation_dir }} + needs: prepare + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ needs.prepare.outputs.transformation_dir }} + services: + postgres: + image: postgres:11 + env: + POSTGRES_PASSWORD: pass + POSTGRES_USER: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + if: needs.prepare.outputs.bigquery == 'true' + with: + workload_identity_provider: 'projects/151868820337/locations/global/workloadIdentityPools/integration-test-pool/providers/integration-test-provider' + service_account: 'integration-service-account@cq-integration-tests.iam.gserviceaccount.com' + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "${{ needs.prepare.outputs.transformation_dir }}/requirements.txt" + - name: Install dependencies + run: pip install -r requirements.txt + - name: Setup CloudQuery + uses: cloudquery/setup-cloudquery@v3 + with: + version: v4.3.2 + - name: Test Postgres + run: | + cloudquery migrate tests/postgres.yml + dbt run --target dev-pg --profiles-dir ./tests + if: needs.prepare.outputs.postgres == 'true' + env: + CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres + - name: Test Snowflake + run: | + cloudquery migrate tests/snowflake.yml + dbt run --target dev-snowflake --profiles-dir ./tests + if: needs.prepare.outputs.snowflake == 'true' + env: + SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}" + - name: Test BigQuery + if: needs.prepare.outputs.bigquery == 'true' + run: | + cloudquery migrate tests/bigquery.yml + dbt run --target dev-bigquery --profiles-dir ./tests diff --git a/.github/workflows/transformations_k8s_compliance_premium_postgres.yml b/.github/workflows/transformations_k8s_compliance_premium_postgres.yml deleted file mode 100644 index cd755417f..000000000 --- a/.github/workflows/transformations_k8s_compliance_premium_postgres.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Test K8S Compliance Premium Policies: Postgres" - -on: - pull_request: - paths: - - "transformations/k8s/compliance-premium/**" - - ".github/workflows/transformations_k8s_compliance_premium_postgres.yml" - - "transformations/k8s/macros/**" - - "transformations/k8s/models/**" - - "transformations/macros/**" - push: - branches: - - main - paths: - - "transformations/k8s/compliance-premium/**" - - ".github/workflows/transformations_k8s_compliance_premium_postgres.yml" - - "transformations/k8s/macros/**" - - "transformations/k8s/models/**" - - "transformations/macros/**" - -jobs: - transformations-k8s-compliance-premium: - name: transformations/k8s/compliance-premium - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./transformations/k8s/compliance-premium - services: - postgres: - image: postgres:11 - env: - POSTGRES_PASSWORD: pass - POSTGRES_USER: postgres - POSTGRES_DB: postgres - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - cache: "pip" - cache-dependency-path: "./transformations/k8s/compliance-premium/requirements.txt" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Setup CloudQuery - uses: cloudquery/setup-cloudquery@v3 - with: - version: v4.3.2 - - name: Migrate DB - run: cloudquery migrate tests/postgres.yml - env: - CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - - name: Run Policies - run: | - dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql b/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql index a2164234b..d3c031c34 100644 --- a/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql +++ b/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql @@ -1,3 +1,5 @@ +{{ config(enabled=is_snowflake()) }} + with aggregated as ( ({{ access_logs_enabled('foundational_security','cloudfront.5') }}) diff --git a/transformations/aws/cost/dbt_project.yml b/transformations/aws/cost/dbt_project.yml index 934aa38a6..887e44e1c 100644 --- a/transformations/aws/cost/dbt_project.yml +++ b/transformations/aws/cost/dbt_project.yml @@ -23,9 +23,12 @@ clean-targets: # directories to be removed by `dbt clean` - "target" - "dbt_packages" +vars: + cost_usage_table: cost_usage_table + seeds: aws_cost: # you must include the project name - cost_check: + cost_usage_table: +column_types: bill_billing_period_end_date: TIMESTAMP WITHOUT TIME ZONE reservation_amortized_upfront_cost_for_usage: DOUBLE PRECISION diff --git a/transformations/aws/cost/seeds/cost_check.csv b/transformations/aws/cost/seeds/cost_usage_table.csv similarity index 100% rename from transformations/aws/cost/seeds/cost_check.csv rename to transformations/aws/cost/seeds/cost_usage_table.csv diff --git a/transformations/aws/compliance-premium/models/aws_compliance__api_gateway_method_settings.sql b/transformations/aws/models/aws_compliance__api_gateway_method_settings.sql similarity index 100% rename from transformations/aws/compliance-premium/models/aws_compliance__api_gateway_method_settings.sql rename to transformations/aws/models/aws_compliance__api_gateway_method_settings.sql diff --git a/transformations/macros/is_snowflake.sql b/transformations/macros/is_snowflake.sql index c9f5f0ac8..c475eceba 100644 --- a/transformations/macros/is_snowflake.sql +++ b/transformations/macros/is_snowflake.sql @@ -1,7 +1,7 @@ {% macro is_snowflake() %} - {% if target.name == 'snowflake' %} + {% if target.type == 'snowflake' %} {{ return(true) }} {% else %} {{ return(false) }} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %}