chore: Compile for all targets before packing (#359)

cloudquery · Dec 7, 2023 · 59d02cc · 59d02cc
1 parent 92a00be
commit 59d02cc
Show file tree

Hide file tree

Showing 15 changed files with 61 additions and 250 deletions.
diff --git a/.github/workflows/publish_transformation.yml b/.github/workflows/publish_transformation.yml
@@ -5,6 +5,15 @@ on:
     tags:
       - "transformation-*-v*.*.*"
 
+env:
+  SNOW_USER: ${{ secrets.SNOW_USER }}
+  SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }}
+  # DBT assumes the account is in the form of <account>.<region>
+  SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}"
+  SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }}
+  SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }}
+  SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }}
+  SNOW_REGION: ${{ secrets.SNOW_REGION }}
 jobs:
   prepare:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml b/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
           dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/aws/asset-inventory-free
-      - name: Unzip Packed DBT
-        run: unzip -o build/aws_asset_inventory_free.zip -d build/aws_asset_inventory_free
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/aws/asset-inventory-free/build/aws_asset_inventory_free
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
diff --git a/.github/workflows/transformations_aws_compliance_free_postgres.yml b/.github/workflows/transformations_aws_compliance_free_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
-          dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/aws/compliance-free
-      - name: Unzip Packed DBT
-        run: unzip -o build/aws_compliance_free.zip -d build/aws_compliance_free
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/aws/compliance-free/build/aws_compliance_free
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
+          dbt run --target dev-pg --profiles-dir ./tests
diff --git a/.github/workflows/transformations_aws_compliance_free_snowflake.yml b/.github/workflows/transformations_aws_compliance_free_snowflake.yml
@@ -42,7 +42,6 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/snowflake.yml
         env:

diff --git a/.github/workflows/transformations_aws_compliance_premium_postgres.yml b/.github/workflows/transformations_aws_compliance_premium_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
-          dbt run --target dev-pg --profiles-dir ./tests --exclude aws_compliance__foundational_security
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/aws/compliance-premium
-      - name: Unzip Packed DBT
-        run: unzip -o build/aws_compliance_premium.zip -d build/aws_compliance_premium
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/aws/compliance-premium/build/aws_compliance_premium
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests --exclude aws_compliance__foundational_security
+          dbt run --target dev-pg --profiles-dir ./tests --exclude aws_compliance__foundational_security
diff --git a/.github/workflows/transformations_aws_compliance_premium_snowflake.yml b/.github/workflows/transformations_aws_compliance_premium_snowflake.yml
@@ -42,7 +42,6 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/snowflake.yml
         env:

diff --git a/.github/workflows/transformations_aws_cost.yml b/.github/workflows/transformations_aws_cost.yml
@@ -57,7 +57,6 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
@@ -66,23 +65,3 @@ jobs:
         run: |
           dbt seed --target dev-pg --profiles-dir ./tests
           dbt run --target dev-pg --profiles-dir ./tests --vars 'cost_usage_table: cost_check'
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: |
-          node index.js dbt-pack --project-dir=../../transformations/aws/cost --dbt-arg=--vars --dbt-arg='cost_usage_table: cost_check'
-      - name: Unzip Packed DBT
-        run: unzip -o build/aws_cost.zip -d build/aws_cost
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/aws/cost/build/aws_cost
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests --vars 'cost_usage_table: cost_check'
diff --git a/.github/workflows/transformations_aws_data_resilience.yml b/.github/workflows/transformations_aws_data_resilience.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
-          dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/aws/data-resilience
-      - name: Unzip Packed DBT
-        run: unzip -o build/aws_data_resilience.zip -d build/aws_data_resilience
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/aws/data-resilience/build/aws_data_resilience
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
+          dbt run --target dev-pg --profiles-dir ./tests
diff --git a/.github/workflows/transformations_azure_compliance_free_postgres.yml b/.github/workflows/transformations_azure_compliance_free_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
-          dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/azure/compliance-free
-      - name: Unzip Packed DBT
-        run: unzip -o build/azure_compliance_free.zip -d build/azure_compliance_free
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/azure/compliance-free/build/azure_compliance_free
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
+          dbt run --target dev-pg --profiles-dir ./tests
diff --git a/.github/workflows/transformations_azure_compliance_premium_postgres.yml b/.github/workflows/transformations_azure_compliance_premium_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
           dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/azure/compliance-premium
-      - name: Unzip Packed DBT
-        run: unzip -o build/azure_compliance_premium.zip -d build/azure_compliance_premium
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/azure/compliance-premium/build/azure_compliance_premium
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
diff --git a/.github/workflows/transformations_gcp_compliance_free_postgres.yml b/.github/workflows/transformations_gcp_compliance_free_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
           dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/gcp/compliance-free
-      - name: Unzip Packed DBT
-        run: unzip -o build/gcp_compliance_free.zip -d build/gcp_compliance_free
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/gcp/compliance-free/build/gcp_compliance_free
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
diff --git a/.github/workflows/transformations_gcp_compliance_premium_postgres.yml b/.github/workflows/transformations_gcp_compliance_premium_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
           dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/gcp/compliance-premium
-      - name: Unzip Packed DBT
-        run: unzip -o build/gcp_compliance_premium.zip -d build/gcp_compliance_premium
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/gcp/compliance-premium/build/gcp_compliance_premium
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
diff --git a/.github/workflows/transformations_k8s_compliance_free_postgres.yml b/.github/workflows/transformations_k8s_compliance_free_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
-          dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-free
-      - name: Unzip Packed DBT
-        run: unzip -o build/k8s_compliance_free.zip -d build/k8s_compliance_free
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/k8s/compliance-free/build/k8s_compliance_free
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
+          dbt run --target dev-pg --profiles-dir ./tests
diff --git a/.github/workflows/transformations_k8s_compliance_premium_postgres.yml b/.github/workflows/transformations_k8s_compliance_premium_postgres.yml
@@ -57,30 +57,10 @@ jobs:
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
-      # Test unpacked version
       - name: Migrate DB
         run: cloudquery migrate tests/postgres.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Run Policies
         run: |
-          dbt run --target dev-pg --profiles-dir ./tests
-      # Test packed version
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 'lts/*'
-          cache: 'npm'
-          cache-dependency-path: scripts/dbt-pack/package-lock.json
-      - name: Install Dependencies
-        run: npm ci
-        working-directory: ./scripts/dbt-pack
-      - name: Pack DBT
-        working-directory: ./scripts/dbt-pack
-        run: node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-premium
-      - name: Unzip Packed DBT
-        run: unzip -o build/k8s_compliance_premium.zip -d build/k8s_compliance_premium
-      - name: Run Unpacked DBT
-        working-directory: ./transformations/k8s/compliance-premium/build/k8s_compliance_premium
-        run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
+          dbt run --target dev-pg --profiles-dir ./tests