From 59d02cc527bb8ddc27c68ac04abca856a11d727d Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Thu, 7 Dec 2023 16:03:13 +0100 Subject: [PATCH] chore: Compile for all targets before packing (#359) --- .github/workflows/publish_transformation.yml | 9 +++ ...ions_aws_asset_inventory_free_postgres.yml | 20 ------ ...ormations_aws_compliance_free_postgres.yml | 22 +----- ...rmations_aws_compliance_free_snowflake.yml | 1 - ...ations_aws_compliance_premium_postgres.yml | 22 +----- ...tions_aws_compliance_premium_snowflake.yml | 1 - .../workflows/transformations_aws_cost.yml | 21 ------ .../transformations_aws_data_resilience.yml | 22 +----- ...mations_azure_compliance_free_postgres.yml | 22 +----- ...ions_azure_compliance_premium_postgres.yml | 20 ------ ...ormations_gcp_compliance_free_postgres.yml | 20 ------ ...ations_gcp_compliance_premium_postgres.yml | 20 ------ ...ormations_k8s_compliance_free_postgres.yml | 22 +----- ...ations_k8s_compliance_premium_postgres.yml | 22 +----- scripts/dbt-pack/src/index.js | 67 +++++++++++++------ 15 files changed, 61 insertions(+), 250 deletions(-) diff --git a/.github/workflows/publish_transformation.yml b/.github/workflows/publish_transformation.yml index a3cfdbeb4..095f84b39 100644 --- a/.github/workflows/publish_transformation.yml +++ b/.github/workflows/publish_transformation.yml @@ -5,6 +5,15 @@ on: tags: - "transformation-*-v*.*.*" +env: + SNOW_USER: ${{ secrets.SNOW_USER }} + SNOW_PASSWORD: ${{ secrets.SNOW_PASSWORD }} + # DBT assumes the account is in the form of . + SNOW_ACCOUNT: "${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}" + SNOW_WAREHOUSE: ${{ secrets.SNOW_WAREHOUSE }} + SNOW_DATABASE: ${{ secrets.SNOW_DATABASE }} + SNOW_SCHEMA: ${{ secrets.SNOW_SCHEMA }} + SNOW_REGION: ${{ secrets.SNOW_REGION }} jobs: prepare: runs-on: ubuntu-latest diff --git a/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml b/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml index 40c2d9e54..0ff0d7423 100644 --- a/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml +++ b/.github/workflows/transformations_aws_asset_inventory_free_postgres.yml @@ -57,7 +57,6 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: @@ -65,22 +64,3 @@ jobs: - name: Run Policies run: | dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/aws/asset-inventory-free - - name: Unzip Packed DBT - run: unzip -o build/aws_asset_inventory_free.zip -d build/aws_asset_inventory_free - - name: Run Unpacked DBT - working-directory: ./transformations/aws/asset-inventory-free/build/aws_asset_inventory_free - run: | - dbt run --target dev-pg --profiles-dir ../../tests diff --git a/.github/workflows/transformations_aws_compliance_free_postgres.yml b/.github/workflows/transformations_aws_compliance_free_postgres.yml index 82b26be69..4f10d2aae 100644 --- a/.github/workflows/transformations_aws_compliance_free_postgres.yml +++ b/.github/workflows/transformations_aws_compliance_free_postgres.yml @@ -57,30 +57,10 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - name: Run Policies run: | - dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/aws/compliance-free - - name: Unzip Packed DBT - run: unzip -o build/aws_compliance_free.zip -d build/aws_compliance_free - - name: Run Unpacked DBT - working-directory: ./transformations/aws/compliance-free/build/aws_compliance_free - run: | - dbt run --target dev-pg --profiles-dir ../../tests \ No newline at end of file + dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/.github/workflows/transformations_aws_compliance_free_snowflake.yml b/.github/workflows/transformations_aws_compliance_free_snowflake.yml index 97205324e..d84073292 100644 --- a/.github/workflows/transformations_aws_compliance_free_snowflake.yml +++ b/.github/workflows/transformations_aws_compliance_free_snowflake.yml @@ -42,7 +42,6 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/snowflake.yml env: diff --git a/.github/workflows/transformations_aws_compliance_premium_postgres.yml b/.github/workflows/transformations_aws_compliance_premium_postgres.yml index a7821e273..6b1d0acfa 100644 --- a/.github/workflows/transformations_aws_compliance_premium_postgres.yml +++ b/.github/workflows/transformations_aws_compliance_premium_postgres.yml @@ -57,30 +57,10 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - name: Run Policies run: | - dbt run --target dev-pg --profiles-dir ./tests --exclude aws_compliance__foundational_security - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/aws/compliance-premium - - name: Unzip Packed DBT - run: unzip -o build/aws_compliance_premium.zip -d build/aws_compliance_premium - - name: Run Unpacked DBT - working-directory: ./transformations/aws/compliance-premium/build/aws_compliance_premium - run: | - dbt run --target dev-pg --profiles-dir ../../tests --exclude aws_compliance__foundational_security \ No newline at end of file + dbt run --target dev-pg --profiles-dir ./tests --exclude aws_compliance__foundational_security \ No newline at end of file diff --git a/.github/workflows/transformations_aws_compliance_premium_snowflake.yml b/.github/workflows/transformations_aws_compliance_premium_snowflake.yml index e11eae80b..85c629a7b 100644 --- a/.github/workflows/transformations_aws_compliance_premium_snowflake.yml +++ b/.github/workflows/transformations_aws_compliance_premium_snowflake.yml @@ -42,7 +42,6 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/snowflake.yml env: diff --git a/.github/workflows/transformations_aws_cost.yml b/.github/workflows/transformations_aws_cost.yml index 3be531376..c48ea02af 100644 --- a/.github/workflows/transformations_aws_cost.yml +++ b/.github/workflows/transformations_aws_cost.yml @@ -57,7 +57,6 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: @@ -66,23 +65,3 @@ jobs: run: | dbt seed --target dev-pg --profiles-dir ./tests dbt run --target dev-pg --profiles-dir ./tests --vars 'cost_usage_table: cost_check' - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: | - node index.js dbt-pack --project-dir=../../transformations/aws/cost --dbt-arg=--vars --dbt-arg='cost_usage_table: cost_check' - - name: Unzip Packed DBT - run: unzip -o build/aws_cost.zip -d build/aws_cost - - name: Run Unpacked DBT - working-directory: ./transformations/aws/cost/build/aws_cost - run: | - dbt run --target dev-pg --profiles-dir ../../tests --vars 'cost_usage_table: cost_check' diff --git a/.github/workflows/transformations_aws_data_resilience.yml b/.github/workflows/transformations_aws_data_resilience.yml index 9e7a9eff4..439e6b683 100644 --- a/.github/workflows/transformations_aws_data_resilience.yml +++ b/.github/workflows/transformations_aws_data_resilience.yml @@ -57,30 +57,10 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - name: Run Policies run: | - dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/aws/data-resilience - - name: Unzip Packed DBT - run: unzip -o build/aws_data_resilience.zip -d build/aws_data_resilience - - name: Run Unpacked DBT - working-directory: ./transformations/aws/data-resilience/build/aws_data_resilience - run: | - dbt run --target dev-pg --profiles-dir ../../tests \ No newline at end of file + dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/.github/workflows/transformations_azure_compliance_free_postgres.yml b/.github/workflows/transformations_azure_compliance_free_postgres.yml index 614157779..2018fb2ed 100644 --- a/.github/workflows/transformations_azure_compliance_free_postgres.yml +++ b/.github/workflows/transformations_azure_compliance_free_postgres.yml @@ -57,30 +57,10 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - name: Run Policies run: | - dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/azure/compliance-free - - name: Unzip Packed DBT - run: unzip -o build/azure_compliance_free.zip -d build/azure_compliance_free - - name: Run Unpacked DBT - working-directory: ./transformations/azure/compliance-free/build/azure_compliance_free - run: | - dbt run --target dev-pg --profiles-dir ../../tests \ No newline at end of file + dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/.github/workflows/transformations_azure_compliance_premium_postgres.yml b/.github/workflows/transformations_azure_compliance_premium_postgres.yml index cf424e6eb..9421fac71 100644 --- a/.github/workflows/transformations_azure_compliance_premium_postgres.yml +++ b/.github/workflows/transformations_azure_compliance_premium_postgres.yml @@ -57,7 +57,6 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: @@ -65,22 +64,3 @@ jobs: - name: Run Policies run: | dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/azure/compliance-premium - - name: Unzip Packed DBT - run: unzip -o build/azure_compliance_premium.zip -d build/azure_compliance_premium - - name: Run Unpacked DBT - working-directory: ./transformations/azure/compliance-premium/build/azure_compliance_premium - run: | - dbt run --target dev-pg --profiles-dir ../../tests diff --git a/.github/workflows/transformations_gcp_compliance_free_postgres.yml b/.github/workflows/transformations_gcp_compliance_free_postgres.yml index c638c0376..7003f4303 100644 --- a/.github/workflows/transformations_gcp_compliance_free_postgres.yml +++ b/.github/workflows/transformations_gcp_compliance_free_postgres.yml @@ -57,7 +57,6 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: @@ -65,22 +64,3 @@ jobs: - name: Run Policies run: | dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/gcp/compliance-free - - name: Unzip Packed DBT - run: unzip -o build/gcp_compliance_free.zip -d build/gcp_compliance_free - - name: Run Unpacked DBT - working-directory: ./transformations/gcp/compliance-free/build/gcp_compliance_free - run: | - dbt run --target dev-pg --profiles-dir ../../tests diff --git a/.github/workflows/transformations_gcp_compliance_premium_postgres.yml b/.github/workflows/transformations_gcp_compliance_premium_postgres.yml index 723d6c9b0..6c47f8ae9 100644 --- a/.github/workflows/transformations_gcp_compliance_premium_postgres.yml +++ b/.github/workflows/transformations_gcp_compliance_premium_postgres.yml @@ -57,7 +57,6 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: @@ -65,22 +64,3 @@ jobs: - name: Run Policies run: | dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/gcp/compliance-premium - - name: Unzip Packed DBT - run: unzip -o build/gcp_compliance_premium.zip -d build/gcp_compliance_premium - - name: Run Unpacked DBT - working-directory: ./transformations/gcp/compliance-premium/build/gcp_compliance_premium - run: | - dbt run --target dev-pg --profiles-dir ../../tests diff --git a/.github/workflows/transformations_k8s_compliance_free_postgres.yml b/.github/workflows/transformations_k8s_compliance_free_postgres.yml index 35bb05ac9..078ab384e 100644 --- a/.github/workflows/transformations_k8s_compliance_free_postgres.yml +++ b/.github/workflows/transformations_k8s_compliance_free_postgres.yml @@ -57,30 +57,10 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - name: Run Policies run: | - dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-free - - name: Unzip Packed DBT - run: unzip -o build/k8s_compliance_free.zip -d build/k8s_compliance_free - - name: Run Unpacked DBT - working-directory: ./transformations/k8s/compliance-free/build/k8s_compliance_free - run: | - dbt run --target dev-pg --profiles-dir ../../tests + dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/.github/workflows/transformations_k8s_compliance_premium_postgres.yml b/.github/workflows/transformations_k8s_compliance_premium_postgres.yml index 89aad915a..36f7d0b32 100644 --- a/.github/workflows/transformations_k8s_compliance_premium_postgres.yml +++ b/.github/workflows/transformations_k8s_compliance_premium_postgres.yml @@ -57,30 +57,10 @@ jobs: uses: cloudquery/setup-cloudquery@v3 with: version: v4.0.0 - # Test unpacked version - name: Migrate DB run: cloudquery migrate tests/postgres.yml env: CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres - name: Run Policies run: | - dbt run --target dev-pg --profiles-dir ./tests - # Test packed version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: 'npm' - cache-dependency-path: scripts/dbt-pack/package-lock.json - - name: Install Dependencies - run: npm ci - working-directory: ./scripts/dbt-pack - - name: Pack DBT - working-directory: ./scripts/dbt-pack - run: node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-premium - - name: Unzip Packed DBT - run: unzip -o build/k8s_compliance_premium.zip -d build/k8s_compliance_premium - - name: Run Unpacked DBT - working-directory: ./transformations/k8s/compliance-premium/build/k8s_compliance_premium - run: | - dbt run --target dev-pg --profiles-dir ../../tests + dbt run --target dev-pg --profiles-dir ./tests \ No newline at end of file diff --git a/scripts/dbt-pack/src/index.js b/scripts/dbt-pack/src/index.js index a53b8556d..7798d79fd 100644 --- a/scripts/dbt-pack/src/index.js +++ b/scripts/dbt-pack/src/index.js @@ -29,24 +29,46 @@ const validateProjectDirectory = async (dbtProjectDirectory) => { }; const compileDbtProject = async (dbtProjectDirectory, dbtArguments) => { + const profilesFile = `${dbtProjectDirectory}/tests/profiles.yml`; + const profiles = parse(await fs.readFile(profilesFile, "utf8")); + const profileName = Object.keys(profiles)[0]; + const targets = Object.keys(profiles[profileName].outputs); const fullProjectDirectory = path.resolve(dbtProjectDirectory); - console.log(`Compiling dbt project in ${fullProjectDirectory}`); - await execa( - "dbt", - ["compile", "--target", "dev-pg", "--profiles-dir", "tests", ...dbtArguments], - { - cwd: fullProjectDirectory, - stdout: "inherit", - stderr: "inherit", - }, + for (const target of targets) { + console.log( + `Compiling dbt project in ${fullProjectDirectory} with target: '${target}'`, + ); + await execa( + "dbt", + [ + "compile", + "--target", + target, + "--profiles-dir", + "tests", + "--target-path", + `target/${target}`, + ...dbtArguments, + ], + { + cwd: fullProjectDirectory, + stdout: "inherit", + stderr: "inherit", + }, + ); + console.log(`Done compiling dbt project in ${fullProjectDirectory}`); + } + + const targetDirectories = targets.map( + (target) => `${fullProjectDirectory}/target/${target}`, ); - console.log(`Done compiling dbt project in ${fullProjectDirectory}`); + return targetDirectories; }; const addDependencies = (node, allNodes, allMacros, filesToPack) => { if (node.unique_id.startsWith("macro.dbt.")) { // Skip dbt internal macros - return + return; } filesToPack.add(node.original_file_path); const dependsOnNodes = node.depends_on?.nodes ?? []; @@ -59,16 +81,19 @@ const addDependencies = (node, allNodes, allMacros, filesToPack) => { } }; -const analyzeManifestFile = async (dbtProjectDirectory) => { - const manifestFile = `${dbtProjectDirectory}/target/manifest.json`; - console.log(`Analyzing manifest file ${manifestFile}`); - const manifest = JSON.parse(await fs.readFile(manifestFile, "utf8")); - const { nodes: allNodes, macros: allMacros } = manifest; - +const analyzeManifestFiles = async (targetDirectories) => { const filesToPack = new Set(); - for (const node of Object.values(allNodes)) { - addDependencies(node, allNodes, allMacros, filesToPack); + for (const targetDirectory of targetDirectories) { + const manifestFile = `${targetDirectory}/manifest.json`; + console.log(`Analyzing manifest file ${manifestFile}`); + const manifest = JSON.parse(await fs.readFile(manifestFile, "utf8")); + const { nodes: allNodes, macros: allMacros } = manifest; + + for (const node of Object.values(allNodes)) { + addDependencies(node, allNodes, allMacros, filesToPack); + } } + return [...filesToPack]; }; @@ -129,7 +154,7 @@ const zipProject = async (dbtProjectDirectory, filesToPack) => { export const pack = async ({ projectDir, dbtArgs }) => { await validateProjectDirectory(projectDir); - await compileDbtProject(projectDir, dbtArgs); - const filesToPack = await analyzeManifestFile(projectDir); + const targetDirectories = await compileDbtProject(projectDir, dbtArgs); + const filesToPack = await analyzeManifestFiles(targetDirectories); await zipProject(projectDir, filesToPack); };