merging main

cloudquery · Nov 26, 2024 · 4fcbbe3 · 4fcbbe3
1 parent ccb5606
commit 4fcbbe3
Show file tree

Hide file tree

Showing 84 changed files with 812 additions and 300 deletions.
diff --git a/.github/clickhouse.xml b/.github/clickhouse.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <profiles>
+        <default>
+            <max_query_size>10000000</max_query_size>
+            <max_ast_elements>150000</max_ast_elements>
+        </default>
+    </profiles>
+</clickhouse>
diff --git a/.github/workflows/publish_transformation.yml b/.github/workflows/publish_transformation.yml
@@ -26,6 +26,7 @@ jobs:
       snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }}
       bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }}
       s3: ${{ fromJson(steps.set-result.outputs.result).s3 }}
+      clickhouse: ${{ fromJson(steps.set-result.outputs.result).clickhouse }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -40,19 +41,21 @@ jobs:
             const [_, topLevelDir, ...rest] = TRANSFORMATION_TAG.split('-');
             const version = rest.pop();
             const transformation_dir = `transformations/${topLevelDir}/${rest.join('-')}`;
-            const [postgres, snowflake, bigquery, s3] = await Promise.all([
+            const [postgres, snowflake, bigquery, s3, clickhouse] = await Promise.all([
               fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/s3.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
+              fs.access(`${transformation_dir}/tests/clickhouse.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
             ]);
             return {
               transformation_dir,
               transformation_version: version,
               postgres,
               snowflake,
               bigquery,
-              s3
+              s3,
+              clickhouse,
             };
   publish-transformation:
     permissions:
@@ -88,7 +91,7 @@ jobs:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: arn:aws:iam::615713231484:role/cq-playground-aws-github-action
+          role-to-assume: arn:aws:iam::590184131402:role/cq-integration-tests-aws-github-action
           aws-region: us-east-1
       - name: Update Release Notes
         uses: actions/github-script@v7
@@ -118,7 +121,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Migrate DB Postgres
         if: needs.prepare.outputs.postgres == 'true'
         run: cloudquery migrate tests/postgres.yml
@@ -139,6 +142,24 @@ jobs:
         if: needs.prepare.outputs.s3 == 'true'
         run: cloudquery migrate tests/s3.yml
         working-directory: ${{ needs.prepare.outputs.transformation_dir }}
+      - name: Start ClickHouse Container
+        if: needs.prepare.outputs.clickhouse == 'true'
+        run: |
+          docker run --platform linux/amd64 -d --name clickhouse-server --rm -p 8123:8123 -p 9000:9000 \
+            -e CLICKHOUSE_PASSWORD=test \
+            -e CLICKHOUSE_USER=cq \
+            -e CLICKHOUSE_DB=cloudquery \
+            -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
+            -v ${{ github.workspace }}/.github/clickhouse.xml:/etc/clickhouse-server/users.d/cloudquery.xml \
+            clickhouse/clickhouse-server:22.1.2
+          sudo apt update && sudo apt install wait-for-it -y
+          wait-for-it -h localhost -p 9000
+      - name: Migrate DB Clickhouse
+        if: needs.prepare.outputs.clickhouse == 'true'
+        working-directory: ${{ needs.prepare.outputs.transformation_dir }}
+        env:
+          CLICKHOUSE_CONNECTION_STRING: "clickhouse://cq:test@localhost:9000/cloudquery"
+        run: cloudquery migrate tests/clickhouse.yml
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
@@ -154,7 +175,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Publish tranformation
         working-directory: ${{ needs.prepare.outputs.transformation_dir }}
         env:

diff --git a/.github/workflows/publish_visualization_aws_asset_inventory.yml b/.github/workflows/publish_visualization_aws_asset_inventory.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Publish visualization
         env:
           CLOUDQUERY_API_KEY: ${{ secrets.CLOUDQUERY_API_KEY }}

diff --git a/.github/workflows/publish_visualization_aws_compliance.yml b/.github/workflows/publish_visualization_aws_compliance.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Publish visualization
         env:
           CLOUDQUERY_API_KEY: ${{ secrets.CLOUDQUERY_API_KEY }}

diff --git a/.github/workflows/publish_visualization_aws_data_resilience.yml b/.github/workflows/publish_visualization_aws_data_resilience.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Publish visualization
         env:
           CLOUDQUERY_API_KEY: ${{ secrets.CLOUDQUERY_API_KEY }}

diff --git a/.github/workflows/transformations_aws_asset_inventory_free.yml b/.github/workflows/transformations_aws_asset_inventory_free.yml
@@ -39,6 +39,7 @@ jobs:
       postgres: ${{ fromJson(steps.set-result.outputs.result).postgres }}
       snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }}
       bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }}
+      clickhouse: ${{ fromJson(steps.set-result.outputs.result).clickhouse }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -50,17 +51,19 @@ jobs:
           script: |
             const fs = require('fs/promises');
             const { TRANSFORMATION_DIR: transformation_dir } = process.env;
-            const [postgres, snowflake, bigquery] = await Promise.all([
+            const [postgres, snowflake, bigquery, clickhouse] = await Promise.all([
               fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
+              fs.access(`${transformation_dir}/tests/clickhouse.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
             ]);
             console.log(JSON.stringify({ transformation_dir, postgres, snowflake, bigquery }));
             return {
               transformation_dir,
               postgres,
               snowflake,
               bigquery,
+              clickhouse,
             };
   transformations-aws-asset-inventory-free:
     permissions:
@@ -107,7 +110,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml
@@ -127,3 +130,22 @@ jobs:
         run: |
           cloudquery migrate tests/bigquery.yml
           dbt run --target dev-bigquery --profiles-dir ./tests
+      - name: Start ClickHouse Container
+        if: needs.prepare.outputs.clickhouse == 'true'
+        run: |
+          docker run --platform linux/amd64 -d --name clickhouse-server --rm -p 8123:8123 -p 9000:9000 \
+            -e CLICKHOUSE_PASSWORD=test \
+            -e CLICKHOUSE_USER=cq \
+            -e CLICKHOUSE_DB=cloudquery \
+            -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
+            -v ${{ github.workspace }}/.github/clickhouse.xml:/etc/clickhouse-server/users.d/cloudquery.xml \
+            clickhouse/clickhouse-server:22.1.2
+          sudo apt update && sudo apt install wait-for-it -y
+          wait-for-it -h localhost -p 9000 -t 120
+      - name: Test Clickhouse
+        if: needs.prepare.outputs.clickhouse == 'true'
+        env:
+          CLICKHOUSE_CONNECTION_STRING: "clickhouse://cq:test@localhost:9000/cloudquery"
+        run: |
+          cloudquery migrate tests/clickhouse.yml
+          dbt run --target dev-clickhouse --profiles-dir ./tests
diff --git a/.github/workflows/transformations_aws_compliance_premium.yml b/.github/workflows/transformations_aws_compliance_premium.yml
@@ -104,7 +104,7 @@ jobs:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: arn:aws:iam::615713231484:role/cq-playground-aws-github-action
+          role-to-assume: arn:aws:iam::590184131402:role/cq-integration-tests-aws-github-action
           aws-region: us-east-1
       - uses: actions/setup-python@v5
         with:
@@ -116,7 +116,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_aws_cost.yml b/.github/workflows/transformations_aws_cost.yml
@@ -106,7 +106,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_aws_data_resilience.yml b/.github/workflows/transformations_aws_data_resilience.yml
@@ -106,7 +106,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_aws_encryption.yml b/.github/workflows/transformations_aws_encryption.yml
@@ -106,7 +106,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_azure_asset_inventory_free.yml b/.github/workflows/transformations_azure_asset_inventory_free.yml
@@ -107,7 +107,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_azure_compliance_premium.yml b/.github/workflows/transformations_azure_compliance_premium.yml
@@ -108,7 +108,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_docs_deploy.yml b/.github/workflows/transformations_docs_deploy.yml
@@ -44,12 +44,10 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.9"
-      - name: Install dbt packages
-        run: pip install dbt-postgres
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Generate the docs
         run: make shard=${{ matrix.shard }} gen-site
         env:

diff --git a/.github/workflows/transformations_gcp_asset_inventory_free.yml b/.github/workflows/transformations_gcp_asset_inventory_free.yml
@@ -107,7 +107,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_gcp_compliance_premium.yml b/.github/workflows/transformations_gcp_compliance_premium.yml
@@ -108,7 +108,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/transformations_k8s_compliance_premium.yml b/.github/workflows/transformations_k8s_compliance_premium.yml
@@ -108,7 +108,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml

diff --git a/.github/workflows/validate_transformation_release.yml b/.github/workflows/validate_transformation_release.yml
@@ -26,6 +26,7 @@ jobs:
       snowflake: ${{ fromJson(steps.set-result.outputs.result).snowflake }}
       bigquery: ${{ fromJson(steps.set-result.outputs.result).bigquery }}
       s3: ${{ fromJson(steps.set-result.outputs.result).s3 }}
+      clickhouse: ${{ fromJson(steps.set-result.outputs.result).clickhouse }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -43,19 +44,21 @@ jobs:
             const transformation_dir = `transformations/${topLevelDir}/${rest.join('-')}`;
             const manifestFile = `${transformation_dir}/manifest.json`;
             const zipPath = JSON.parse(await fs.readFile(manifestFile)).path;
-            const [postgres, snowflake, bigquery, s3] = await Promise.all([
+            const [postgres, snowflake, bigquery, s3, clickhouse] = await Promise.all([
               fs.access(`${transformation_dir}/tests/postgres.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/snowflake.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/bigquery.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
               fs.access(`${transformation_dir}/tests/s3.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
+              fs.access(`${transformation_dir}/tests/clickhouse.yml`, fs.constants.F_OK).then(() => true).catch(() => false),
             ]);
             return {
               transformation_dir,
               zipPath: path.resolve(transformation_dir, zipPath),
               postgres,
               snowflake,
               bigquery,
-              s3
+              s3,
+              clickhouse,
             };
   validate-transformation:
     permissions:
@@ -91,7 +94,7 @@ jobs:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: arn:aws:iam::615713231484:role/cq-playground-aws-github-action
+          role-to-assume: arn:aws:iam::590184131402:role/cq-integration-tests-aws-github-action
           aws-region: us-east-1
       - uses: actions/setup-python@v5
         with:
@@ -104,7 +107,7 @@ jobs:
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v4
         with:
-          version: v6.8.2
+          version: v6.11.0
       - name: Migrate DB Postgres
         if: needs.prepare.outputs.postgres == 'true'
         run: cloudquery migrate tests/postgres.yml
@@ -125,6 +128,24 @@ jobs:
         if: needs.prepare.outputs.s3 == 'true'
         run: cloudquery migrate tests/s3.yml
         working-directory: ${{ needs.prepare.outputs.transformation_dir }}
+      - name: Start ClickHouse Container
+        if: needs.prepare.outputs.clickhouse == 'true'
+        run: |
+          docker run --platform linux/amd64 -d --name clickhouse-server --rm -p 8123:8123 -p 9000:9000 \
+            -e CLICKHOUSE_PASSWORD=test \
+            -e CLICKHOUSE_USER=cq \
+            -e CLICKHOUSE_DB=cloudquery \
+            -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
+            -v ${{ github.workspace }}/.github/clickhouse.xml:/etc/clickhouse-server/users.d/cloudquery.xml \
+            clickhouse/clickhouse-server:22.1.2
+          sudo apt update && sudo apt install wait-for-it -y
+          wait-for-it -h localhost -p 9000
+      - name: Migrate DB Clickhouse
+        if: needs.prepare.outputs.clickhouse == 'true'
+        working-directory: ${{ needs.prepare.outputs.transformation_dir }}
+        env:
+          CLICKHOUSE_CONNECTION_STRING: "clickhouse://cq:test@localhost:9000/cloudquery"
+        run: cloudquery migrate tests/clickhouse.yml
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with: