chore: Reorganize k8s

.github/workflows/transformations_azure_compliance_free_postgres.yml

@@ -50,7 +50,7 @@ jobs:
         with:
           version: v4.0.0
       - name: Migrate DB
-        run: cloudquery migrate tests/azure.yml tests/pg.yml
+        run: cloudquery migrate tests/spec.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: DBT dependencies

.github/workflows/transformations_azure_compliance_premium_postgres.yml

@@ -50,7 +50,7 @@ jobs:
         with:
           version: v4.0.0
       - name: Migrate DB
-        run: cloudquery migrate tests/azure.yml tests/pg.yml
+        run: cloudquery migrate tests/spec.yml
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: DBT dependencies

.github/workflows/transformations_k8s_compliance.yml → .github/workflows/transformations_k8s_compliance_free_postgres.yml

@@ -1,24 +1,25 @@
-name: Test K8s Policies
+name: "Test K8S Compliance Free Policies: Postgres"
 
 on:
   pull_request:
     paths:
-      - "transformations/k8s_compliance/**"
-      - ".github/workflows/transformations_k8s_compliance.yml"
+      - "transformations/k8s/compliance-free/**"
+      - ".github/workflows/transformations_k8s_compliance_free_postgres.yml"
   push:
     branches:
       - main
     paths:
-      - "transformations/k8s_compliance/**"
-      - ".github/workflows/transformations_k8s_compliance.yml"
+      - "transformations/k8s/compliance-free/**"
+      - ".github/workflows/transformations_k8s_compliance_free_postgres.yml"
 
 jobs:
-  test-policies:
+  transformations-k8s-compliance-free:
+    name: transformations/k8s/compliance-free
     timeout-minutes: 30
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: ./transformations/k8s_compliance
+        working-directory: ./transformations/k8s/compliance-free
     services:
       postgres:
         image: postgres:11
@@ -34,15 +35,14 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
           cache: "pip"
-          cache-dependency-path: "./transformations/k8s_compliance/requirements.txt"
+          cache-dependency-path: "./transformations/k8s/compliance-free/requirements.txt"
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Setup CloudQuery
@@ -52,10 +52,10 @@ jobs:
       - name: Migrate DB
         run: cloudquery migrate tests/spec.yml
         env:
-          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres  
+          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: DBT dependencies
         run: |
-          dbt deps --target dev --profiles-dir ./tests 
+          dbt deps --target dev-pg --profiles-dir ./tests
       - name: Run Policies
         run: |
-          dbt run --target dev --profiles-dir ./tests
+          dbt run --target dev-pg --profiles-dir ./tests

.github/workflows/transformations_k8s_compliance_premium_postgres.yml

@@ -0,0 +1,61 @@
+name: "Test K8S Compliance Premium Policies: Postgres"
+
+on:
+  pull_request:
+    paths:
+      - "transformations/k8s/compliance-premium/**"
+      - ".github/workflows/transformations_k8s_compliance_premium_postgres.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "transformations/k8s/compliance-premium/**"
+      - ".github/workflows/transformations_k8s_compliance_premium_postgres.yml"
+
+jobs:
+  transformations-k8s-compliance-premium:
+    name: transformations/k8s/compliance-premium
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./transformations/k8s/compliance-premium
+    services:
+      postgres:
+        image: postgres:11
+        env:
+          POSTGRES_PASSWORD: pass
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: "./transformations/k8s/compliance-premium/requirements.txt"
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Setup CloudQuery
+        uses: cloudquery/setup-cloudquery@v3
+        with:
+          version: v4.0.0
+      - name: Migrate DB
+        run: cloudquery migrate tests/spec.yml
+        env:
+          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
+      - name: DBT dependencies
+        run: |
+          dbt deps --target dev-pg --profiles-dir ./tests
+      - name: Run Policies
+        run: |
+          dbt run --target dev-pg --profiles-dir ./tests

release-please-config-free.json

@@ -8,10 +8,6 @@
       "component": "transformation-gcp-compliance-free",
       "changelog-path": "CHANGELOG-free.md"
     },
-    "transformations/k8s_compliance": {
-      "component": "transformation-k8s-compliance-free",
-      "changelog-path": "CHANGELOG-free.md"
-    },
     "transformations/aws_data_resilience": {
       "component": "transformation-aws-data-resilience",
       "changelog-path": "CHANGELOG.md"

release-please-config-premium.json

@@ -7,10 +7,6 @@
     "transformations/gcp_compliance": {
       "component": "transformation-gcp-compliance-premium",
       "changelog-path": "CHANGELOG-premium.md"
-    },
-    "transformations/k8s_compliance": {
-      "component": "transformation-k8s-compliance-premium",
-      "changelog-path": "CHANGELOG-premium.md"
     }
   },
   "pull-request-title-pattern": "chore${scope}: Release${component} v${version}",

scripts/dbt-pack/README.md

@@ -17,4 +17,7 @@ npm ci
 ```bash
 node index.js dbt-pack --project-dir=../../transformations/azure/compliance-free
 node index.js dbt-pack --project-dir=../../transformations/azure/compliance-premium
+
+node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-free
+node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-premium
 ```

scripts/dbt-pack/src/index.js

@@ -52,9 +52,10 @@ const addDependencies = (node, allNodes, allMacros, filesToPack) => {
 };
 
 const analyzeManifestFile = async (dbtProjectDirectory) => {
-  console.log(`Reading manifest.json from ${dbtProjectDirectory}`);
+  const manifestFile = `${dbtProjectDirectory}/target/manifest.json`;
+  console.log(`Analyzing manifest file ${manifestFile}`);
   const manifest = JSON.parse(
-    await fs.readFile(`${dbtProjectDirectory}/target/manifest.json`, "utf8"),
+    await fs.readFile(manifestFile, "utf8"),
   );
   const { nodes: allNodes, macros: allMacros } = manifest;
 
@@ -114,6 +115,7 @@ const zipProject = async (dbtProjectDirectory, filesToPack, ) => {
       stdout: "inherit",
       stderr: "inherit",
     });
+    console.log(`Done packing to ${outputFile}`);
   });
 };
 

transformations/azure/compliance-free/tests/spec.yml

@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: azure
+  path: cloudquery/azure
+  registry: cloudquery
+  version: "v10.2.0" # latest version of source azure plugin
+  destinations: ["postgresql"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: "postgresql"
+  path: "cloudquery/postgresql"
+  registry: cloudquery
+  version: "v7.1.0" # latest version of postgresql plugin
+  spec:
+    batch_size: 10000
+    connection_string: ${CQ_DSN}

transformations/azure/compliance-premium/tests/spec.yml

@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: azure
+  path: cloudquery/azure
+  registry: cloudquery
+  version: "v10.2.0" # latest version of source azure plugin
+  destinations: ["postgresql"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: "postgresql"
+  path: "cloudquery/postgresql"
+  registry: cloudquery
+  version: "v7.1.0" # latest version of postgresql plugin
+  spec:
+    batch_size: 10000
+    connection_string: ${CQ_DSN}

transformations/k8s_compliance/dbt_project.yml → transformations/k8s/compliance-free/dbt_project.yml

@@ -12,11 +12,11 @@ profile: 'k8s_compliance'
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
-model-paths: ["models"]
+model-paths: ["models","../models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
-macro-paths: ["macros"]
+macro-paths: ["macros","../macros"]
 snapshot-paths: ["snapshots"]
 
 clean-targets:         # directories to be removed by `dbt clean`

transformations/k8s_compliance/manifest-free.json → transformations/k8s/compliance-free/manifest.json

@@ -6,7 +6,7 @@
   "addon_type": "transformation",
   "addon_format": "zip",
   "message": "@./changelog.md",
-  "doc": "./README-free.md",
+  "doc": "./README.md",
   "path": "./build/k8s_compliance_free.zip",
   "plugin_deps": ["cloudquery/source/[email protected]"],
   "addon_deps": []

transformations/k8s_compliance/tests/profiles.yml → transformations/k8s/compliance-free/tests/profiles.yml

@@ -1,7 +1,7 @@
 k8s_compliance: # This should match the name in your dbt_project.yml
   target: dev
   outputs:
-    dev:
+    dev-pg:
       type: postgres
       host: 127.0.0.1
       user: postgres

transformations/k8s_compliance/tests/spec.yml → transformations/k8s/compliance-free/tests/spec.yml

@@ -4,7 +4,7 @@ spec:
   name: k8s
   path: cloudquery/k8s
   registry: cloudquery
-  version: "v5.2.1"
+  version: "v5.2.1" # latest version of source k8s plugin
   tables: ["*"]
   destinations: ["postgresql"]
   spec:
@@ -14,7 +14,8 @@ kind: destination
 spec:
   name: "postgresql"
   path: "cloudquery/postgresql"
-  version: "v6.1.1"
+  registry: cloudquery
+  version: "v7.1.0" # latest version of postgresql plugin
   spec:
     batch_size: 10000
     connection_string: ${CQ_DSN}

transformations/k8s/compliance-premium/dbt_project.yml

@@ -0,0 +1,40 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'k8s_compliance'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'k8s_compliance'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models","../models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros","../macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+models:
+  k8s_compliance:
+    # Config indicated by + and applies to all files under models/example/
+    # example:
+    #   +materialized: view
+
+
+
+