Restore CI side-effects tests (#47)

* Restore CI side-effects tests * Add primary key to side effect table * Remove debug output from molecule CI job * Reset default delivery semantics to at-least-once
nephelaiio · Jul 6, 2023 · 79512ab · 79512ab
1 parent 5cd1379
commit 79512ab
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 38 deletions.
diff --git a/bin/test b/bin/test
@@ -5,20 +5,22 @@ function debug {
     make kubectl get all -- -n dataplane
     make strimzi-connector-trace
     make strimzi-connector-status
-    make kubectl get pod -- -n dataplane -o name | grep connector-restart | xargs -I{} make kubectl describe {} -- -n dataplane
-    make kubectl get jobs -- -n dataplane -o name | xargs -I{} make kubectl describe {} -- -n dataplane
-    make kubectl get jobs -- -n dataplane -o name | xargs -I{} make kubectl logs {} -- -n dataplane
-    make warehouse -- -c "'\dt'"
-    make kubectl get pod -- -A | grep -i metabase-app-setup | awk '{ print $2 }' | xargs -I{} make kubectl logs pod/{} -- -n dataplane
+    make kubectl describe svc/dataplane-connect-api -- -n dataplane
+    make kubectl logs svc/dataplane-connect-api -- -n dataplane
+    make kubectl describe job/dataplane-registry-connector-restart -- -n dataplane
+    make kubectl logs job/dataplane-registry-connector-restart -- -n dataplane
+    make kubectl describe job/dataplane-metabase-app-setup -- -n dataplane
+    make kubectl logs job/dataplane-metabase-app-setup -- -n dataplane
 }
 
-make template &&
+trap debug EXIT
+make template >/dev/null 2>&1 &&
     make molecule destroy &&
     make molecule create &&
-    make images &&
+    make images >/dev/null 2>&1 &&
     make molecule converge &&
-    make wait
-
-trap debug EXIT
-make molecule verify
+    make wait &&
+    make molecule verify &&
+    make molecule side-effect &&
+    make molecule verify
 trap - EXIT
diff --git a/charts/dataplane/Chart.yaml b/charts/dataplane/Chart.yaml
@@ -16,10 +16,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.17
+version: 0.1.18
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: 0.1.17
+appVersion: 0.1.18
diff --git a/charts/dataplane/templates/strimzi/connector.yaml b/charts/dataplane/templates/strimzi/connector.yaml
@@ -149,8 +149,9 @@ spec:
     tasks.max: "1"
     topics.regex: "cdc.(.*)"
     auto.create: true
-    pk.mode: "record_key"
-    pk.fields: ""
+    {{- with .Values.cdc.sink.config }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
     key.converter: io.confluent.connect.avro.AvroConverter
     key.converter.schema.registry.url: http://{{ $registryName }}:{{ $registryPort }}
     value.converter: io.confluent.connect.avro.AvroConverter

diff --git a/charts/dataplane/values.yaml b/charts/dataplane/values.yaml
@@ -7,10 +7,14 @@ util:
   image:
     repository: nephelaiio/dataplane-util
     pullPolicy: IfNotPresent
-    tag: dataplane-0.1.17
+    tag: dataplane-0.1.18
   resources: {}
 
 cdc:
+  sink:
+    config:
+      pk.mode: "none"
+      pk.fields: ""
   postgres: []
   #  postgres:
   #    - hostname: "network.hostname"
@@ -45,7 +49,7 @@ strimzi:
   connect:
     image:
       repository: nephelaiio/dataplane-connect
-      tag: dataplane-0.1.17
+      tag: dataplane-0.1.18
     replicas: 1
     config:
       group.id: connect-cluster
@@ -100,7 +104,7 @@ metabase:
     image:
       repository: nephelaiio/dataplane-util
       pullPolicy: IfNotPresent
-      tag: dataplane-0.1.17
+      tag: dataplane-0.1.18
     securityContext: {}
 
   image:

diff --git a/molecule/default/side_effect.yml b/molecule/default/side_effect.yml
@@ -6,15 +6,12 @@
   gather_facts: true
 
   vars:
-
     extra_table: languages
 
   roles:
-
     - nephelaiio.plugins
 
   tasks:
-
     - name: query pagila connection data
       ansible.builtin.set_fact:
         pagila_user: "{{ pagila_db_secret_data.data.username | b64decode }}"
@@ -24,10 +21,10 @@
         pagila_db_secret_data: "{{ secret_query | selectattr('metadata.name', 'equalto', pagila_db_secret_name) | first }}"
         secret_query: "{{
           query(
-            'kubernetes.core.k8s',
-            namespace=dataplane_pagila_namespace,
-            kind='Secret',
-            kubeconfig=k8s_kubeconfig
+          'kubernetes.core.k8s',
+          namespace=dataplane_pagila_namespace,
+          kind='Secret',
+          kubeconfig=k8s_kubeconfig
           )}}"
 
     - name: query pagila service data
@@ -38,10 +35,10 @@
         pagila_db_svc_data: "{{ service_query | selectattr('metadata.name', 'equalto', pagila_db_svc_name) | first }}"
         service_query: "{{
           query(
-            'kubernetes.core.k8s',
-            namespace=dataplane_pagila_namespace,
-            kind='Service',
-            kubeconfig=k8s_kubeconfig
+          'kubernetes.core.k8s',
+          namespace=dataplane_pagila_namespace,
+          kind='Service',
+          kubeconfig=k8s_kubeconfig
           )}}"
 
     - name: create pagila debezium signal schema

diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml
@@ -119,8 +119,8 @@
                 login_host: "{{ warehouse_host }}"
               vars:
                 warehouse_tables_expected: "{{ pagila_target_tables | map('map_format', 'pagila_public_%s') | list }}"
-                warehouse_tables_pagila: "{{ warehouse_tables_found | select('match', '^pagila_.*') }}"
-                warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^include_.*') }}"
+                warehouse_tables_pagila: "{{ warehouse_tables_found | select('match', '^pagila_public_.*') }}"
+                warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^include_public_.*') }}"
                 warehouse_tables_found: "{{ warehouse_table_query.query_result | map(attribute='table_name') | list }}"
               register: warehouse_table_query
               retries: 20
@@ -146,8 +146,8 @@
               ansible.builtin.debug:
                 msg: "table diff=[{{ ', '.join(warehouse_tables_diff) }}]"
               vars:
-                warehouse_tables_expected: ['include_public_staff']
-                warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^pagila_.*') }}"
+                warehouse_tables_expected: ["include_public_staff"]
+                warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^pagila_public_.*') }}"
                 warehouse_tables_found: "{{ warehouse_table_query.query_result | map(attribute='table_name') | list }}"
                 warehouse_tables_diff_expected: "{{ warehouse_tables_expected | difference(warehouse_tables_include) }}"
                 warehouse_tables_diff_found: "{{ warehouse_tables_include | difference(warehouse_tables_expected) }}"
@@ -164,10 +164,10 @@
             connector_present: "{{ connector_matches | int > 0 }}"
             connector_query: "{{
               query(
-                'kubernetes.core.k8s',
-                api_version='',
-                kind='KafkaConnector',
-                kubeconfig=k8s_kubeconfig
+              'kubernetes.core.k8s',
+              api_version='',
+              kind='KafkaConnector',
+              kubeconfig=k8s_kubeconfig
               ) }}"
           until: connector_present
           failed_when: not connector_present

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dataplane"
-version = "0.1.17"
+version = "0.1.18"
 description = ""
 authors = ["Teodoro Cook <[email protected]>"]