From 79512abe15803351546b884b9ec066c2d199683d Mon Sep 17 00:00:00 2001 From: Teodoro Cook Date: Thu, 6 Jul 2023 14:47:05 -0600 Subject: [PATCH] Restore CI side-effects tests (#47) * Restore CI side-effects tests * Add primary key to side effect table * Remove debug output from molecule CI job * Reset default delivery semantics to at-least-once --- bin/test | 24 ++++++++++--------- charts/dataplane/Chart.yaml | 4 ++-- .../templates/strimzi/connector.yaml | 5 ++-- charts/dataplane/values.yaml | 10 +++++--- molecule/default/side_effect.yml | 19 +++++++-------- molecule/default/verify.yml | 16 ++++++------- pyproject.toml | 2 +- 7 files changed, 42 insertions(+), 38 deletions(-) diff --git a/bin/test b/bin/test index a2789ea..ad51e5d 100755 --- a/bin/test +++ b/bin/test @@ -5,20 +5,22 @@ function debug { make kubectl get all -- -n dataplane make strimzi-connector-trace make strimzi-connector-status - make kubectl get pod -- -n dataplane -o name | grep connector-restart | xargs -I{} make kubectl describe {} -- -n dataplane - make kubectl get jobs -- -n dataplane -o name | xargs -I{} make kubectl describe {} -- -n dataplane - make kubectl get jobs -- -n dataplane -o name | xargs -I{} make kubectl logs {} -- -n dataplane - make warehouse -- -c "'\dt'" - make kubectl get pod -- -A | grep -i metabase-app-setup | awk '{ print $2 }' | xargs -I{} make kubectl logs pod/{} -- -n dataplane + make kubectl describe svc/dataplane-connect-api -- -n dataplane + make kubectl logs svc/dataplane-connect-api -- -n dataplane + make kubectl describe job/dataplane-registry-connector-restart -- -n dataplane + make kubectl logs job/dataplane-registry-connector-restart -- -n dataplane + make kubectl describe job/dataplane-metabase-app-setup -- -n dataplane + make kubectl logs job/dataplane-metabase-app-setup -- -n dataplane } -make template && +trap debug EXIT +make template >/dev/null 2>&1 && make molecule destroy && make molecule create && - make images && + make images >/dev/null 2>&1 && make molecule converge && - make wait - -trap debug EXIT -make molecule verify + make wait && + make molecule verify && + make molecule side-effect && + make molecule verify trap - EXIT diff --git a/charts/dataplane/Chart.yaml b/charts/dataplane/Chart.yaml index c33e809..605aa57 100644 --- a/charts/dataplane/Chart.yaml +++ b/charts/dataplane/Chart.yaml @@ -16,10 +16,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.17 +version: 0.1.18 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.1.17 +appVersion: 0.1.18 diff --git a/charts/dataplane/templates/strimzi/connector.yaml b/charts/dataplane/templates/strimzi/connector.yaml index 493e6da..6b6999d 100644 --- a/charts/dataplane/templates/strimzi/connector.yaml +++ b/charts/dataplane/templates/strimzi/connector.yaml @@ -149,8 +149,9 @@ spec: tasks.max: "1" topics.regex: "cdc.(.*)" auto.create: true - pk.mode: "record_key" - pk.fields: "" + {{- with .Values.cdc.sink.config }} + {{- toYaml . | nindent 4 }} + {{- end }} key.converter: io.confluent.connect.avro.AvroConverter key.converter.schema.registry.url: http://{{ $registryName }}:{{ $registryPort }} value.converter: io.confluent.connect.avro.AvroConverter diff --git a/charts/dataplane/values.yaml b/charts/dataplane/values.yaml index 6622d62..b2372a3 100644 --- a/charts/dataplane/values.yaml +++ b/charts/dataplane/values.yaml @@ -7,10 +7,14 @@ util: image: repository: nephelaiio/dataplane-util pullPolicy: IfNotPresent - tag: dataplane-0.1.17 + tag: dataplane-0.1.18 resources: {} cdc: + sink: + config: + pk.mode: "none" + pk.fields: "" postgres: [] # postgres: # - hostname: "network.hostname" @@ -45,7 +49,7 @@ strimzi: connect: image: repository: nephelaiio/dataplane-connect - tag: dataplane-0.1.17 + tag: dataplane-0.1.18 replicas: 1 config: group.id: connect-cluster @@ -100,7 +104,7 @@ metabase: image: repository: nephelaiio/dataplane-util pullPolicy: IfNotPresent - tag: dataplane-0.1.17 + tag: dataplane-0.1.18 securityContext: {} image: diff --git a/molecule/default/side_effect.yml b/molecule/default/side_effect.yml index d1e571c..1257252 100644 --- a/molecule/default/side_effect.yml +++ b/molecule/default/side_effect.yml @@ -6,15 +6,12 @@ gather_facts: true vars: - extra_table: languages roles: - - nephelaiio.plugins tasks: - - name: query pagila connection data ansible.builtin.set_fact: pagila_user: "{{ pagila_db_secret_data.data.username | b64decode }}" @@ -24,10 +21,10 @@ pagila_db_secret_data: "{{ secret_query | selectattr('metadata.name', 'equalto', pagila_db_secret_name) | first }}" secret_query: "{{ query( - 'kubernetes.core.k8s', - namespace=dataplane_pagila_namespace, - kind='Secret', - kubeconfig=k8s_kubeconfig + 'kubernetes.core.k8s', + namespace=dataplane_pagila_namespace, + kind='Secret', + kubeconfig=k8s_kubeconfig )}}" - name: query pagila service data @@ -38,10 +35,10 @@ pagila_db_svc_data: "{{ service_query | selectattr('metadata.name', 'equalto', pagila_db_svc_name) | first }}" service_query: "{{ query( - 'kubernetes.core.k8s', - namespace=dataplane_pagila_namespace, - kind='Service', - kubeconfig=k8s_kubeconfig + 'kubernetes.core.k8s', + namespace=dataplane_pagila_namespace, + kind='Service', + kubeconfig=k8s_kubeconfig )}}" - name: create pagila debezium signal schema diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index 6811493..bb0d228 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -119,8 +119,8 @@ login_host: "{{ warehouse_host }}" vars: warehouse_tables_expected: "{{ pagila_target_tables | map('map_format', 'pagila_public_%s') | list }}" - warehouse_tables_pagila: "{{ warehouse_tables_found | select('match', '^pagila_.*') }}" - warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^include_.*') }}" + warehouse_tables_pagila: "{{ warehouse_tables_found | select('match', '^pagila_public_.*') }}" + warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^include_public_.*') }}" warehouse_tables_found: "{{ warehouse_table_query.query_result | map(attribute='table_name') | list }}" register: warehouse_table_query retries: 20 @@ -146,8 +146,8 @@ ansible.builtin.debug: msg: "table diff=[{{ ', '.join(warehouse_tables_diff) }}]" vars: - warehouse_tables_expected: ['include_public_staff'] - warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^pagila_.*') }}" + warehouse_tables_expected: ["include_public_staff"] + warehouse_tables_include: "{{ warehouse_tables_found | select('match', '^pagila_public_.*') }}" warehouse_tables_found: "{{ warehouse_table_query.query_result | map(attribute='table_name') | list }}" warehouse_tables_diff_expected: "{{ warehouse_tables_expected | difference(warehouse_tables_include) }}" warehouse_tables_diff_found: "{{ warehouse_tables_include | difference(warehouse_tables_expected) }}" @@ -164,10 +164,10 @@ connector_present: "{{ connector_matches | int > 0 }}" connector_query: "{{ query( - 'kubernetes.core.k8s', - api_version='', - kind='KafkaConnector', - kubeconfig=k8s_kubeconfig + 'kubernetes.core.k8s', + api_version='', + kind='KafkaConnector', + kubeconfig=k8s_kubeconfig ) }}" until: connector_present failed_when: not connector_present diff --git a/pyproject.toml b/pyproject.toml index 3b530dd..dea7d22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dataplane" -version = "0.1.17" +version = "0.1.18" description = "" authors = ["Teodoro Cook "]