diff --git a/Makefile b/Makefile index 9de0563..f0b10f2 100644 --- a/Makefile +++ b/Makefile @@ -105,23 +105,15 @@ warehouse: @PGPASSWORD=$(WAREHOUSE_PASS) psql -h $(WAREHOUSE_HOST) -U $(WAREHOUSE_USER) -d $(WAREHOUSE_DB) ${WAREHOUSE_ARGS} images: dataplane-connect dataplane-util - docker image prune --force; \ + docker image prune --force && \ curl -s http://localhost:$(DOCKER_REGISTRY_PORT)/v2/_catalog | jq -dataplane-init: - cd init ; \ - docker build \ - --rm \ - --tag "$(DOCKER_REGISTRY)$(DOCKER_USER)/$@:$(DATAPLANE_RELEASE)" \ - . ; \ - docker image push "$(DOCKER_REGISTRY)$(DOCKER_USER)/$@:$(DATAPLANE_RELEASE)" - dataplane-connect: - cd connect ; \ + cd connect && \ docker build \ --rm \ --tag "$(DOCKER_REGISTRY)$(DOCKER_USER)/$@:$(DATAPLANE_RELEASE)" \ - . ; \ + . && \ docker image push "$(DOCKER_REGISTRY)$(DOCKER_USER)/$@:$(DATAPLANE_RELEASE)" dataplane-util: @@ -133,50 +125,61 @@ dataplane-util: docker image push "$(DOCKER_REGISTRY)$(DOCKER_USER)/$@:$(DATAPLANE_RELEASE)" wait: - @echo wait for service startup ; \ - make --no-print-directory kubectl rollout status deployment/$(DATAPLANE_CHART)-registry -- -n $(DATAPLANE_NS) ; \ - make --no-print-directory kubectl rollout status deployment/$(DATAPLANE_CHART)-connect -- -n $(DATAPLANE_NS) ; \ + @export KUBECONFIG=$(EPHEMERAL_DIR)/config && \ + echo wait for service startup && \ + kubectl rollout status deployment/$(DATAPLANE_CHART)-registry -n $(DATAPLANE_NS) && \ + kubectl get pod -l "app.kubernetes.io/name=kafka-connect" -n $(DATAPLANE_NS) -o name | \ + xargs kubectl wait --for=jsonpath='{.status.phase}'=Running -n $(DATAPLANE_NS) strimzi: strimzi-topics strimzi-topics: - make --no-print-directory kubectl exec -- \ + @export KUBECONFIG=$(EPHEMERAL_DIR)/config && \ + kubectl exec \ -it pod/$(DATAPLANE_CHART)-strimzi-kafka-0 \ -n $(DATAPLANE_NS) -- \ - "/opt/kafka/bin/kafka-topics.sh --bootstrap-server $(DATAPLANE_CHART)-strimzi-kafka-bootstrap:9092 --list" + /opt/kafka/bin/kafka-topics.sh --bootstrap-server $(DATAPLANE_CHART)-strimzi-kafka-bootstrap:9092 --list strimzi-connectors: - @make --no-print-directory kubectl exec -- -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ + @export KUBECONFIG=$(EPHEMERAL_DIR)/config && \ + kubectl exec \ + -it svc/$(DATAPLANE_CHART)-connect-api \ + -n $(DATAPLANE_NS) -- \ curl -s http://localhost:8083/connectors \ | jq '.[]' -r \ - | xargs -I{} make --no-print-directory kubectl exec svc/$(DATAPLANE_CHART)-connect-api -- -it -n $(DATAPLANE_NS) -- \ + | xargs -I{} kubectl exec svc/$(DATAPLANE_CHART)-connect-api -it -n $(DATAPLANE_NS) -- \ curl -s http://localhost:8083/connectors/\{\} 2>/dev/null strimzi-connector-status: - @make --no-print-directory kubectl exec -- -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ - curl -s http://localhost:8083/connectors \ - | jq '.[]' -r \ - | xargs -I{} make --no-print-directory kubectl exec svc/$(DATAPLANE_CHART)-connect-api -- -it -n $(DATAPLANE_NS) -- \ + @export KUBECONFIG=$(EPHEMERAL_DIR)/config && \ + kubectl exec \ + -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ + curl -s http://localhost:8083/connectors \ + | jq '.[]' -r \ + | xargs -I{} kubectl exec svc/$(DATAPLANE_CHART)-connect-api -it -n $(DATAPLANE_NS) -- \ curl -s http://localhost:8083/connectors/\{\}/status 2>/dev/null | jq '.tasks | map(.state) | .[]' -r strimzi-connector-trace: - @make --no-print-directory kubectl exec -- -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ + @export KUBECONFIG=$(EPHEMERAL_DIR)/config && \ + kubectl exec \ + -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ curl -s http://localhost:8083/connectors \ - | jq '.[]' -r \ - | xargs -I{} make --no-print-directory kubectl exec svc/$(DATAPLANE_CHART)-connect-api -- -it -n $(DATAPLANE_NS) -- \ + | jq '.[]' -r \ + | xargs -I{} kubectl exec svc/$(DATAPLANE_CHART)-connect-api -it -n $(DATAPLANE_NS) -- \ curl -s http://localhost:8083/connectors/\{\}/status 2>/dev/null | jq '.tasks | map(.trace) | .[]' -r strimzi-connector-restart: wait @echo restart kafka connectors ; \ - make --no-print-directory kubectl exec -- -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ + export KUBECONFIG=$(EPHEMERAL_DIR)/config && \ + kubectl exec -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ curl -s http://localhost:8083/connectors \ - | jq '.[]' -r \ - | xargs -I{} make --no-print-directory kubectl exec svc/$(DATAPLANE_CHART)-connect-api -- -it -n $(DATAPLANE_NS) -- \ + | jq '.[]' -r \ + | xargs -I{} kubectl exec svc/$(DATAPLANE_CHART)-connect-api -it -n $(DATAPLANE_NS) -- \ curl -s -XPOST http://localhost:8083/connectors/\{\}/restart 2>/dev/null ; \ - make --no-print-directory kubectl exec -- -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ + kubectl exec -it svc/$(DATAPLANE_CHART)-connect-api -n $(DATAPLANE_NS) -- \ curl -s http://localhost:8083/connectors \ - | jq '.[]' -r \ - | xargs -I{} make --no-print-directory kubectl exec svc/$(DATAPLANE_CHART)-connect-api -- -it -n $(DATAPLANE_NS) -- \ + | jq '.[]' -r \ + | xargs -I{} kubectl exec svc/$(DATAPLANE_CHART)-connect-api -it -n $(DATAPLANE_NS) -- \ curl -s -XPOST http://localhost:8083/connectors/\{\}/tasks/0/restart 2>/dev/null template: diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 380f60e..9c457db 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -2,8 +2,6 @@ - name: Converge hosts: localhost gather_facts: false - roles: - - nephelaiio.plugins tasks: - name: Wait for strimzi crd deployment kubernetes.core.k8s_info: diff --git a/molecule/default/destroy.yml b/molecule/default/destroy.yml index c4ad857..bd3a8e0 100644 --- a/molecule/default/destroy.yml +++ b/molecule/default/destroy.yml @@ -5,7 +5,6 @@ vars: kind_cluster_state: absent roles: - - nephelaiio.plugins - nephelaiio.kind tasks: - name: Delete certificate from edge truststore diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index ab74c27..2d31158 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -113,7 +113,7 @@ provisioner: image: repository: "{{ kind_registry_hostname }}:{{ kind_registry_port }}/nephelaiio/dataplane-connect" tag: latest - secret: "{{ dataplane_pagila_user }}-{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}" + secret: "postgres-{{ dataplane_pagila_team }}-{{ dataplane_pagila_db }}" config: acks: all config.storage.replication.factor: 3 diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index 034168d..a1ecf5e 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -6,7 +6,6 @@ k8s_deploy: true k8s_verify: false roles: - - nephelaiio.plugins - nephelaiio.k8s tasks: - name: Install PostgreSQL binary diff --git a/molecule/default/side_effect.yml b/molecule/default/side_effect.yml index f265fa9..8d3e59a 100644 --- a/molecule/default/side_effect.yml +++ b/molecule/default/side_effect.yml @@ -4,8 +4,6 @@ gather_facts: true vars: extra_table: languages - roles: - - nephelaiio.plugins tasks: - name: Query pagila connection data ansible.builtin.set_fact: diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index 111f466..8ad998b 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -10,9 +10,6 @@ k8s_zalando_verify: false k8s_volume_verify: false verify_pagila_user: postgres - roles: - - name: Nephelaiio.plugins - - name: Nephelaiio.k8s tasks: - name: Verify dataplane sink database when: lookup('ansible.builtin.env', 'CDC_VERIFY', default='true') | bool @@ -73,7 +70,7 @@ - name: Record pagila table data ansible.builtin.set_fact: - pagila_data: "{{ pagila_data_query.results | list_to_dict('item') }}" + pagila_data: "{{ pagila_data_query.results | nephelaiio.plugins.list_to_dict('item') }}" - name: Query warehouse connection data ansible.builtin.set_fact: @@ -187,7 +184,7 @@ - name: Record warehouse table data ansible.builtin.set_fact: - warehouse_data: "{{ warehouse_data_query.results | list_to_dict('item') }}" + warehouse_data: "{{ warehouse_data_query.results | nephelaiio.plugins.list_to_dict('item') }}" - name: Verify warehouse pagila table data ansible.builtin.fail: