feat: Replace NATS Streaming with CloudEvents broker (#1082)

Fixes #930. - [x] pdcout: Subscriber - [x] pdcout: Publisher - [x] crcin: Publisher - [x] crcin: Subscriber - [x] Remove all traces of NATS / Stan. - [x] Delete unused methods from `ParcelStore`. - [x] Update docs. - [x] Undo 59bfafd BREAKING CHANGE: Use a CloudEvents-compatible broker instead of NATS.
relaycorp · Oct 21, 2023 · 2132538 · 2132538
1 parent 0a82395
commit 2132538
Show file tree

Hide file tree

Showing 105 changed files with 2,412 additions and 3,256 deletions.
diff --git a/.github/scripts/port-forward.sh b/.github/scripts/port-forward.sh
@@ -10,7 +10,6 @@ kubectl port-forward --address 127.0.0.1 svc/public-gateway-poweb 8080:8082 &
 kubectl port-forward --address 127.0.0.1 svc/public-gateway-pohttp 8081:80 &
 kubectl port-forward --address 127.0.0.1 svc/public-gateway-cogrpc 8082:8081 &
 kubectl port-forward --address 127.0.0.1 svc/pong-pohttp 8083:80 &
-kubectl port-forward --address 127.0.0.1 svc/nats 4222:4222 &
 kubectl port-forward --address 127.0.0.1 svc/minio 9000:9000 &
 
 # Check at least one of the ports:

diff --git a/chart/dev-backing-services/Chart.lock b/chart/dev-backing-services/Chart.lock
@@ -8,17 +8,11 @@ dependencies:
 - name: mongodb
   repository: https://charts.bitnami.com/bitnami
   version: 12.1.20
-- name: nats
-  repository: https://nats-io.github.io/k8s/helm/charts
-  version: 0.7.2
-- name: stan
-  repository: https://nats-io.github.io/k8s/helm/charts
-  version: 0.7.2
-- name: relaynet-pong
-  repository: https://h.cfcr.io/relaycorp/public
-  version: 3.0.37
 - name: redis
   repository: https://charts.bitnami.com/bitnami
   version: 16.12.2
-digest: sha256:7c2f79a8d9d07692e87b8812cdb92d010a06f2d523966c70236a22d91789b0b0
-generated: "2022-08-19T17:01:14.652985945+01:00"
+- name: relaynet-pong
+  repository: https://h.cfcr.io/relaycorp/public
+  version: 3.0.37
+digest: sha256:3a6af898672e34c94198a98b7b46e849e606ee0fa084ae0675a8e81c02e8ef05
+generated: "2023-10-21T11:39:00.465509372+01:00"
diff --git a/chart/dev-backing-services/Chart.yaml b/chart/dev-backing-services/Chart.yaml
@@ -15,12 +15,6 @@ dependencies:
   - name: mongodb
     version: 12.1.20
     repository: https://charts.bitnami.com/bitnami
-  - name: nats
-    version: 0.7.2
-    repository: https://nats-io.github.io/k8s/helm/charts
-  - name: stan
-    version: 0.7.2
-    repository: https://nats-io.github.io/k8s/helm/charts
   - name: redis
     version: 16.12.2
     repository: https://charts.bitnami.com/bitnami

diff --git a/chart/dev-backing-services/values.yml b/chart/dev-backing-services/values.yml
@@ -30,19 +30,6 @@ mongodb:
   persistence:
     enabled: false
 
-nats:
-  nameOverride: nats
-  natsbox:
-    enabled: false
-
-stan:
-  nameOverride: stan
-  stan:
-    nats:
-      url: nats://nats:4222
-  store:
-    type: memory
-
 redis:
   fullnameOverride: redis
   auth:

diff --git a/chart/templates/certrotator-cronjob.yml b/chart/templates/certrotator-cronjob.yml
@@ -48,5 +48,3 @@ spec:
                     name: {{ include "relaynet-internet-gateway.fullname" . }}-keystore
                 - secretRef:
                     name: {{ include "relaynet-internet-gateway.fullname" . }}-keystore
-              resources:
-                {{- toYaml .Values.crcQueue.resources | nindent 16 }}
diff --git a/chart/templates/global-cm.yml b/chart/templates/global-cm.yml
@@ -17,9 +17,6 @@ data:
   LOG_ENV_NAME: {{ .Values.logging.envName }}
   {{- end }}
 
-  NATS_SERVER_URL: {{ .Values.nats.serverUrl }}
-  NATS_CLUSTER_ID: {{ .Values.nats.clusterId }}
-
   REDIS_URL: {{ .Values.redis.url }}
 
   OBJECT_STORE_BACKEND: {{ .Values.objectStore.backend }}
@@ -31,3 +28,5 @@ data:
   OBJECT_STORE_ACCESS_KEY_ID: {{ .Values.objectStore.accessKeyId }}
   {{- end }}
   OBJECT_STORE_TLS_ENABLED: '{{ .Values.objectStore.tlsEnabled }}'
+
+  CE_CHANNEL: {{ .Values.queueChannel }}
diff --git a/chart/templates/global-secret.yml b/chart/templates/global-secret.yml
@@ -6,7 +6,7 @@ metadata:
     {{- include "relaynet-internet-gateway.labels" . | nindent 4 }}
 type: Opaque
 data:
-  MONGO_PASSWORD: {{ .Values.mongo.password | b64enc }}
+  MONGODB_PASSWORD: {{ .Values.mongo.password | b64enc }}
 
   {{- if .Values.objectStore.secretKey }}
   OBJECT_STORE_SECRET_KEY: {{ .Values.objectStore.secretKey | b64enc }}

diff --git a/chart/templates/keygen-cm.yml b/chart/templates/keygen-cm.yml
@@ -19,8 +19,8 @@ data:
   LOG_ENV_NAME: {{ .Values.logging.envName }}
   {{- end }}
 
-  MONGO_URI: {{ .Values.mongo.uri }}
-  MONGO_DB: {{ .Values.mongo.db }}
-  MONGO_USER: {{ .Values.mongo.user }}
+  MONGODB_URI: {{ .Values.mongo.uri }}
+  MONGODB_DB: {{ .Values.mongo.db }}
+  MONGODB_USER: {{ .Values.mongo.user }}
 
   {{- include "relaynet-internet-gateway.keystoreNonSecretEnvVars" . | nindent 2 }}
diff --git a/chart/templates/keygen-secret.yml b/chart/templates/keygen-secret.yml
@@ -10,6 +10,6 @@ metadata:
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
 type: Opaque
 data:
-  MONGO_PASSWORD: {{ .Values.mongo.password | b64enc }}
+  MONGODB_PASSWORD: {{ .Values.mongo.password | b64enc }}
 
   {{- include "relaynet-internet-gateway.keystoreSecretEnvVars" . | nindent 2 }}
diff --git a/chart/templates/mongo-cm.yml b/chart/templates/mongo-cm.yml
@@ -5,6 +5,6 @@ metadata:
   labels:
     {{- include "relaynet-internet-gateway.labels" . | nindent 4 }}
 data:
-  MONGO_URI: {{ .Values.mongo.uri }}
-  MONGO_DB: {{ .Values.mongo.db }}
-  MONGO_USER: {{ .Values.mongo.user }}
+  MONGODB_URI: {{ .Values.mongo.uri }}
+  MONGODB_DB: {{ .Values.mongo.db }}
+  MONGODB_USER: {{ .Values.mongo.user }}
diff --git a/chart/templates/pdcout-deploy.yml b/chart/templates/pdcout-deploy.yml
diff --git a/chart/templates/pdcout-hpa.yaml b/chart/templates/pdcout-hpa.yaml
diff --git a/chart/templates/crcin-deploy.yml → chart/templates/queue-deploy.yml b/chart/templates/crcin-deploy.yml → chart/templates/queue-deploy.yml
@@ -1,16 +1,16 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "relaynet-internet-gateway.fullname" . }}-crcin
+  name: {{ include "relaynet-internet-gateway.fullname" . }}-queue
   labels:
-    {{- include "relaynet-internet-gateway.labels" (merge (dict "Component" "crcin") .) | nindent 4 }}
+    {{- include "relaynet-internet-gateway.labels" (merge (dict "Component" "queue") .) | nindent 4 }}
 spec:
 {{- if not .Values.autoscaling.enabled }}
-  replicas: {{ .Values.crcQueue.replicas }}
+  replicas: {{ .Values.queue.replicas }}
 {{- end }}
   selector:
     matchLabels:
-      {{- include "relaynet-internet-gateway.selectorLabels" (merge (dict "Component" "crcin") .) | nindent 6 }}
+      {{- include "relaynet-internet-gateway.selectorLabels" (merge (dict "Component" "queue") .) | nindent 6 }}
   template:
     metadata:
       annotations:
@@ -23,25 +23,27 @@ spec:
         keystore-cm-digest: {{ include "relaynet-internet-gateway.resourceDigest" (merge (dict "fileName" "keystore-cm.yml") .) }}
         keystore-secret-digest: {{ include "relaynet-internet-gateway.resourceDigest" (merge (dict "fileName" "keystore-secret.yml") .) }}
       labels:
-        {{- include "relaynet-internet-gateway.selectorLabels" (merge (dict "Component" "crcin") .) | nindent 8 }}
+        {{- include "relaynet-internet-gateway.selectorLabels" (merge (dict "Component" "queue") .) | nindent 8 }}
     spec:
       serviceAccountName: {{ include "relaynet-internet-gateway.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       shareProcessNamespace: true
       containers:
-        - name: crcin
+        - name: queue
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: {{ include "relaynet-internet-gateway.image" . }}
           imagePullPolicy: IfNotPresent
           args:
-            - build/main/bin/crc-queue-worker.js
+            - build/main/bin/queue-server.js
           env:
-            - name: WORKER_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
+            - name: POHTTP_USE_TLS
+              value: {{ ternary "false" "true" .Values.tags.gwDev | quote }}
+            {{- if .Values.proxyRequestIdHeader }}
+            - name: REQUEST_ID_HEADER
+              value: {{ .Values.proxyRequestIdHeader | quote }}
+            {{- end }}
           envFrom:
             - configMapRef:
                 name: {{ include "relaynet-internet-gateway.fullname" . }}
@@ -53,13 +55,25 @@ spec:
                 name: {{ include "relaynet-internet-gateway.fullname" . }}-keystore
             - secretRef:
                 name: {{ include "relaynet-internet-gateway.fullname" . }}-keystore
+          ports:
+            - name: queue
+              containerPort: 8080
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: queue
+          readinessProbe:
+            httpGet:
+              path: /
+              port: queue
           resources:
-            {{- toYaml .Values.crcQueue.resources | nindent 12 }}
+            {{- toYaml .Values.queue.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.crcQueue.affinity }}
+      {{- with .Values.queue.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}

diff --git a/chart/templates/crcin-hpa.yaml → chart/templates/queue-hpa.yaml b/chart/templates/crcin-hpa.yaml → chart/templates/queue-hpa.yaml
@@ -2,14 +2,14 @@
 apiVersion: autoscaling/v2beta1
 kind: HorizontalPodAutoscaler
 metadata:
-  name: {{ include "relaynet-internet-gateway.fullname" . }}-crcin
+  name: {{ include "relaynet-internet-gateway.fullname" . }}-queue
   labels:
-    {{- include "relaynet-internet-gateway.labels" (merge (dict "Component" "crcin") .) | nindent 4 }}
+    {{- include "relaynet-internet-gateway.labels" (merge (dict "Component" "queue") .) | nindent 4 }}
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: {{ include "relaynet-internet-gateway.fullname" . }}-crcin
+    name: {{ include "relaynet-internet-gateway.fullname" . }}-queue
   minReplicas: {{ .Values.autoscaling.minReplicas }}
   maxReplicas: {{ .Values.autoscaling.maxReplicas }}
   metrics:

diff --git a/chart/templates/queue-svc.yml b/chart/templates/queue-svc.yml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "relaynet-internet-gateway.fullname" . }}-queue
+  labels:
+    {{- include "relaynet-internet-gateway.labels" (merge (dict "Component" "queue") .) | nindent 4 }}
+  {{- with .Values.service.annotations }}
+  annotations:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: 80
+      targetPort: queue
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "relaynet-internet-gateway.selectorLabels" (merge (dict "Component" "queue") .) | nindent 4 }}
diff --git a/chart/values.dev.yml b/chart/values.dev.yml
@@ -16,19 +16,15 @@ ingress:
 
 internetAddress: gateway.tld
 
-pdcQueue:
-  replicas: 2
+# Skip the broker; post directly to the CloudEvents endpoint in development
+queueChannel: http://public-gateway-queue
 
 mongo:
   uri: mongodb://mongodb
   db: gw
   user: root
   password: letmein
 
-nats:
-  serverUrl: nats://nats:4222
-  clusterId: stan
-
 redis:
   url: redis://redis-headless:6379