Merge pull request #3662 from uselagoon/sshkey-handler

feat: use api-sidecar-handler to offload sshkey handling and add more types

Makefile

@@ -168,6 +168,7 @@ services :=	api \
 			actions-handler \
 			backup-handler \
 			broker \
+			api-sidecar-handler \
 			keycloak \
 			keycloak-db \
 			logs2notifications \
@@ -193,6 +194,7 @@ build/api-redis: services/api-redis/Dockerfile
 build/actions-handler: services/actions-handler/Dockerfile
 build/backup-handler: services/backup-handler/Dockerfile
 build/broker: services/broker/Dockerfile
+build/api-sidecar-handler: services/api-sidecar-handler/Dockerfile
 build/keycloak-db: services/keycloak-db/Dockerfile
 build/keycloak: services/keycloak/Dockerfile
 build/logs2notifications: services/logs2notifications/Dockerfile
@@ -266,7 +268,7 @@ wait-for-keycloak:
 	&& echo ""
 
 # Define a list of which Lagoon Services are needed for running any deployment testing
-main-test-services = actions-handler broker logs2notifications api api-db api-redis keycloak keycloak-db ssh auth-server local-git local-api-data-watcher-pusher local-minio
+main-test-services = actions-handler broker api-sidecar-handler logs2notifications api api-db api-redis api-sidecar-handler keycloak keycloak-db ssh auth-server local-git local-api-data-watcher-pusher local-minio
 
 # List of Lagoon Services needed for webhook endpoint testing
 webhooks-test-services = webhook-handler webhooks2tasks backup-handler
@@ -364,22 +366,22 @@ local-dev-yarn-stop:
 
 .PHONY: ui-development
 ui-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis
 	$(MAKE) wait-for-keycloak
 
 .PHONY: api-development
 api-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db local-api-data-watcher-pusher keycloak keycloak-db broker api-redis
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis
 	$(MAKE) wait-for-keycloak
 
 .PHONY: ui-logs-development
 ui-logs-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db actions-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
 	$(MAKE) wait-for-keycloak
 
 .PHONY: api-logs-development
 api-logs-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db actions-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
 	$(MAKE) wait-for-keycloak
 
 ## CI targets
@@ -392,7 +394,7 @@ STERN_VERSION = v2.6.1
 CHART_TESTING_VERSION = v3.11.0
 K3D_IMAGE = docker.io/rancher/k3s:v1.30.1-k3s1
 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
-CHARTS_TREEISH = lagoon-220
+CHARTS_TREEISH = sshkey-handler
 TASK_IMAGES = task-activestandby
 
 # the name of the docker network to create
@@ -506,7 +508,7 @@ ifeq ($(ARCH), darwin)
       tcp-listen:32080,fork,reuseaddr tcp-connect:target:32080
 endif
 
-K3D_SERVICES = api api-db api-redis auth-server actions-handler broker keycloak keycloak-db logs2notifications webhook-handler webhooks2tasks local-api-data-watcher-pusher local-git ssh tests workflows $(TASK_IMAGES)
+K3D_SERVICES = api api-db api-redis auth-server actions-handler broker api-sidecar-handler keycloak keycloak-db logs2notifications webhook-handler webhooks2tasks local-api-data-watcher-pusher local-git ssh tests workflows $(TASK_IMAGES)
 K3D_TESTS = local-api-data-watcher-pusher local-git tests
 K3D_TOOLS = k3d helm kubectl jq stern
 
@@ -535,7 +537,7 @@ k3d/test: k3d/setup
 			"quay.io/helmpack/chart-testing:$(CHART_TESTING_VERSION)" \
 			ct install --helm-extra-args "--timeout 60m"
 
-LOCAL_DEV_SERVICES = api auth-server actions-handler logs2notifications webhook-handler webhooks2tasks
+LOCAL_DEV_SERVICES = api auth-server actions-handler api-sidecar-handler logs2notifications webhook-handler webhooks2tasks
 
 # install lagoon charts in a Kind cluster
 .PHONY: k3d/setup

docker-bake.hcl

@@ -51,6 +51,7 @@ group "default" {
     "auth-server",
     "backup-handler",
     "broker",
+    "api-sidecar-handler",
     "keycloak-db",
     "keycloak",
     "local-api-data-watcher-pusher",
@@ -72,6 +73,7 @@ group "ui-logs-development" {
     "api-redis",
     "api",
     "broker",
+    "api-sidecar-handler",
     "keycloak-db",
     "keycloak",
     "local-api-data-watcher-pusher",
@@ -95,6 +97,7 @@ group "prod-images" {
     "auth-server",
     "backup-handler",
     "broker",
+    "api-sidecar-handler",
     "keycloak-db",
     "keycloak",
     "logs2notifications",
@@ -182,6 +185,15 @@ target "broker" {
   tags = ["${IMAGE_REPO}/broker:${TAG}"]
 }
 
+target "api-sidecar-handler" {
+  inherits = ["default"]
+  context = "services/api-sidecar-handler"
+  labels = {
+    "org.opencontainers.image.title": "lagoon-core/api-sidecar-handler - the api-sidecar-handler service for Lagoon"
+  }
+  tags = ["${IMAGE_REPO}/api-sidecar-handler:${TAG}"]
+}
+
 target "keycloak" {
   inherits = ["default"]
   context = "services/keycloak"

docker-compose.yaml

@@ -41,6 +41,12 @@ services:
     depends_on:
       broker:
         condition: service_started
+  api-sidecar-handler:
+    # this is neded for the internal dns references
+    container_name: apisidecarhandler
+    image: ${IMAGE_REPO:-lagoon}/api-sidecar-handler
+    ports:
+      - '3333:3333'
   logs2notifications:
     image: ${IMAGE_REPO:-lagoon}/logs2notifications
     environment:
@@ -58,6 +64,7 @@ services:
       - ./node-packages:/app/node-packages:delegated
     environment:
       - CONSOLE_LOGGING_LEVEL=trace
+      - SIDECAR_HANDLER_HOST=apisidecarhandler
   api-db-init:
     image: ${IMAGE_REPO:-lagoon}/api
     command: >
@@ -86,6 +93,8 @@ services:
         condition: service_completed_successfully # don't start the lagoon migrations until the db migrations is completed
       keycloak:
         condition: service_started
+      api-sidecar-handler:
+        condition: service_started
   api:
     image: ${IMAGE_REPO:-lagoon}/api
     command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
@@ -104,11 +113,14 @@ services:
       - S3_BAAS_ACCESS_KEY_ID=minio
       - S3_BAAS_SECRET_ACCESS_KEY=minio123
       - CONSOLE_LOGGING_LEVEL=debug
+      - SIDECAR_HANDLER_HOST=apisidecarhandler
     depends_on:
       api-lagoon-migrations:
         condition: service_started
       keycloak:
         condition: service_started
+      api-sidecar-handler:
+        condition: service_started
     ports:
       - '3000:3000'
     # Uncomment for local new relic tracking

local-dev/api-data-watcher-pusher/api-data/01-populate-api-data-lagoon-demo.gql

@@ -757,25 +757,36 @@ mutation PopulateApi {
   ) {
     name
   }
-  PlatformOwnerSshKeyEd25519: addSshKey(
+  PlatformOwnerSshKeyEd25519: addUserSSHPublicKey(
     input: {
       id: 5
       name: "sshkey-platform-owner-ed25519"
-      keyValue: "AAAAC3NzaC1lZDI1NTE5AAAAIMdEs1h19jv2UrbtKcqPDatUxT9lPYcbGlEAbInsY8Ka"
-      keyType: SSH_ED25519
+      publicKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMdEs1h19jv2UrbtKcqPDatUxT9lPYcbGlEAbInsY8Ka"
       user: {
         email: "[email protected]"
       }
     }
   ) {
     id
   }
-  MaintainerSshKeyEcdsa: addSshKey(
+  MaintainerSshKeyEcdsa: addUserSSHPublicKey(
     input: {
       id: 6
       name: "sshkey-maintainer-ecdsa"
-      keyValue: "AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAD8E5wfvLg8vvfO9mmHVsZQK8dNgdKM5FrTxL4ORDq66Z50O8zUzBwF1VTO5Zx+qwB7najMdWsnW00BC6PMysSNJQD5HI4CokyKqmGdeSXcROYwvYOjlDQ+jD5qOSmkllRZZnkEYXE5FVBXaZWToyfGUGIoECvKGUQZxkBDHsbK13JdfA=="
-      keyType: ECDSA_SHA2_NISTP521
+      publicKey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAD8E5wfvLg8vvfO9mmHVsZQK8dNgdKM5FrTxL4ORDq66Z50O8zUzBwF1VTO5Zx+qwB7najMdWsnW00BC6PMysSNJQD5HI4CokyKqmGdeSXcROYwvYOjlDQ+jD5qOSmkllRZZnkEYXE5FVBXaZWToyfGUGIoECvKGUQZxkBDHsbK13JdfA=="
+      user: {
+        email: "[email protected]"
+      }
+    }
+  ) {
+    id
+  }
+
+  MaintainerSshKeyEd25519SK: addUserSSHPublicKey(
+    input: {
+      id: 7
+      name: "sshkey-maintainer-ed25519-sk"
+      publicKey: "[email protected] AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIPjqGSQd+w7qQxioI6qj+KWX/pEg9mNvVGZ7aUoXfsC0AAAABHNzaDo="
       user: {
         email: "[email protected]"
       }

local-dev/api-data-watcher-pusher/api-data/03-populate-api-data-ci-local-control-k8s.gql

@@ -38,12 +38,11 @@ mutation PopulateApi {
   }
 
   ### SSH Keys
-  CiCustomerSshKeyRsa: addSshKey(
+  CiCustomerSshKeyRsa: addUserSSHPublicKey(
     input: {
       id: 4
       name: "sshkey-ci-customer-user-rsa"
-      keyValue: "AAAAB3NzaC1yc2EAAAADAQABAAACAQDEZlms5XsiyWjmnnUyhpt93VgHypse9Bl8kNkmZJTiM3Ex/wZAfwogzqd2LrTEiIOWSH1HnQazR+Cc9oHCmMyNxRrLkS/MEl0yZ38Q+GDfn37h/llCIZNVoHlSgYkqD0MQrhfGL5AulDUKIle93dA6qdCUlnZZjDPiR0vEXR36xGuX7QYAhK30aD2SrrBruTtFGvj87IP/0OEOvUZe8dcU9G/pCoqrTzgKqJRpqs/s5xtkqLkTIyR/SzzplO21A+pCKNax6csDDq3snS8zfx6iM8MwVfh8nvBW9seax1zBvZjHAPSTsjzmZXm4z32/ujAn/RhIkZw3ZgRKrxzryttGnWJJ8OFyF31JTJgwWWuPdH53G15PC83ZbmEgSV3win51RZRVppN4uQUuaqZWG9wwk2a6P5aen1RLCSLpTkd2mAEk9PlgmJrf8vITkiU9pF9n68ENCoo556qSdxW2pxnjrzKVPSqmqO1Xg5K4LOX4/9N4n4qkLEOiqnzzJClhFif3O28RW86RPxERGdPT81UI0oDAcU5euQr8Emz+Hd+PY1115UIld3CIHib5PYL9Ee0bFUKiWpR/acSe1fHB64mCoHP7hjFepGsq7inkvg2651wUDKBshGltpNkMj6+aZedNc0/rKYyjl80nT8g8QECgOSRzpmYp0zli2HpFoLOiWw=="
-      keyType: SSH_RSA
+      publicKey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEZlms5XsiyWjmnnUyhpt93VgHypse9Bl8kNkmZJTiM3Ex/wZAfwogzqd2LrTEiIOWSH1HnQazR+Cc9oHCmMyNxRrLkS/MEl0yZ38Q+GDfn37h/llCIZNVoHlSgYkqD0MQrhfGL5AulDUKIle93dA6qdCUlnZZjDPiR0vEXR36xGuX7QYAhK30aD2SrrBruTtFGvj87IP/0OEOvUZe8dcU9G/pCoqrTzgKqJRpqs/s5xtkqLkTIyR/SzzplO21A+pCKNax6csDDq3snS8zfx6iM8MwVfh8nvBW9seax1zBvZjHAPSTsjzmZXm4z32/ujAn/RhIkZw3ZgRKrxzryttGnWJJ8OFyF31JTJgwWWuPdH53G15PC83ZbmEgSV3win51RZRVppN4uQUuaqZWG9wwk2a6P5aen1RLCSLpTkd2mAEk9PlgmJrf8vITkiU9pF9n68ENCoo556qSdxW2pxnjrzKVPSqmqO1Xg5K4LOX4/9N4n4qkLEOiqnzzJClhFif3O28RW86RPxERGdPT81UI0oDAcU5euQr8Emz+Hd+PY1115UIld3CIHib5PYL9Ee0bFUKiWpR/acSe1fHB64mCoHP7hjFepGsq7inkvg2651wUDKBshGltpNkMj6+aZedNc0/rKYyjl80nT8g8QECgOSRzpmYp0zli2HpFoLOiWw=="
       user: {
         email: "[email protected]"
       }

node-packages/commons/src/util/func.ts

@@ -1,4 +1,7 @@
 import { unless, is, isNil, isEmpty, partialRight, complement } from 'ramda';
+import http from 'http';
+import querystring from 'querystring';
+import { getConfigFromEnv } from './config';
 
 export const isNumber = is(Number);
 export const isArray = is(Array);
@@ -28,3 +31,67 @@ export const jsonMerge = function(a, b, prop) {
 // a2 = [1,2,3,5]
 // arrayDiff(a1,a2) = [4]
 export const arrayDiff = (a:Array<any>, b:Array<any>) =>  a.filter(e => !b.includes(e));
+
+// helper that will use the crypto handler service to check if a public or private key is valid or not
+export async function validateKey(key, type) {
+  const data = querystring.stringify({'key': key});
+  const options = {
+      hostname: getConfigFromEnv("SIDECAR_HANDLER_HOST", "localhost"),
+      port: 3333,
+      path: `/validate/${type}`,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Content-Length': Buffer.byteLength(data)
+      },
+  };
+  let p = new Promise((resolve, reject) => {
+      const req = http.request(options, (res) => {
+          res.setEncoding('utf8');
+          let responseBody = '';
+
+          res.on('data', (chunk) => {
+              responseBody += chunk;
+          });
+
+          res.on('end', () => {
+              resolve(JSON.parse(responseBody));
+          });
+      });
+      req.on('error', (err) => {
+          reject(err);
+      });
+      req.write(data)
+      req.end();
+  });
+  return await p;
+}
+
+// helper that will use the crypto handler service to generate a private key with associated public key
+export async function generatePrivateKey() {
+  const options = {
+      hostname: getConfigFromEnv("SIDECAR_HANDLER_HOST", "localhost"),
+      port: 3333,
+      path: '/generate/ed25519',
+      method: 'GET',
+  };
+  let p = new Promise((resolve, reject) => {
+      const req = http.request(options, (res) => {
+          res.setEncoding('utf8');
+          let responseBody = '';
+
+          res.on('data', (chunk) => {
+              responseBody += chunk;
+          });
+
+          res.on('end', () => {
+              resolve(JSON.parse(responseBody));
+          });
+      });
+      req.on('error', (err) => {
+          reject(err);
+      });
+      req.end();
+  });
+  return await p;
+}

services/api-sidecar-handler/Dockerfile

@@ -0,0 +1,26 @@
+# build the binary
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+FROM golang:1.21-alpine AS builder
+# bring in all the packages
+COPY . /go/src/github.com/uselagoon/lagoon/services/api-sidecar-handler/
+WORKDIR /go/src/github.com/uselagoon/lagoon/services/api-sidecar-handler/
+
+# compile
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -o api-sidecar-handler .
+
+# put the binary into container
+# use the commons image to get entrypoints
+FROM ${UPSTREAM_REPO:-uselagoon}/commons:${UPSTREAM_TAG:-latest}
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+WORKDIR /app/
+COPY --from=builder /go/src/github.com/uselagoon/lagoon/services/api-sidecar-handler/api-sidecar-handler .
+
+ENV LAGOON=api-sidecar-handler
+
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["/app/api-sidecar-handler"]

services/api-sidecar-handler/README.md

@@ -0,0 +1,5 @@
+# api-sidecar-handler
+
+This is just a simple microservice that is run as a sidecar to `api` and `webhooks2tasks` to perform validations and generations on ssh keys used in Lagoon.
+
+The purpose of this sidecar is to initially replace the functionality of the node `sshpk` package, as it doesn't support all types of ssh-keys that could be used.

services/api-sidecar-handler/go.mod

@@ -0,0 +1,10 @@
+module github.com/uselagoon/lagoon/services/api-sidecar-handler
+
+go 1.21
+
+require (
+	github.com/gorilla/mux v1.8.1
+	golang.org/x/crypto v0.20.0
+)
+
+require golang.org/x/sys v0.17.0 // indirect

services/api-sidecar-handler/go.sum

@@ -0,0 +1,8 @@
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
+golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=