Skip to content

Commit

Permalink
feat: use sshkey-handler to allow for more sshkey types
Browse files Browse the repository at this point in the history
  • Loading branch information
shreddedbacon committed Apr 2, 2024
1 parent 5585b7e commit efa31cc
Show file tree
Hide file tree
Showing 32 changed files with 844 additions and 234 deletions.
16 changes: 9 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ services := api \
actions-handler \
backup-handler \
broker \
sshkey-handler \
keycloak \
keycloak-db \
logs2notifications \
Expand All @@ -185,6 +186,7 @@ build/api-redis: services/api-redis/Dockerfile
build/actions-handler: services/actions-handler/Dockerfile
build/backup-handler: services/backup-handler/Dockerfile
build/broker: services/broker/Dockerfile
build/sshkey-handler: services/sshkey-handler/Dockerfile
build/keycloak-db: services/keycloak-db/Dockerfile
build/keycloak: services/keycloak/Dockerfile
build/logs2notifications: services/logs2notifications/Dockerfile
Expand Down Expand Up @@ -253,7 +255,7 @@ wait-for-keycloak:
grep -m 1 "Config of Keycloak done." <(docker-compose -p $(CI_BUILD_TAG) --compatibility logs -f keycloak 2>&1)

# Define a list of which Lagoon Services are needed for running any deployment testing
main-test-services = actions-handler broker logs2notifications api api-db api-redis keycloak keycloak-db ssh auth-server local-git local-api-data-watcher-pusher local-minio
main-test-services = actions-handler broker sshkey-handler logs2notifications api api-db api-redis sshkey-handler keycloak keycloak-db ssh auth-server local-git local-api-data-watcher-pusher local-minio

# List of Lagoon Services needed for webhook endpoint testing
webhooks-test-services = webhook-handler webhooks2tasks backup-handler
Expand Down Expand Up @@ -351,15 +353,15 @@ local-dev-yarn-stop:

.PHONY: ui-development
ui-development: build-ui-logs-development
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db sshkey-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis

.PHONY: api-development
api-development: build-ui-logs-development
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db local-api-data-watcher-pusher keycloak keycloak-db broker api-redis
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db sshkey-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis

.PHONY: ui-logs-development
ui-logs-development: build-ui-logs-development
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db actions-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db actions-handler sshkey-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog

## CI targets

Expand All @@ -371,7 +373,7 @@ STERN_VERSION = v2.6.1
CHART_TESTING_VERSION = v3.10.1
K3D_IMAGE = docker.io/rancher/k3s:v1.28.6-k3s2
TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows]
CHARTS_TREEISH = main
CHARTS_TREEISH = sshkey-handler
TASK_IMAGES = task-activestandby

# Symlink the installed kubectl client if the correct version is already
Expand Down Expand Up @@ -481,7 +483,7 @@ ifeq ($(ARCH), darwin)
tcp-listen:32080,fork,reuseaddr tcp-connect:target:32080
endif

K3D_SERVICES = api api-db api-redis auth-server actions-handler broker keycloak keycloak-db logs2notifications webhook-handler webhooks2tasks local-api-data-watcher-pusher local-git ssh tests workflows $(TASK_IMAGES)
K3D_SERVICES = api api-db api-redis auth-server actions-handler broker sshkey-handler keycloak keycloak-db logs2notifications webhook-handler webhooks2tasks local-api-data-watcher-pusher local-git ssh tests workflows $(TASK_IMAGES)
K3D_TESTS = local-api-data-watcher-pusher local-git tests
K3D_TOOLS = k3d helm kubectl jq stern

Expand Down Expand Up @@ -517,7 +519,7 @@ k3d/test: k3d/cluster helm/repos $(addprefix local-dev/,$(K3D_TOOLS)) build
"quay.io/helmpack/chart-testing:$(CHART_TESTING_VERSION)" \
ct install --helm-extra-args "--timeout 60m"

LOCAL_DEV_SERVICES = api auth-server actions-handler logs2notifications webhook-handler webhooks2tasks
LOCAL_DEV_SERVICES = api auth-server actions-handler sshkey-handler logs2notifications webhook-handler webhooks2tasks

# install lagoon charts in a Kind cluster
.PHONY: k3d/setup
Expand Down
12 changes: 12 additions & 0 deletions docker-bake.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ group "default" {
"auth-server",
"backup-handler",
"broker",
"sshkey-handler",
"keycloak-db",
"keycloak",
"local-api-data-watcher-pusher",
Expand All @@ -72,6 +73,7 @@ group "ui-logs-development" {
"api-redis",
"api",
"broker",
"sshkey-handler",
"keycloak-db",
"keycloak",
"local-api-data-watcher-pusher",
Expand All @@ -95,6 +97,7 @@ group "prod-images" {
"auth-server",
"backup-handler",
"broker",
"sshkey-handler",
"keycloak-db",
"keycloak",
"logs2notifications",
Expand Down Expand Up @@ -182,6 +185,15 @@ target "broker" {
tags = ["${IMAGE_REPO}/broker:${TAG}"]
}

target "sshkey-handler" {
inherits = ["default"]
context = "services/sshkey-handler"
labels = {
"org.opencontainers.image.title": "lagoon-core/sshkey-handler - the sshkey-handler service for Lagoon"
}
tags = ["${IMAGE_REPO}/sshkey-handler:${TAG}"]
}

target "keycloak" {
inherits = ["default"]
context = "services/keycloak"
Expand Down
12 changes: 12 additions & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ services:
ports:
- '15672:15672'
- '5672:5672'
sshkey-handler:
# this is neded for the internal dns references
container_name: sshkeyhandler
image: ${IMAGE_REPO:-lagoon}/sshkey-handler
ports:
- '3333:3333'
logs2notifications:
image: ${IMAGE_REPO:-lagoon}/logs2notifications
environment:
Expand All @@ -43,6 +49,7 @@ services:
- ./node-packages:/app/node-packages:delegated
environment:
- CONSOLE_LOGGING_LEVEL=trace
- SSHKEY_HANDLER_HOST=sshkeyhandler
api-db-init:
image: ${IMAGE_REPO:-lagoon}/api
command: >
Expand Down Expand Up @@ -71,6 +78,8 @@ services:
condition: service_completed_successfully # don't start the lagoon migrations until the db migrations is completed
keycloak:
condition: service_started
sshkey-handler:
condition: service_started
api:
image: ${IMAGE_REPO:-lagoon}/api
command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
Expand All @@ -89,11 +98,14 @@ services:
- S3_BAAS_ACCESS_KEY_ID=minio
- S3_BAAS_SECRET_ACCESS_KEY=minio123
- CONSOLE_LOGGING_LEVEL=debug
- SSHKEY_HANDLER_HOST=sshkeyhandler
depends_on:
api-lagoon-migrations:
condition: service_started
keycloak:
condition: service_started
keycloak:
condition: service_started
ports:
- '3000:3000'
# Uncomment for local new relic tracking
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,45 +38,54 @@ mutation PopulateApi {
}

### SSH Keys
CiCustomerSshKeyRsa: addSshKey(
CiCustomerSshKeyRsa: addUserSSHPublicKey(
input: {
id: 4
name: "ci-customer-sshkey-rsa"
keyValue: "AAAAB3NzaC1yc2EAAAADAQABAAACAQDEZlms5XsiyWjmnnUyhpt93VgHypse9Bl8kNkmZJTiM3Ex/wZAfwogzqd2LrTEiIOWSH1HnQazR+Cc9oHCmMyNxRrLkS/MEl0yZ38Q+GDfn37h/llCIZNVoHlSgYkqD0MQrhfGL5AulDUKIle93dA6qdCUlnZZjDPiR0vEXR36xGuX7QYAhK30aD2SrrBruTtFGvj87IP/0OEOvUZe8dcU9G/pCoqrTzgKqJRpqs/s5xtkqLkTIyR/SzzplO21A+pCKNax6csDDq3snS8zfx6iM8MwVfh8nvBW9seax1zBvZjHAPSTsjzmZXm4z32/ujAn/RhIkZw3ZgRKrxzryttGnWJJ8OFyF31JTJgwWWuPdH53G15PC83ZbmEgSV3win51RZRVppN4uQUuaqZWG9wwk2a6P5aen1RLCSLpTkd2mAEk9PlgmJrf8vITkiU9pF9n68ENCoo556qSdxW2pxnjrzKVPSqmqO1Xg5K4LOX4/9N4n4qkLEOiqnzzJClhFif3O28RW86RPxERGdPT81UI0oDAcU5euQr8Emz+Hd+PY1115UIld3CIHib5PYL9Ee0bFUKiWpR/acSe1fHB64mCoHP7hjFepGsq7inkvg2651wUDKBshGltpNkMj6+aZedNc0/rKYyjl80nT8g8QECgOSRzpmYp0zli2HpFoLOiWw=="
keyType: SSH_RSA
publicKey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDEZlms5XsiyWjmnnUyhpt93VgHypse9Bl8kNkmZJTiM3Ex/wZAfwogzqd2LrTEiIOWSH1HnQazR+Cc9oHCmMyNxRrLkS/MEl0yZ38Q+GDfn37h/llCIZNVoHlSgYkqD0MQrhfGL5AulDUKIle93dA6qdCUlnZZjDPiR0vEXR36xGuX7QYAhK30aD2SrrBruTtFGvj87IP/0OEOvUZe8dcU9G/pCoqrTzgKqJRpqs/s5xtkqLkTIyR/SzzplO21A+pCKNax6csDDq3snS8zfx6iM8MwVfh8nvBW9seax1zBvZjHAPSTsjzmZXm4z32/ujAn/RhIkZw3ZgRKrxzryttGnWJJ8OFyF31JTJgwWWuPdH53G15PC83ZbmEgSV3win51RZRVppN4uQUuaqZWG9wwk2a6P5aen1RLCSLpTkd2mAEk9PlgmJrf8vITkiU9pF9n68ENCoo556qSdxW2pxnjrzKVPSqmqO1Xg5K4LOX4/9N4n4qkLEOiqnzzJClhFif3O28RW86RPxERGdPT81UI0oDAcU5euQr8Emz+Hd+PY1115UIld3CIHib5PYL9Ee0bFUKiWpR/acSe1fHB64mCoHP7hjFepGsq7inkvg2651wUDKBshGltpNkMj6+aZedNc0/rKYyjl80nT8g8QECgOSRzpmYp0zli2HpFoLOiWw=="
user: {
email: "[email protected]"
}
}
) {
id
}
CiCustomerSshKeyEd25519: addSshKey(
CiCustomerSshKeyEd25519: addUserSSHPublicKey(
input: {
id: 5
name: "ci-customer-sshkey-ed25519"
keyValue: "AAAAC3NzaC1lZDI1NTE5AAAAIMdEs1h19jv2UrbtKcqPDatUxT9lPYcbGlEAbInsY8Ka"
keyType: SSH_ED25519
publicKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMdEs1h19jv2UrbtKcqPDatUxT9lPYcbGlEAbInsY8Ka"
user: {
email: "[email protected]"
}
}
) {
id
}
CiCustomerSshKeyEcdsa: addSshKey(
CiCustomerSshKeyEcdsa: addUserSSHPublicKey(
input: {
id: 6
name: "ci-customer-sshkey-ecdsa"
keyValue: "AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAD8E5wfvLg8vvfO9mmHVsZQK8dNgdKM5FrTxL4ORDq66Z50O8zUzBwF1VTO5Zx+qwB7najMdWsnW00BC6PMysSNJQD5HI4CokyKqmGdeSXcROYwvYOjlDQ+jD5qOSmkllRZZnkEYXE5FVBXaZWToyfGUGIoECvKGUQZxkBDHsbK13JdfA=="
keyType: ECDSA_SHA2_NISTP521
publicKey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAD8E5wfvLg8vvfO9mmHVsZQK8dNgdKM5FrTxL4ORDq66Z50O8zUzBwF1VTO5Zx+qwB7najMdWsnW00BC6PMysSNJQD5HI4CokyKqmGdeSXcROYwvYOjlDQ+jD5qOSmkllRZZnkEYXE5FVBXaZWToyfGUGIoECvKGUQZxkBDHsbK13JdfA=="
user: {
email: "[email protected]"
}
}
) {
id
}
CiCustomerSshKeyEd25519SK: addUserSSHPublicKey(
input: {
id: 7
name: "ci-customer-sshkey-ed25519-sk"
publicKey: "[email protected] AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIPjqGSQd+w7qQxioI6qj+KWX/pEg9mNvVGZ7aUoXfsC0AAAABHNzaDo="
user: {
email: "[email protected]"
}
}
) {
id
}

### Add Users to Groups
CiCustomerUserAddRsa: addUserToGroup(
Expand Down
67 changes: 67 additions & 0 deletions node-packages/commons/src/util/func.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import { unless, is, isNil, isEmpty, partialRight, complement } from 'ramda';
import http from 'http';
import querystring from 'querystring';
import { getConfigFromEnv } from './config';

export const isNumber = is(Number);
export const isArray = is(Array);
Expand Down Expand Up @@ -28,3 +31,67 @@ export const jsonMerge = function(a, b, prop) {
// a2 = [1,2,3,5]
// arrayDiff(a1,a2) = [4]
export const arrayDiff = (a:Array<any>, b:Array<any>) => a.filter(e => !b.includes(e));

// helper that will use the crypto handler service to check if a public or private key is valid or not
export async function validateKey(key, type) {
const data = querystring.stringify({'key': key});
const options = {
hostname: getConfigFromEnv("SSHKEY_HANDLER_HOST", "localhost"),
port: 3333,
path: `/validate/${type}`,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(data)
},
};
let p = new Promise((resolve, reject) => {
const req = http.request(options, (res) => {
res.setEncoding('utf8');
let responseBody = '';

res.on('data', (chunk) => {
responseBody += chunk;
});

res.on('end', () => {
resolve(JSON.parse(responseBody));
});
});
req.on('error', (err) => {
reject(err);
});
req.write(data)
req.end();
});
return await p;
}

// helper that will use the crypto handler service to generate a private key with associated public key
export async function generatePrivateKey() {
const options = {
hostname: getConfigFromEnv("SSHKEY_HANDLER_HOST", "localhost"),
port: 3333,
path: '/generate/ed25519',
method: 'GET',
};
let p = new Promise((resolve, reject) => {
const req = http.request(options, (res) => {
res.setEncoding('utf8');
let responseBody = '';

res.on('data', (chunk) => {
responseBody += chunk;
});

res.on('end', () => {
resolve(JSON.parse(responseBody));
});
});
req.on('error', (err) => {
reject(err);
});
req.end();
});
return await p;
}
41 changes: 41 additions & 0 deletions services/api/database/migrations/20240304000000_sshkey.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/**
* @param { import("knex").Knex } knex
* @returns { Promise<void> }
*/
exports.up = async function(knex) {
return knex.schema.table('ssh_key', (table) => {
table.string('key_type_new').notNullable().defaultTo('ssh-rsa');
}).then(() => {
return knex('ssh_key').update({
key_type_new: knex.ref('key_type')
});
}).then(function () {
return knex.schema.table('ssh_key', (table) => {
table.dropColumn('key_type');
table.renameColumn('key_type_new', 'key_type');
})
});
};

/**
* @param { import("knex").Knex } knex
* @returns { Promise<void> }
*/
exports.down = async function(knex) {
return knex.schema.table('ssh_key', (table) => {
table.enu('key_type_new', ['ssh-rsa', 'ssh-ed25519','ecdsa-sha2-nistp256','ecdsa-sha2-nistp384','ecdsa-sha2-nistp521']).notNullable().defaultTo('ssh-rsa');
}).then(() => {
return knex('ssh_key')
.whereNotIn('key_type', ['ssh-rsa', 'ssh-ed25519','ecdsa-sha2-nistp256','ecdsa-sha2-nistp384','ecdsa-sha2-nistp521'])
.del();
}).then(function () {
return knex('ssh_key').update({
key_type_new: knex.ref('key_type')
});
}).then(function () {
return knex.schema.table('ssh_key', (table) => {
table.dropColumn('key_type');
table.renameColumn('key_type_new', 'key_type');
})
});
};
1 change: 0 additions & 1 deletion services/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@
"ramda": "0.25.0",
"redis": "^3.0.2",
"snakecase-keys": "^1.2.0",
"sshpk": "^1.14.2",
"validator": "^10.8.0",
"winston": "^3",
"winston-transport": "^4.4.0"
Expand Down
12 changes: 4 additions & 8 deletions services/api/src/gitlab-sync/projects.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import * as R from 'ramda';
import * as sshpk from 'sshpk';
import * as gitlabApi from '@lagoon/commons/dist/gitlab/api';
import {
sanitizeGroupName,
Expand All @@ -10,6 +9,7 @@ import {
addUserToGroup
} from '@lagoon/commons/dist/api';
import { logger } from '@lagoon/commons/dist/logs/local-logger';
import { validateKey } from '../util/func';

interface GitlabProject {
id: number,
Expand Down Expand Up @@ -56,14 +56,10 @@ const syncProject = async (project) => {
}

try {
const privateKey = R.pipe(
R.prop('privateKey'),
sshpk.parsePrivateKey,
)(lagoonProject);
//@ts-ignore
const publicKey = privateKey.toPublic();
const privkey = new Buffer((R.prop('privateKey', lagoonProject))).toString('base64')
const publickey = await validateKey(privkey, "private")

await gitlabApi.addDeployKeyToProject(id, publicKey.toString());
await gitlabApi.addDeployKeyToProject(id, publickey['publickey']);
} catch (err) {
if (!err.message.includes('has already been taken')) {
throw new Error(`Could not add deploy_key to gitlab project ${id}, reason: ${err}`);
Expand Down
Loading

0 comments on commit efa31cc

Please sign in to comment.