refactor: prepare service for self host option (#230)

.github/workflows/publish-docker.yml

@@ -42,9 +42,13 @@ jobs:
         id: app_name
         run: |
           if [[ "${{ github.ref_type }}" == "tag" ]]; then
-            echo "value=$(echo $GITHUB_REF_NAME | cut -d '@' -f 1)" >> $GITHUB_OUTPUT
+            VALUE=$(echo $GITHUB_REF_NAME | cut -d '@' -f 1)
+            echo "value=$VALUE" >> $GITHUB_OUTPUT
+            echo "publish_value=$VALUE" >> $GITHUB_OUTPUT
           else
-            echo "value=${{ github.event.inputs.app_name }}-next" >> $GITHUB_OUTPUT
+            VALUE=${{ github.event.inputs.app_name }}
+            echo "value=$VALUE" >> $GITHUB_OUTPUT
+            echo "publish_value=$VALUE-next" >> $GITHUB_OUTPUT
           fi
 
       # validate the version is the same as the tag
@@ -83,7 +87,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
-            ghcr.io/lironer/bundlemon-${{ steps.app_name.outputs.value }}
+            ghcr.io/lironer/bundlemon-${{ steps.app_name.outputs.publish_value }}
           flavor: latest=auto
           tags: |
             # create SHA tag for non-tag pushes 
@@ -102,8 +106,6 @@ jobs:
           labels: ${{ steps.image_meta.outputs.labels }}
           cache-from: |
             type=registry,ref=ghcr.io/lironer/bundlemon-${{ steps.app_name.outputs.value }}:latest
-            # in case this is a manual build, use latest image from the original image varient (not -next)
-            type=registry,ref=ghcr.io/lironer/bundlemon-${{ github.event.inputs.app_name }}:latest
           cache-to: type=inline
           platforms: linux/amd64,linux/arm64
           provenance: true

apps/platform/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:20-alpine
 
-ENV HOST=0.0.0.0 PORT=3333 SHOULD_SERVE_WEBSITE=true NODE_ENV=production
+ENV NODE_ENV=production
 
 WORKDIR /app
 
@@ -18,4 +18,7 @@ COPY dist service
 
 RUN chown -R service:service .
 
-CMD [ "node", "-r", "source-map-support/register", "service/app.js" ]
+CMD [ "node", "-r", "source-map-support/register", "service/app.js" ]
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=3 \
+  CMD sh -c "curl -f http://localhost:${PORT-8080}/is-alive || exit 1"

apps/platform/package.json

@@ -9,9 +9,10 @@
     "lint": "yarn --cwd ../../ nx lint platform --verbose",
     "start:mock-services": "docker compose -f ../service/docker-compose.test.yml up --remove-orphans",
     "stop:mock-services": "docker compose -f ../service/docker-compose.test.yml down",
-    "start:platform": "docker run --rm -d --name bundlemon-platform --env-file ../service/.development.env -e NODE_ENV=producation -e MONGO_DB_NAME=test -p 3333:3333 bundlemon-platform",
+    "start:base-platform": "docker run --rm -d --env-file ../service/.development.env -e MONGO_DB_NAME=test -e MONGO_URL=mongodb://host.docker.internal:51651 -e SHOULD_RUN_DB_INIT=false",
+    "start:platform": "yarn start:base-platform --name bundlemon-platform -e SHOULD_SERVE_WEBSITE=true -e ROOT_DOMAIN=localhost:3333 -p 3333:8080 bundlemon-platform",
     "stop:platform": "docker stop bundlemon-platform",
-    "start:platform-no-website": "docker run --rm -d --name bundlemon-platform-no-website --env-file ../service/.development.env -e NODE_ENV=producation -e MONGO_DB_NAME=test -e SHOULD_SERVE_WEBSITE=false -p 4444:3333 bundlemon-platform",
+    "start:platform-no-website": "yarn start:base-platform --name bundlemon-platform-no-website -e SHOULD_SERVE_WEBSITE=false -e ROOT_DOMAIN=localhost:4444 -p 4444:8080 bundlemon-platform",
     "stop:platform-no-website": "docker stop bundlemon-platform-no-website"
   },
   "dependencies": {},

apps/service/.development.env

@@ -1,8 +1,9 @@
 NODE_ENV=development
-APP_DOMAIN=localhost:4000
+HTTP_SCHEMA=http
+ROOT_DOMAIN=localhost:4000
 MONGO_URL=mongodb://localhost:51651
 MONGO_DB_NAME=dev
 MONGO_DB_USER=dev-user
 MONGO_DB_PASSWORD=password
-ROOT_DOMAIN=localhost
 SECRET_SESSION_KEY=74925e5027a05d9e31082271747a92b11a3b6988fc303bbb2aae330bef92b3a7
+SHOULD_SERVE_WEBSITE=false

apps/service/README.md

@@ -1 +1,55 @@
 # BundleMon service
+
+### Environment variables
+
+| Name                    | Description                                                                                                                        | Default               |
+| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| MONGO_URL               | MongoDB connection URL                                                                                                             | `-`                   |
+| MONGO_DB_NAME           | MongoDB database name                                                                                                              | `-`                   |
+| MONGO_DB_USER           | MongoDB username                                                                                                                   | `-`                   |
+| MONGO_DB_PASSWORD       | MongoDB password                                                                                                                   | `-`                   |
+| HTTP_SCHEMA             | HTTP schema (`http` or `https`)                                                                                                    | `https`               |
+| PORT                    | Port number for the service                                                                                                        | `8080`                |
+| ROOT_DOMAIN             | Root domain for the service                                                                                                        | `bundlemon.dev`       |
+| APP_DOMAIN              | Application domain, defaults to ROOT_DOMAIN                                                                                        | same as `ROOT_DOMAIN` |
+| SHOULD_SERVE_WEBSITE    | Flag to determine if the website should be served                                                                                  | `true`                |
+| SECRET_SESSION_KEY      | This key will be used for securely signing session cookies.<br />Auto generated each time the service starts, Prefer to set a key. | Auto generated        |
+| MAX_SESSION_AGE_SECONDS | Maximum session age in seconds                                                                                                     | `21600` (6 hours)     |
+| MAX_BODY_SIZE_BYTES     | Max body size in bytes                                                                                                             | `1048576` (1 MB)      |
+
+<details>
+  <summary>Generate secret session key</summary>
+
+```sh
+yarn install
+
+# prints the secret key
+node apps/service/scripts/generateSecretKey.js
+```
+
+</details>
+
+### GitHub integration (optional)
+
+If you want your self hosted BundleMon service to interact with GitHub, you will need to create GitHub App.
+
+#### Create GitHub App
+
+1. [Go to register new GitHub App](https://github.com/settings/apps/new)
+1. Choose name
+1. Setup Repository permissions
+   - Metadata - Read
+   - Pull requests - Read & write
+   - Checks - Read & write
+   - Commit statuses - Read & write
+1. Create App
+1. Generate private key, Replace private key new lines with `\n`
+
+#### GitHub App environment variables
+
+| Name                     | Description              |
+| ------------------------ | ------------------------ |
+| GITHUB_APP_ID            | GitHub App ID            |
+| GITHUB_APP_PRIVATE_KEY   | GitHub App private key   |
+| GITHUB_APP_CLIENT_ID     | GitHub App client ID     |
+| GITHUB_APP_CLIENT_SECRET | GitHub App client secret |

apps/service/jest.config.ts

@@ -4,6 +4,5 @@ export default async (): Promise<Config.InitialOptions> => ({
   displayName: 'service',
   preset: '../../jest.preset.js',
   coverageDirectory: '../../coverage/apps/service',
-  setupFilesAfterEnv: ['<rootDir>/tests/hooks.ts'],
   setupFiles: ['<rootDir>/tests/setup.ts'],
 });

apps/service/package.json

@@ -29,6 +29,7 @@
     "mongodb": "^6.3.0"
   },
   "devDependencies": {
+    "@types/sodium-native": "^2.3.9",
     "dotenv": "^16.3.1",
     "ts-json-schema-generator": "^1.3.0",
     "typescript": "^5.1.6",

apps/service/project.json

@@ -89,7 +89,7 @@
         "buildTargetOptions": {
           "progress": true
         },
-        "args": ["--listen"],
+        "args": ["--listen", "--local-certs"],
         "debounce": 1000
       },
       "configurations": {
@@ -112,6 +112,7 @@
     "test": {
       "executor": "@nx/jest:jest",
       "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "cache": false,
       "options": {
         "jestConfig": "{projectRoot}/jest.config.ts",
         "runInBand": true

apps/service/src/app.ts

@@ -8,14 +8,13 @@ import secureSession, { SecureSessionPluginOptions } from '@fastify/secure-sessi
 import routes from '@/routes';
 import * as schemas from '@/consts/schemas';
 import { closeMongoClient } from '@/framework/mongo/client';
-import { nodeEnv, secretSessionKey, rootDomain, isTestEnv, shouldServeWebsite } from '@/framework/env';
-import { DEFAULT_SESSION_AGE_SECONDS } from '@/consts/auth';
+import { secretSessionKey, rootDomain, isTestEnv, shouldServeWebsite } from '@/framework/env';
 import { RequestError as OctokitRequestError } from '@octokit/request-error';
-import { MAX_BODY_SIZE_BYTES } from './consts/server';
+import { host, port, maxSessionAgeSeconds, maxBodySizeBytes, shouldRunDbInit } from '@/framework/env';
 
 import type { ServerOptions } from 'https';
-
-const STATIC_DIR = nodeEnv === 'development' ? path.join(__dirname, '..', 'public') : path.join(__dirname, 'public');
+import { overrideWebsiteConfig } from './utils/website';
+import { initDb } from './framework/mongo/init';
 
 const _gracefulShutdown = async (app: FastifyInstance, signal: string) => {
   app.log.info(`${signal} signal received: closing server`);
@@ -30,10 +29,14 @@ const _gracefulShutdown = async (app: FastifyInstance, signal: string) => {
   }
 };
 
-function init() {
+interface InitParams {
+  isServerless: boolean;
+}
+
+async function init({ isServerless }: InitParams) {
   let https: ServerOptions | null = null;
 
-  if (nodeEnv === 'development') {
+  if (process.argv.includes('--local-certs')) {
     https = {
       key: fs.readFileSync(path.join(__dirname, 'local-certs', 'key.pem')),
       cert: fs.readFileSync(path.join(__dirname, 'local-certs', 'cert.pem')),
@@ -42,17 +45,18 @@ function init() {
 
   const app = fastify({
     https,
-    bodyLimit: MAX_BODY_SIZE_BYTES,
+    bodyLimit: maxBodySizeBytes,
     logger: {
+      level: 'info',
       serializers: {
         req(req) {
           return {
             method: req.method,
             url: req.url,
             hostname: req.hostname,
             remoteAddress: req.ip,
-            clientName: req?.headers?.['x-api-client-name'] || 'unknown',
-            clientVersion: req?.headers?.['x-api-client-version'] || 'unknown',
+            clientName: req?.headers?.['x-api-client-name'],
+            clientVersion: req?.headers?.['x-api-client-version'],
           };
         },
       },
@@ -66,11 +70,14 @@ function init() {
     });
 
   if (shouldServeWebsite) {
+    overrideWebsiteConfig();
+
     app.register(fastifyStatic, {
-      root: STATIC_DIR,
+      root: '/app/service/public',
       prefix: '/',
       wildcard: false,
       index: 'index.html',
+      logLevel: 'silent',
     });
 
     app.setNotFoundHandler((req, res) => {
@@ -93,7 +100,7 @@ function init() {
     httpOnly: true,
     secure: true,
     sameSite: isTestEnv ? 'none' : 'strict',
-    maxAge: DEFAULT_SESSION_AGE_SECONDS,
+    maxAge: maxSessionAgeSeconds,
   };
 
   app.register(cookie, {
@@ -135,7 +142,7 @@ function init() {
       return res.status(413).send({
         message: 'Request body too large',
         bodySize,
-        limitBytes: MAX_BODY_SIZE_BYTES,
+        limitBytes: maxBodySizeBytes,
       });
     }
 
@@ -149,21 +156,25 @@ function init() {
   process.on('SIGTERM', () => _gracefulShutdown(app, 'SIGTERM'));
   process.on('SIGINT', () => _gracefulShutdown(app, 'SIGINT'));
 
+  if (!isServerless && shouldRunDbInit) {
+    await initDb(app.log);
+  }
+
   return app;
 }
 
 // If called directly i.e. "node app"
 if (require.main === module || process.argv.includes('--listen')) {
-  const app = init();
-  const host = process.env.HOST ?? '0.0.0.0';
-  const port = process.env.PORT ? Number(process.env.PORT) : 3333;
-
-  app.listen({ host, port }, (err) => {
-    if (err) {
-      app.log.fatal(err);
-      process.exit(1);
-    }
-  });
+  (async () => {
+    const app = await init({ isServerless: false });
+
+    app.listen({ host, port }, (err) => {
+      if (err) {
+        app.log.fatal(err);
+        process.exit(1);
+      }
+    });
+  })();
 }
 
 // Required as a module => executed on vercel / other serverless platform

apps/service/src/controllers/authController.ts

@@ -1,16 +1,15 @@
 import type { RouteHandlerMethod } from 'fastify';
 import { RequestError as OctokitRequestError } from '@octokit/request-error';
 import { loginWithCode } from '@/framework/github';
-import { DEFAULT_SESSION_AGE_SECONDS } from '@/consts/auth';
-
+import { maxSessionAgeSeconds } from '@/framework/env';
 import type { FastifyValidatedRoute, LoginRequestSchema } from '@/types/schemas';
 
 export const loginController: FastifyValidatedRoute<LoginRequestSchema> = async (req, res) => {
   try {
     const { code } = req.body;
 
     const { sessionData, expiresAt } = await loginWithCode(code);
-    const expires = expiresAt ?? new Date(new Date().getTime() + 1000 * DEFAULT_SESSION_AGE_SECONDS);
+    const expires = expiresAt ?? new Date(new Date().getTime() + 1000 * maxSessionAgeSeconds);
 
     req.session.options({ expires });
     req.session.set('user', sessionData);

apps/service/src/framework/env.ts

@@ -1,20 +1,34 @@
 import * as env from 'env-var';
-
+import { Buffer } from 'buffer';
+import * as sodium from 'sodium-native';
 const getRequiredString = (key: string) => env.get(key).required().asString();
 const getOptionalString = (key: string) => env.get(key).asString();
+const getOptionalIntPositive = (key: string) => env.get(key).asIntPositive();
 const getOptionalBoolean = (key: string) => env.get(key).asBool();
 
+function generateSecretKey() {
+  const buf = Buffer.allocUnsafe(sodium.crypto_secretbox_KEYBYTES);
+  sodium.randombytes_buf(buf);
+  return buf.toString('hex');
+}
+
 export const nodeEnv = getRequiredString('NODE_ENV');
-export const appDomain = getRequiredString('APP_DOMAIN');
 export const mongoUrl = getRequiredString('MONGO_URL');
 export const mongoDbName = getRequiredString('MONGO_DB_NAME');
 export const mongoDbUser = getRequiredString('MONGO_DB_USER');
 export const mongoDbPassword = getRequiredString('MONGO_DB_PASSWORD');
-export const secretSessionKey = getRequiredString('SECRET_SESSION_KEY');
 
+export const httpSchema = getOptionalString('HTTP_SCHEMA') || 'https';
+export const host = getOptionalString('HOST') || '0.0.0.0';
+export const port = getOptionalIntPositive('PORT') || 8080;
 export const rootDomain = getOptionalString('ROOT_DOMAIN') || 'bundlemon.dev';
+export const appDomain = getOptionalString('APP_DOMAIN') || rootDomain;
+export const secretSessionKey = getOptionalString('SECRET_SESSION_KEY') || generateSecretKey();
 export const isTestEnv = getOptionalBoolean('IS_TEST_ENV') ?? false;
-export const shouldServeWebsite = getOptionalBoolean('SHOULD_SERVE_WEBSITE') ?? false;
+export const shouldServeWebsite = getOptionalBoolean('SHOULD_SERVE_WEBSITE') ?? true;
+export const maxSessionAgeSeconds = getOptionalIntPositive('MAX_SESSION_AGE_SECONDS') || 60 * 60 * 6; // 6 hours
+export const maxBodySizeBytes = getOptionalIntPositive('MAX_BODY_SIZE_BYTES') || 1024 * 1024; // 1MB
+export const shouldRunDbInit = getOptionalBoolean('SHOULD_RUN_DB_INIT') ?? true;
 
 export const githubAppId = getOptionalString('GITHUB_APP_ID');
 export const githubAppPrivateKey = getOptionalString('GITHUB_APP_PRIVATE_KEY');

apps/service/src/framework/mongo/client.ts

@@ -10,9 +10,15 @@ const getClient = async () => {
       const auth: MongoClientOptions['auth'] =
         nodeEnv === 'production' ? { username: mongoDbUser, password: mongoDbPassword } : undefined;
 
-      client = await MongoClient.connect(`${mongoUrl}/${mongoDbName}?retryWrites=true&w=majority`, {
+      client = await MongoClient.connect(`${mongoUrl}/${mongoDbName}`, {
         auth,
         readPreference: ReadPreference.PRIMARY,
+        writeConcern: {
+          w: 'majority',
+        },
+        retryWrites: true,
+        connectTimeoutMS: 5000,
+        serverSelectionTimeoutMS: 5000,
       });
     } catch (err) {
       throw new Error('Could not connect to mongo\n ' + err);