🛰️ Backend deployments (#4364)

* Add a development Dockerfile

* Only run the API if `APP_SECRET_KEY` is defined

* Update the dev render.com blueprint

* Add a production Dockerfile

* Update the dev Dockerfile

* Add a production deployment

* Fix the deployment buttons

* Upgrade the render.com web server plan

* Fix the CI

* Fix the production deployment

* Generate the prisma client in the images

* Do not crash when the email provider isn't setup

* Fix the image build

* Set the docker images node version

* Add an simpler way to run the backend

* Crash on wrong email provider config on production

* Add a bit more documentation

* Serve logos from a s3 bucket

* Allow setting custom QN and Pioneer instances

* Deploy render config from `main` by default

* Fix server dev db scripts

* Use postgres:16

* Upgrade the render.com db to the `starter` plan

* Use non alpine images for the builder steps

* Reverse last change

* Base prod image on `node:20-slim`

* Build the prod image based on the full debian docker node image

* Set the default cron frequency to 10min

backend.Dockerfile

@@ -0,0 +1,48 @@
+FROM node:20 AS base
+WORKDIR /app
+
+RUN set -eux; \
+	apt-get update -y; \
+	apt-get install -y --no-install-recommends openssl; \
+	rm -rf /var/lib/apt/list/*
+
+FROM base AS builder
+
+COPY packages/server/package.json ./
+COPY yarn.lock ./
+RUN yarn --immutable
+
+COPY tsconfig.json ./base.tsconfig.json
+COPY packages/server/tsconfig.json ./
+RUN sed -i 's/"extends":.*/"extends": ".\/base.tsconfig.json",/' tsconfig.json
+
+COPY packages/server/prisma ./
+COPY packages/server/codegen.ts ./
+COPY packages/server/src ./src
+RUN yarn build:local
+
+RUN rm -rf node_modules
+ENV NODE_ENV=production
+RUN yarn --prod --immutable
+RUN yarn prisma generate
+
+FROM base
+
+USER www-data
+
+COPY --from=builder --chown=www-data /app/dist ./dist
+COPY --from=builder --chown=www-data /app/node_modules ./node_modules
+COPY packages/server/prisma ./prisma
+COPY packages/server/docker/prod/entrypoint.sh /entrypoint.sh
+COPY packages/server/docker/prod/notify.sh /usr/bin/notify
+
+ENV QUERY_NODE_ENDPOINT "https://query.joystream.org/graphql"
+ENV PIONEER_URL "https://pioneerapp.xyz"
+ENV STARTING_BLOCK 1
+ENV NODE_ENV=production
+ENV APP_LOG_LEVEL "verbose"
+ENV PORT 3000
+EXPOSE 3000/tcp
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["api"]

backend.dev.Dockerfile

@@ -0,0 +1,42 @@
+FROM node:18-alpine3.18 AS builder
+WORKDIR /app
+
+COPY packages/server/package.json ./
+COPY yarn.lock ./
+RUN yarn --immutable
+
+COPY tsconfig.json ./base.tsconfig.json
+COPY packages/server/tsconfig.json ./
+RUN sed -i 's/"extends":.*/"extends": ".\/base.tsconfig.json",/' tsconfig.json
+
+COPY packages/server/prisma ./
+COPY packages/server/codegen.ts ./
+COPY packages/server/src ./src
+RUN yarn build:local
+
+RUN rm -rf node_modules
+ENV NODE_ENV=production
+RUN yarn --prod --immutable
+RUN yarn prisma generate
+
+FROM postgres:16.0-alpine3.18
+WORKDIR /app
+RUN apk add --no-cache nodejs
+
+COPY --from=builder --chown=postgres /app/dist ./dist
+COPY --from=builder --chown=postgres /app/node_modules ./node_modules
+COPY packages/server/prisma ./prisma
+COPY packages/server/docker/dev/entrypoint.sh /entrypoint.sh
+COPY packages/server/docker/dev/notify.sh /usr/bin/notify
+COPY packages/server/docker/dev/env.sh ./
+COPY packages/server/docker/dev/prisma-deploy.sh /docker-entrypoint-initdb.d/
+
+ENV QUERY_NODE_ENDPOINT "https://query.joystream.org/graphql"
+ENV PIONEER_URL "https://pioneerapp.xyz"
+ENV STARTING_BLOCK 1
+ENV APP_LOG_LEVEL "verbose"
+ENV PORT 3000
+EXPOSE 3000/tcp
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["api"]

packages/server/.env.dev

@@ -6,27 +6,43 @@ POSTGRES_DB: "pioneer"
 POSTGRES_USER: "pioneer"
 POSTGRES_PASSWORD: "pioneer"
 
-# Use a non standard port to not interfere with the Joystream mono-repo `db` service
+# Use a non standard port to not interfere with the local Joystream mono-repo `db` service.
 DB_PORT: 5433
 
+# URL Prisma uses to connect to the database (this value is not used when runing the api from the docker compose file).
 DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${DB_PORT}/${POSTGRES_DB}"
 
 #
 # Joystream environment
 #
 
+# Query node to fetch from:
+# - http://localhost:8081/graphql to connect to a query node running locally.
+# - http://graphql-server:8081/graphql to connect to the Joystream mono-repo docker service from its network.
+# - https://query.joystream.org/graphql to connect to the Joystream's mainnet query node.
 QUERY_NODE_ENDPOINT: "https://query.joystream.org/graphql"
+
+# Pioneer instance to link to in the notification (Use "http://localhost:8080" for the local instance).
 PIONEER_URL: "https://pioneerapp.xyz"
+
+# Block to start fetching the events from.
 STARTING_BLOCK: 1
 
 #
 # General
 #
 
+# Port to run the api on.
 PORT: 3000
+
+# The key used to to sign JSON Web Token with.
 APP_SECRET_KEY: "SECRET_1234"
+
+# Log level.
 APP_LOG_LEVEL: "verbose"
 
+NODE_ENV: "development"
+
 #
 # Email
 #
@@ -41,7 +57,8 @@ APP_LOG_LEVEL: "verbose"
 # MAILGUN_API_URL: "https://api.eu.mailgun.net" # this is needed for EU domains
 
 #
-# Useful on personal deployment
+# For development only:
 #
 
+# Create default members (their tokens will be displayed when the server start).
 INITIAL_MEMBERSHIPS: '[{ "id": 1, "name": "Alice", "email": "[email protected]" }, { "id": 2, "name": "Bob", "email": "[email protected]" }]'

packages/server/README.md

@@ -30,7 +30,26 @@ It is composed of 3 parts:
 
 ## Quick Start
 
-[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/Joystream/pioneer/tree/feature/backend-poc)
+### render.com deployments
+
+### Run with docker
+
+```shell
+yarn workspace server docker:up
+```
+
+This runs the api on: http://localhost:3000
+
+Configurations are available in `packages/server/.env`.
+
+To run the notification script:
+```shell
+yarn workspace server docker:notify
+```
+
+### Demo deployment
+
+[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/Joystream/pioneer/tree/backend-dev-blueprint)
 
 > **Warning**
 >
@@ -51,6 +70,10 @@ Mapping existing Joystream memberships id to a name and an email address in the
 
 In order to customize the default notification behavior with the GraphQL API, an authorization token can be found for each membership in the "Logs" section.
 
+### Production
+
+[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/Joystream/pioneer/tree/main)
+
 ## Production CLI usage
 
 > **Note**
@@ -59,7 +82,7 @@ In order to customize the default notification behavior with the GraphQL API, an
 
 - `yarn start:api`: starts the API server.
 - `yarn notify`: run the notify job.
-- `yarn start:all`: for environments where cron is not available, starts both the API server, and schedule the notify job every 30 minutes via [`node-cron`](https://www.npmjs.com/package/node-cron).
+- `yarn start:all`: for environments where cron is not available, starts both the API server, and schedule the notify job every 10 minutes via [`node-cron`](https://www.npmjs.com/package/node-cron).
 
 ## API usage
 
@@ -210,7 +233,7 @@ query {
 
 To run the API to develop locally:
 
-1. `yarn --frozen-lockfile`: Install the dependencies.
+1. `yarn --immutable`: Install the dependencies.
 2. Create and configure a `packages/server/.env`.
 3. Prepare the database and generate the code by running either:
     - `yarn workspace server dev:db:build`: To use docker for the db.

packages/server/docker-compose.yml

@@ -1,14 +1,33 @@
 version: '3.4'
 
 volumes:
-  db-data:
+  pioneer-db-data:
+
+networks:
+  default:
+    name: joystream_default
+    external: true
 
 services:
-  db:
-    image: postgres:12
+  pioneer-api:
+    image: thesan/pioneer-backend # TODO change this to joystream/...
+    build:
+      context: ../..
+      dockerfile: backend.Dockerfile
+    ports:
+      - '${PORT}:${PORT}'
+    depends_on:
+      - pioneer-db
+    env_file:
+      - .env # ensure `.env` exist
+    environment:
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@pioneer-db:5432/${POSTGRES_DB}
+
+  pioneer-db:
+    image: postgres:16
     ports:
       - '127.0.0.1:${DB_PORT}:5432'
     volumes:
-      - db-data:/var/lib/postgresql/data
+      - pioneer-db-data:/var/lib/postgresql/data
     env_file:
-      - .env
+      - .env # ensure `.env` exist

packages/server/docker/dev/entrypoint.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+export PATH=$PATH:/app/node_modules/.bin
+. /app/env.sh
+
+case "$1" in
+    api)
+        docker-entrypoint.sh postgres &
+
+        prismaClient=/app/node_modules/.prisma/client/index.js
+        while grep -oh "@prisma/client did not initialize yet." $prismaClient ; do
+            sleep 1
+        done
+
+        node /app/dist/common/scripts/startApi
+    ;;
+
+    postgres)
+        docker-entrypoint.sh postgres
+    ;;
+
+
+    *)
+        exec "$@"
+    ;;
+esac

packages/server/docker/dev/env.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+export POSTGRES_USER=${POSTGRES_USER:="postgres"}
+export POSTGRES_DB=${POSTGRES_DB:=POSTGRES_USER}
+if [ -z "$POSTGRES_PASSWORD" ]; then
+    export DATABASE_URL="postgresql://${POSTGRES_USER}@localhost/${POSTGRES_DB}?host=/var/run/postgresql/"
+else
+    export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost/${POSTGRES_DB}?host=/var/run/postgresql/"
+fi
+
+export APP_SECRET_KEY=${APP_SECRET_KEY:=${POSTGRES_PASSWORD:="pioneer"}}

packages/server/docker/dev/notify.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+. /app/env.sh
+node /app/dist/notifier/scripts/notify

packages/server/docker/dev/prisma-deploy.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+cd /app
+prisma migrate deploy 

packages/server/docker/prod/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+
+export PATH=$PATH:/app/node_modules/.bin
+
+case "$1" in
+    api)
+        cd /app
+        until prisma migrate deploy; do
+          echo "Waiting for the db to be ready..."
+          sleep 1
+        done
+        node ./dist/common/scripts/startApi
+    ;;
+
+    *)
+        exec "$@"
+    ;;
+esac

packages/server/docker/prod/notify.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+node /app/dist/notifier/scripts/notify

packages/server/docker/up.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -x
+
+if [ ! -f .env ]; then
+  sed 's/^QUERY_NODE_ENDPOINT: .*/QUERY_NODE_ENDPOINT: "http:\/\/graphql-server:8081\/graphql"/' .env.dev > .env
+fi
+docker network create joystream_default 2> /dev/null
+yarn docker down
+yarn docker up

packages/server/package.json

@@ -6,12 +6,14 @@
     "prisma": "prisma",
     "codegen": "graphql-codegen",
     "docker": "docker-compose",
+    "docker:up": "./docker/up.sh",
+    "docker:notify": "yarn docker run pioneer-api notify",
     "test": "jest --runInBand",
     "tsc": "tsc",
     "ts-node": "ts-node --transpile-only -r tsconfig-paths/register",
     "nodemon": "nodemon --transpile-only -r tsconfig-paths/register",
     "dev:build": "yarn codegen && yarn prisma db push",
-    "dev:db": "yarn docker up -d",
+    "dev:db": "yarn docker up pioneer-db -d",
     "dev:db:reset": "yarn docker down -v && yarn dev:db && yarn prisma db push",
     "dev:db:build": "yarn dev:db && yarn dev:build",
     "dev:api": "yarn nodemon ./src/common/scripts/startApi.ts",
@@ -37,6 +39,7 @@
     "@react-email/render": "^0.0.7",
     "@sendgrid/mail": "^7.7.0",
     "apollo-server": "^3.11.1",
+    "dotenv": "^16.0.3",
     "graphql": "^16.6.0",
     "graphql-request": "^5.1.0",
     "jsonwebtoken": "^9.0.0",
@@ -66,11 +69,12 @@
     "@types/lodash": "^4",
     "@types/node": "^18.14.0",
     "@types/node-cron": "^3.0.7",
+    "@types/npmlog": "^4.1.4",
     "@types/prettier": "^2",
+    "@types/react": "^18.2.31",
     "@typescript-eslint/eslint-plugin": "^5.52.0",
     "@typescript-eslint/parser": "^5.52.0",
     "babel": "^6.23.0",
-    "dotenv": "^16.0.3",
     "eslint": "^8.33.0",
     "jest": "^29.4.3",
     "nodemon": "^2.0.21",

packages/server/src/common/config.ts

@@ -5,7 +5,7 @@ config()
 
 const {
   PORT = 3000,
-  APP_SECRET_KEY, // TODO check this is defined when running the api
+  APP_SECRET_KEY,
   APP_LOG_LEVEL = 'info',
   QUERY_NODE_ENDPOINT = 'https://query.joystream.org/graphql',
   PIONEER_URL = 'https://pioneerapp.xyz',