Optimise and revamp microservice Docker builds to leverage cache, min…

…imise disk size etc.

.dockerignore

@@ -5,4 +5,4 @@
 **/*.log
 .secrets/*
 data
-grafana
+grafana

.github/workflows/build-images-from-branch.yml

@@ -32,70 +32,70 @@ jobs:
       - uses: actions/checkout@v4
         if: github.event_name == 'push'
 
-      - name: Set version
-        id: set-version
+      - name: Set version and branch
+        id: set-version-and-branch
         run: |
           export VERSION=`git log -1 --pretty=format:%h`
           echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
 
       - name: Get list of services
         id: get-services
         run: |
-          services=$(grep "^  [^ ]" docker-compose.yml | grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
+          services=$(grep "^  [^ ]" docker-compose.yml | grep -v base| grep -v '#' | awk -F: '{print $1}' | sed -e 's/^ *//')
           services_json=$(echo $services | tr '\n' ',' | sed 's/,$//' | jq -R 'split(" ")' | tr -d '\n')
           # Set the list of service names as an output variable
           echo "services=$services_json" >> $GITHUB_OUTPUT
           echo "services=$services_json"
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push base image
+        uses: docker/build-push-action@v5
+        with:
+          file: packages/Dockerfile.base
+          context: .
+          push: true
+          tags: 'opencrvs/ocrvs-base:${{ steps.set-version-and-branch.outputs.branch }}'
+          cache-from: type=registry,ref=opencrvs/ocrvs-base:${{ steps.set-version-and-branch.outputs.branch }}
+          cache-to: type=inline
 
     outputs:
       services: ${{ steps.get-services.outputs.services }}
-      version: ${{ steps.set-version.outputs.version }}
+      version: ${{ steps.set-version-and-branch.outputs.version }}
+      branch: ${{ steps.set-version-and-branch.outputs.branch }}
 
   build:
     needs: base
     strategy:
+      fail-fast: false
       matrix:
         service: ${{ fromJSON(needs.base.outputs.services) }}
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-        if: github.event_name == 'workflow_dispatch'
-        with:
-          ref: '${{ github.event.inputs.branch_name }}'
-
-      - uses: actions/checkout@v4
-        if: github.event_name == 'push'
-
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Build ${{ matrix.service }}
-        run: |
-          export VERSION="${{ needs.base.outputs.version }}"
-          echo ${{ matrix.service }}
-          docker compose build ${{ matrix.service }}
-          docker compose push ${{ matrix.service }}
-
-  security-scans:
-    needs: [base, build]
-    strategy:
-      fail-fast: false
-      matrix:
-        service: ${{ fromJSON(needs.base.outputs.services) }}
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      - name: Build and push
+        uses: docker/build-push-action@v5
         with:
-          sparse-checkout: |
-            trivy.yaml
-            .trivyignore.yaml
-          sparse-checkout-cone-mode: false
+          file: packages/${{ matrix.service }}/Dockerfile
+          build-args: |
+            VERSION=${{ needs.base.outputs.branch }}
+          push: true
+          context: .
+          tags: |
+            opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}
+            opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.branch }}
+          cache-from: type=registry,ref=opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.branch }}
+          cache-to: type=inline
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.22.0
         with:
           image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
           trivy-config: trivy.yaml

.github/workflows/lint-and-test.yml

@@ -130,7 +130,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Run Trivy vulnerability scanner in fs mode
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.22.0
         with:
           scan-type: 'fs'
           scan-ref: '.'

.github/workflows/publish-release.yml

@@ -102,7 +102,7 @@ jobs:
             .trivyignore.yaml
           sparse-checkout-cone-mode: false
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.22.0
         with:
           image-ref: 'opencrvs/ocrvs-${{ matrix.service }}:${{ needs.base.outputs.version }}'
           trivy-config: trivy.yaml

docker-compose.yml

@@ -9,38 +9,61 @@
 version: '3.3'
 
 services:
+  base:
+    image: opencrvs/ocrvs-base:${VERSION}
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: ./packages/Dockerfile.base
+    deploy:
+      replicas: 0
   client:
     image: opencrvs/ocrvs-client:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/client/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
 
   dashboards:
     image: opencrvs/ocrvs-dashboards:${VERSION}
+    platform: linux/amd64
     build:
-      context: ./packages/dashboards
-      dockerfile: ./Dockerfile
+      context: .
+      dockerfile: ./packages/dashboards/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
 
   components:
     image: opencrvs/ocrvs-components:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/components/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
   login:
     image: opencrvs/ocrvs-login:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/login/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
   gateway:
     image: opencrvs/ocrvs-gateway:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/gateway/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - REDIS_HOST=redis
       - CONFIG_SMS_CODE_EXPIRY_SECONDS=600
@@ -62,10 +85,13 @@ services:
   # User facing services
   workflow:
     image: opencrvs/ocrvs-workflow:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/workflow/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - NODE_ENV=development
       - SEARCH_URL=http://search:9090/
@@ -80,10 +106,13 @@ services:
       - COUNTRY=${COUNTRY:-bgd} # PEN_TEST change to gbr
   search:
     image: opencrvs/ocrvs-search:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/search/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - HEARTH_MONGO_URL=mongodb://mongo1/hearth-dev
       - NODE_ENV=development
@@ -93,10 +122,13 @@ services:
       - APPLICATION_CONFIG_URL=http://config:2021/
   metrics:
     image: opencrvs/ocrvs-metrics:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/metrics/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - INFLUX_HOST=influxdb
       - INFLUX_PORT=8086
@@ -113,19 +145,25 @@ services:
   # END User facing services
   scheduler:
     image: opencrvs/ocrvs-scheduler:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/scheduler/Dockerfile
+    depends_on:
+      - base
     environment:
       - NODE_ENV=development
       - METRICS_URL=http://metrics:1050
       - OPENHIM_MONGO_URL=mongodb://mongo1/openhim-dev
   auth:
     image: opencrvs/ocrvs-auth:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/auth/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - REDIS_HOST=redis
       - USER_MANAGEMENT_URL=http://user-mgnt:3030/
@@ -135,10 +173,13 @@ services:
       - METRICS_URL=http://metrics:1050
   user-mgnt:
     image: opencrvs/ocrvs-user-mgnt:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/user-mgnt/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - MONGO_URL=mongodb://mongo1/user-mgnt
       - NOTIFICATION_SERVICE_URL=http://notification:2020/
@@ -148,10 +189,13 @@ services:
       - APPLICATION_CONFIG_URL=http://config:2021/
   webhooks:
     image: opencrvs/ocrvs-webhooks:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/webhooks/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - REDIS_HOST=redis://redis:6379
       - MONGO_URL=mongodb://mongo1/webhooks
@@ -162,20 +206,26 @@ services:
       - FHIR_URL=http://hearth:3447/fhir
   notification:
     image: opencrvs/ocrvs-notification:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/notification/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - COUNTRY=${COUNTRY:-bgd}
       - HOST=0.0.0.0
       - COUNTRY_CONFIG_URL=http://countryconfig:3040
   config:
     image: opencrvs/ocrvs-config:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/config/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - AUTH_URL=http://auth:4040
       - COUNTRY_CONFIG_URL=http://countryconfig:3040
@@ -191,10 +241,13 @@ services:
       - CHECK_INVALID_TOKEN=true
   migration:
     image: opencrvs/ocrvs-migration:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/migration/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - USER_MGNT_MONGO_URL=mongodb://mongo1/user-mgnt
       - APPLICATION_CONFIG_MONGO_URL=mongodb://mongo1/application-config
@@ -208,10 +261,13 @@ services:
       - WAIT_HOSTS=mongo1:27017,influxdb:8086,minio:9000,elasticsearch:9200
   documents:
     image: opencrvs/ocrvs-documents:${VERSION}
+    platform: linux/amd64
     build:
       context: .
       dockerfile: ./packages/documents/Dockerfile
     restart: unless-stopped
+    depends_on:
+      - base
     environment:
       - NODE_ENV=development
       - COUNTRY_CONFIG_URL=http://countryconfig:3040

license-config.json

@@ -21,7 +21,7 @@
     "AUTHORS",
     "LICENSE",
     "license-header.txt",
-    "**/*Dockerfile",
+    "**/*Dockerfile*",
     "**/packages/components/lib",
     "**/packages/*/build",
     "packages/gateway/src/graphql/schema.d.ts",

package.json

@@ -34,7 +34,7 @@
     "cov:client": "cd packages/client && yarn open:cov",
     "cov:login": "cd packages/login && yarn open:cov",
     "db:clear:all": "bash development-environment/clear-all-data.sh",
-    "build": "lerna run build --stream --concurrency=1 && lerna run build-storybook",
+    "build": "lerna run build --stream && lerna run build-storybook",
     "build:image": "export VERSION=`git log -1 --pretty=format:%h` && docker build --build-arg VERSION=${VERSION} -t opencrvs-build . && unset VERSION",
     "build:clean": "lerna run build:clean",
     "check:license": "license-check-and-add check -f license-config.json",

packages/Dockerfile.base

@@ -0,0 +1,18 @@
+FROM node:18-slim
+
+RUN apt-get update && apt-get upgrade -y
+
+RUN apt-get clean && \
+    rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
+
+USER node
+
+WORKDIR /app
+
+COPY --chown=node:node *.json .
+COPY --chown=node:node yarn.lock .
+
+RUN yarn install --production --frozen-lockfile
+
+COPY --chown=node:node packages/commons /app/packages/commons
+RUN cd packages/commons && yarn install --frozen-lockfile && yarn build