move build pipeline to circleci

.circleci/config.yml

@@ -1,6 +1,6 @@
 version: 2
 jobs:
-  build:
+  lint:
     docker:
       - image: circleci/python:3.9-buster
     steps:
@@ -10,15 +10,19 @@ jobs:
           command: |
             sudo pip install --upgrade pre-commit==2.15.0
             pre-commit run --all-files
+  test-and-build:
+    docker:
+      - image: circleci/python:3.9-buster
+    steps:
+      - checkout
       - setup_remote_docker:
-          version: 17.10.0-ce
+          version: 20.10.7
           docker_layer_caching: true
       - run:
-          name: Build docker images
-          command: make build-ci
-      - run:
-          name: Run mocha tests
-          command: make test-js-ci
+          name: Build test image
+          command: |
+            cp .env-build .env
+            DOCKER_BUILDKIT=1 ./bin/dc.sh build --progress=plain test
       - run:
           # copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts:
           # https://circleci.com/docs/2.0/building-docker-images/#mounting-folders
@@ -28,4 +32,23 @@ jobs:
             docker cp ./kitsune/search/dictionaries/synonyms/. project_elasticsearch7_1:/usr/share/elasticsearch/config/synonyms
       - run:
           name: Run unit tests
-          command: make test-ci
+          command: ./bin/dc.sh run test ./bin/run-unit-tests.sh
+      - run:
+          name: Run js tests
+          command: ./bin/dc.sh run test ./bin/run-mocha-tests.sh
+      - run:
+          name: Build prod image
+          command: DOCKER_BUILDKIT=1 ./bin/dc.sh build --progress=plain prod
+      - run:
+          name: Push prod image
+          command: |
+            docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}"
+            source docker/bin/set_git_env_vars.sh
+            # docker push mozilla/kitsune:prod-${GIT_COMMIT_SHORT}
+            docker logout
+workflows:
+  version: 2
+  lint-test-build:
+    jobs:
+      - lint
+      - test-and-build

.env-build

@@ -7,6 +7,5 @@ CELERY_TASK_ALWAYS_EAGER=True
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=sqlite://
 DATABASE_READ_ONLY_URL=sqlite://
-ES7_URLS=elasticsearch:9200
 SESSION_COOKIE_SECURE=False
 SECRET_KEY=secret

.env-test

@@ -1,3 +1,4 @@
+TEST=True
 DEBUG=False
 # Some cron jobs are skipped on stage.
 STAGE=False
@@ -16,7 +17,6 @@ CACHE_URL=redis://redis:6379/3
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=mysql://root:kitsune@mariadb:3306/kitsune
 DB_CONN_MAX_AGE=0
-ES_URLS=elasticsearch:9200
 SECRET_KEY=secret
 REUSE_STATIC=1
 REUSE_DB=0

Dockerfile

@@ -1,16 +1,6 @@
-################################
-# Frontend dependencies builder
-#
-FROM node:12 AS frontend-base
-
-WORKDIR /app
-COPY ["./package.json", "./package-lock.json", "prepare_django_assets.js", "/app/"]
-COPY ./kitsune/sumo/static/sumo /app/kitsune/sumo/static/sumo
-RUN npm run production
-
-################################
-# Python dependencies builder
-#
+#######################
+# Common dependencies #
+#######################
 FROM python:3.9-buster AS base
 
 WORKDIR /app
@@ -26,83 +16,90 @@ RUN python -m venv /venv
 RUN pip install --upgrade "pip==21.0.1"
 RUN useradd -d /app -M --uid 1000 --shell /usr/sbin/nologin kitsune
 
+RUN apt-get update && apt-get install apt-transport-https && \
+    curl -sL https://deb.nodesource.com/setup_12.x | bash -
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gettext build-essential \
     libxml2-dev libxslt1-dev zlib1g-dev git \
     libjpeg-dev libffi-dev libssl-dev libxslt1.1 \
-    libmariadb3 mariadb-client && \
+    libmariadb3 mariadb-client \
+    optipng nodejs zip && \
     rm -rf /var/lib/apt/lists/*
 
 COPY ./requirements/*.txt /app/requirements/
-
-RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt && \
-    pip install --no-cache-dir --require-hashes -r requirements/dev.txt
-
-ARG GIT_SHA=head
-ENV GIT_SHA=${GIT_SHA}
+RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt
 
 
-################################
-# Developer image
-#
-FROM base AS base-dev
-RUN apt-get update && apt-get install apt-transport-https && \
-    curl -sL https://deb.nodesource.com/setup_12.x | bash -
-RUN apt-get update && apt-get install -y --no-install-recommends optipng nodejs zip && \
-    rm -rf /var/lib/apt/lists/*
+#####################
+# Development image #
+#####################
+FROM base AS dev
 
+RUN pip install --no-cache-dir --require-hashes -r requirements/dev.txt
 
-################################
-# Fetch locales
-#
-FROM python:3.9-buster AS locales
 
-WORKDIR /app
+#########################
+# Frontend dependencies #
+#########################
+FROM base AS base-frontend
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends  gettext
+COPY ./package*.json /app/
+RUN npm run install-prod && npm run copy:protocol
 
-ENV PATH="/venv/bin:$PATH"
+COPY prepare_django_assets.js .
+RUN npm run postinstall
 
-COPY --from=base /venv /venv
+COPY kitsune/sumo/static/sumo/scss kitsune/sumo/static/sumo/scss
+RUN npm run build:scss && npm run build:postcss
 
 COPY . .
+RUN cp .env-build .env && \
+    ./manage.py nunjucks_precompile
 
-ARG LOCALE_ENV=master
-ENV LOCALE_ENV=${LOCALE_ENV}
-RUN ./docker/bin/fetch-l10n-files.sh
-RUN ./scripts/compile-linted-mo.sh && \
-    find ./locale ! -name '*.mo' -type f -delete
 
-ARG GIT_SHA=head
-ENV GIT_SHA ${GIT_SHA}
+########################
+# Testing dependencies #
+########################
+FROM base AS test-deps
 
+RUN pip install --no-cache-dir --require-hashes -r requirements/test.txt
 
-################################
-# Staticfiles builder
-#
-FROM base-dev AS staticfiles
 
-COPY --from=frontend-base --chown=kitsune:kitsune /app/assets /app/assets
-COPY --from=frontend-base --chown=kitsune:kitsune /app/node_modules /app/node_modules
-COPY --from=locales /app/locale /app/locale
+#################
+# Testing image #
+#################
+FROM base-frontend AS test
 
-COPY . .
+COPY --from=test-deps /venv /venv
 
-RUN cp .env-build .env && \
-    ./manage.py nunjucks_precompile && \
+RUN cp .env-test .env && \
     ./manage.py compilejsi18n && \
+    ./manage.py collectstatic --noinput
+
+
+##########################
+# Production dependences #
+##########################
+FROM base-frontend AS prod-deps
+
+ARG LOCALE_ENV=master
+ENV LOCALE_ENV=${LOCALE_ENV}
+RUN ./docker/bin/fetch-l10n-files.sh
+RUN ./scripts/compile-linted-mo.sh && \
+    find ./locale ! -name '*.mo' -type f -delete
+
+RUN ./manage.py compilejsi18n && \
     # minify jsi18n files:
     find jsi18n/ -name "*.js" -exec sh -c 'npx uglifyjs "$1" -o "${1%.js}-min.js"' sh {} \; && \
     ./manage.py collectstatic --noinput && \
     npx svgo -r -f static
 
 
-################################
-# Full prod image sans locales
-#
-FROM python:3.9-slim-buster AS full-no-locales
+##########################
+# Clean production image #
+##########################
+FROM python:3.9-slim-buster AS prod
 
 WORKDIR /app
 
@@ -113,32 +110,25 @@ ENV LANG=C.UTF-8
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 
+RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
+
+COPY --from=prod-deps --chown=kitsune:kitsune /venv /venv
+COPY --from=prod-deps --chown=kitsune:kitsune /app/locale /app/locale
+COPY --from=prod-deps --chown=kitsune:kitsune /app/static /app/static
+
+COPY --chown=kitsune:kitsune . .
+
+# apt-get after copying everything to ensure we're always getting the latest packages in the prod image
 RUN apt-get update && \
+    apt-get upgrade && \
     apt-get install -y --no-install-recommends \
     libmariadb3 optipng mariadb-client \
     libxslt1.1 && \
     rm -rf /var/lib/apt/lists/*
 
-RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
-
-COPY --from=base --chown=kitsune:kitsune /venv /venv
-COPY --from=staticfiles --chown=kitsune:kitsune /app/static /app/static
-
-COPY --chown=kitsune:kitsune . .
-
 RUN mkdir /app/media && chown kitsune:kitsune /app/media
 
 USER kitsune
 
 ARG GIT_SHA=head
 ENV GIT_SHA ${GIT_SHA}
-
-
-################################
-# Full final prod image
-#
-FROM full-no-locales AS full
-
-USER root
-COPY --from=locales --chown=kitsune:kitsune /app/locale /app/locale
-USER kitsune

bin/dc.sh

@@ -3,4 +3,4 @@
 source docker/bin/set_git_env_vars.sh
 export LOCALE_ENV=production
 
-docker-compose "$@"
+docker-compose -f docker-compose.yml -f docker-compose.ci.yml "$@"

docker-compose.ci.yml

@@ -0,0 +1,20 @@
+version: "3.4"
+services:
+  test:
+    build:
+      context: .
+      target: test
+    env_file: .env-test
+    depends_on:
+      - mariadb
+      - elasticsearch7
+      - redis
+
+  prod:
+    build:
+      context: .
+      target: prod
+      args:
+        - GIT_SHA
+        - LOCALE_ENV
+    image: mozilla/kitsune:prod-${GIT_COMMIT_SHORT:-latest}