From 2aabb09e5cf7053f41a88779cd594c9bd28288dd Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan Date: Wed, 27 Jul 2022 18:03:56 +0500 Subject: [PATCH] feat: Add Native Docker config --- .dockerignore | 1 + .github/workflows/docker-publish.yml | 50 +++++++++--- Dockerfile | 111 ++++++++++++++++----------- Makefile | 16 ---- 4 files changed, 107 insertions(+), 71 deletions(-) diff --git a/.dockerignore b/.dockerignore index d79256dcf7..74828119fc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ .dev/ htmlcov/ +Dockerfile diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 7938a27cd4..37bcde3fa6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,23 +1,51 @@ -name: Push Docker Images +name: Build and Push Docker Images on: push: branches: - master - + tags: + - open-release/* jobs: - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ push: runs-on: ubuntu-latest - if: github.event_name == 'push' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v2 + + # Use the release name as the image tag if we're building an open release tag. + # Examples: if we're building 'open-release/maple.1', tag the image as 'maple.1'. + # Otherwise, we must be building from a push to master, so use 'latest'. + - name: Get tag name + id: get-tag-name + uses: actions/github-script@v5 + with: + script: | + const releasePrefix = 'refs/tags/open-release/'; + const tagName = context.ref.split(releasePrefix)[1] || 'latest'; + console.log('Will use tag: ' + tagName); + return tagName; + result-encoding: string + + - name: Build and push Dev Docker image + uses: docker/build-push-action@v1 + with: + push: true + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + target: dev + repository: edxops/discovery-dev + tags: ${{ steps.get-tag-name.outputs.result }},${{ github.sha }} - - name: Build and Push docker image - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - run : make docker_push + # The current priority is to get the devstack off of Ansible based Images. Once that is done, we can come back to this part to get + # suitable images for smaller prod environments. + # - name: Build and push prod Docker image + # uses: docker/build-push-action@v1 + # with: + # push: true + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_PASSWORD }} + # target: prod + # repository: edxops/discovery-prod + # tags: ${{ steps.get-tag-name.outputs.result }},${{ github.sha }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index db42fce7d1..d60d1fab40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,21 @@ FROM ubuntu:focal as app +ENV DEBIAN_FRONTEND noninteractive # System requirements. -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get upgrade -qy -RUN apt-get install --yes \ - git \ - language-pack-en \ - python3-venv \ - python3.8-dev \ - python3.8-venv \ - build-essential \ - libffi-dev \ - libmysqlclient-dev \ - libxml2-dev \ - libxslt1-dev \ - libjpeg-dev \ - libssl-dev \ - libcairo2-dev - -RUN rm -rf /var/lib/apt/lists/* - -ENV VIRTUAL_ENV=/venv -RUN python3.8 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -RUN pip install pip==20.2.3 setuptools==50.3.0 nodeenv +RUN apt update && \ + apt-get install -qy \ + curl \ + # required by bower installer + git \ + language-pack-en \ + build-essential \ + python3.8-dev \ + python3-virtualenv \ + python3.8-distutils \ + libmysqlclient-dev \ + libssl-dev \ + libcairo2-dev && \ + rm -rf /var/lib/apt/lists/* # Use UTF-8. RUN locale-gen en_US.UTF-8 @@ -33,43 +23,76 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -# Make necessary directories and environment variables. -RUN mkdir -p /edx/var/discovery/staticfiles -RUN mkdir -p /edx/var/discovery/media -ENV DJANGO_SETTINGS_MODULE course_discovery.settings.production +ARG COMMON_APP_DIR="/edx/app" +ARG COMMON_CFG_DIR="/edx/etc" +ARG DISCOVERY_SERVICE_NAME="discovery" +ARG DISCOVERY_APP_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}" +ARG DISCOVERY_VENV_DIR="${COMMON_APP_DIR}/${DISCOVERY_SERVICE_NAME}/venvs/${DISCOVERY_SERVICE_NAME}" +ARG DISCOVERY_CODE_DIR="${DISCOVERY_APP_DIR}/${DISCOVERY_SERVICE_NAME}" +ARG DISCOVERY_NODEENV_DIR="${DISCOVERY_APP_DIR}/nodeenvs/${DISCOVERY_SERVICE_NAME}" + +ENV PATH "${DISCOVERY_VENV_DIR}/bin:${DISCOVERY_NODEENV_DIR}/bin:$PATH" +ENV DISCOVERY_CFG "/edx/etc/discovery.yml" +ENV DISCOVERY_CODE_DIR "${DISCOVERY_CODE_DIR}" +ENV DISCOVERY_APP_DIR "${DISCOVERY_APP_DIR}" + +RUN virtualenv -p python3.8 --always-copy ${DISCOVERY_VENV_DIR} + +# No need to activate discovery venv as it is already in path +RUN pip install nodeenv + +RUN nodeenv ${DISCOVERY_NODEENV_DIR} --node=16.14.0 --prebuilt && npm install -g npm@8.5.x # Working directory will be root of repo. -WORKDIR /edx/app/discovery +WORKDIR ${DISCOVERY_CODE_DIR} # Copy just JS requirements and install them. COPY package.json package.json COPY package-lock.json package-lock.json -RUN nodeenv /edx/app/nodeenv --node=16.14.2 --npm=8.5.x --prebuilt -ENV PATH /edx/app/nodeenv/bin:${PATH} -RUN npm install --production COPY bower.json bower.json -RUN ./node_modules/.bin/bower install --allow-root --production +RUN npm install --production && ./node_modules/.bin/bower install --allow-root --production + +# Expose canonical Discovery port +EXPOSE 18381 +EXPOSE 8381 + +FROM app as prod + +ENV DJANGO_SETTINGS_MODULE "course_discovery.settings.production" -# Copy just Python requirements & install them. -COPY requirements/ requirements/ -RUN pip install -r requirements/production.txt +COPY requirements/production.txt ${DISCOVERY_CODE_DIR}/requirements/production.txt + +RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/production.txt # Copy over rest of code. # We do this AFTER requirements so that the requirements cache isn't busted # every time any bit of code is changed. COPY . . -# Expose canonical Discovery port -EXPOSE 8381 - CMD gunicorn --bind=0.0.0.0:8381 --workers 2 --max-requests=1000 -c course_discovery/docker_gunicorn_configuration.py course_discovery.wsgi:application -FROM app as newrelic -RUN pip install newrelic -CMD newrelic-admin run-program gunicorn --bind=0.0.0.0:8381 --workers 2 --max-requests=1000 -c course_discovery/docker_gunicorn_configuration.py course_discovery.wsgi:application +FROM app as dev + +ENV DJANGO_SETTINGS_MODULE "course_discovery.settings.devstack" + +COPY requirements/local.txt ${DISCOVERY_CODE_DIR}/requirements/local.txt +COPY requirements/django.txt ${DISCOVERY_CODE_DIR}/requirements/django.txt + +RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/django.txt +RUN pip install -r ${DISCOVERY_CODE_DIR}/requirements/local.txt + +# Devstack related step for backwards compatibility +RUN touch ${DISCOVERY_APP_DIR}/discovery_env + +# Copy over rest of code. +# We do this AFTER requirements so that the requirements cache isn't busted +# every time any bit of code is changed. +COPY . . + +CMD while true; do python ./manage.py runserver 0.0.0.0:8381; sleep 2; done ########################################################### # Define k8s target -FROM app as kubernetes +FROM prod as kubernetes ENV DISCOVERY_SETTINGS='kubernetes' ENV DJANGO_SETTINGS_MODULE="course_discovery.settings.$DISCOVERY_SETTINGS" diff --git a/Makefile b/Makefile index 968c1a0248..836e54cd73 100644 --- a/Makefile +++ b/Makefile @@ -134,22 +134,6 @@ docs: check_keywords: ## Scan the Django models in all installed apps in this project for restricted field names python manage.py check_reserved_keywords --override_file db_keyword_overrides.yml -docker_build: - docker build . -f Dockerfile --target app -t openedx/discovery - docker build . -f Dockerfile --target newrelic -t openedx/discovery:latest-newrelic - -docker_tag: docker_build - docker tag openedx/discovery openedx/discovery:${GITHUB_SHA} - docker tag openedx/discovery:latest-newrelic openedx/discovery:${GITHUB_SHA}-newrelic - -docker_auth: - echo "$$DOCKERHUB_PASSWORD" | docker login -u "$$DOCKERHUB_USERNAME" --password-stdin - -docker_push: docker_tag docker_auth ## push to docker hub - docker push 'openedx/discovery:latest' - docker push "openedx/discovery:${GITHUB_SHA}" - docker push 'openedx/discovery:latest-newrelic' - docker push "openedx/discovery:${GITHUB_SHA}-newrelic" install_transifex_client: ## Install the Transifex client curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash