diff --git a/.github/workflows/build-and-deploy-images.yml b/.github/workflows/build-and-deploy-images.yml deleted file mode 100644 index 1ae46ca1b3..0000000000 --- a/.github/workflows/build-and-deploy-images.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Deploy to Docker Hub - -# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on -on: - push: - tags: - - "v*" - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - build-n-publish: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Set BUILD_VERSION for Production from git tag - # define BUILD_VERSION as stripping off initial 11 characters, i.e. "refs/tags/v" - run: |- - echo "BUILD_VERSION=$(echo ${GITHUB_REF:11})" >> "$GITHUB_ENV" - echo "BUILD_VERSION_LATEST=latest" >> "$GITHUB_ENV" - - - name: Build and tag app - run: docker build -t $IMAGE_NAME:$BUILD_VERSION -t $IMAGE_NAME:$BUILD_VERSION_LATEST --build-arg BUILD_VERSION=${BUILD_VERSION} -f docker/app/Dockerfile . - env: - IMAGE_NAME: sillsdev/web-languageforge - - - name: Reveal images - run: docker images - - - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - - - name: Publish all tagged images to Docker Hub - run: | - docker push -a sillsdev/web-languageforge diff --git a/.github/workflows/deployment-staging.yml b/.github/workflows/deployment-staging.yml deleted file mode 100644 index d091c8c2ed..0000000000 --- a/.github/workflows/deployment-staging.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Deploy to staging - -# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on -on: - push: - branches: - - develop - -jobs: - build: - runs-on: ubuntu-latest - - outputs: - IMAGE: ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG }} - - steps: - - - uses: actions/checkout@v2 - - - name: Build app - working-directory: ./docker/ - run: docker-compose build --build-arg ENVIRONMENT=production app - - - name: Check unit tests - working-directory: ./docker/ - run: make unit-tests - # - - # name: Check e2e tests - # working-directory: ./docker/ - # run: make e2e-tests - - - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - - - name: Establish image name - id: image - run: | - echo ::set-output name=NAMESPACE::sillsdev/web-languageforge - echo ::set-output name=TAG::develop-$(date +%Y%m%d)-${{ github.sha }} - - - name: Tag images - run: docker tag lf-app ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG }} - - - name: Publish image - run: docker push ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG }} - - deploy: - runs-on: [self-hosted, languageforge] - - needs: build - - steps: - - - uses: sillsdev/common-github-actions/install-kubectl@v1 - - - run: kubectl --context ${{ secrets.LTOPS_K8S_STAGING_CONTEXT }} set image deployment/app app=${{ needs.build.outputs.IMAGE }} diff --git a/.github/workflows/integrate-and-deploy.yml b/.github/workflows/integrate-and-deploy.yml new file mode 100644 index 0000000000..6a4b006ee9 --- /dev/null +++ b/.github/workflows/integrate-and-deploy.yml @@ -0,0 +1,81 @@ +name: Integrate changes and deploy + +# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on +on: + workflow_call: + # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#onworkflow_callinputs + inputs: + image-tag: + type: string + required: true + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_callsecrets + secrets: + kube-context: + required: true + image-repo-username: + required: true + image-repo-password: + required: true + +jobs: + integrate: + runs-on: ubuntu-latest + + env: + # https://docs.docker.com/develop/develop-images/build_enhancements/ + DOCKER_BUILDKIT: 1 + + defaults: + run: + working-directory: docker + + steps: + - + uses: actions/checkout@v2 + - + run: | + docker --version + docker-compose --version + - + name: Establish image name + id: image + run: | + echo ::set-output name=NAMESPACE::sillsdev/web-languageforge + echo ::set-output name=TAG::${{ inputs.image-tag }} + - + name: Build app + run: docker-compose build --build-arg ENVIRONMENT=production --build-arg BUILD_VERSION=${{ steps.image.outputs.TAG }} app + - + run: docker-compose run --rm app cat /var/www/html/build-version.txt /var/www/html/version.php + - + name: Check unit tests + run: make unit-tests-ci + - + name: Check e2e tests + run: make e2e-tests-ci + - + name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.image-repo-username }} + password: ${{ secrets.image-repo-password }} + - + name: Tag image + run: docker tag lf-app ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG }} + - + name: Publish image + run: docker push ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG }} + + outputs: + IMAGE: ${{ steps.image.outputs.NAMESPACE }}:${{ steps.image.outputs.TAG }} + + deploy: + runs-on: [self-hosted, languageforge] + + needs: integrate + + steps: + - + uses: sillsdev/common-github-actions/install-kubectl@v1 + - + run: kubectl --context ${{ secrets.kube-context }} set image deployment/app app=${{ needs.integrate.outputs.IMAGE }} diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml new file mode 100644 index 0000000000..afc22d559f --- /dev/null +++ b/.github/workflows/production.yml @@ -0,0 +1,21 @@ +name: Deploy to production + +# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on +on: + push: + tags: + - v* + +jobs: + production: + if: github.event.base_ref == 'refs/heads/master' + + # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_iduses + uses: sillsdev/web-languageforge/.github/workflows/integrate-and-deploy.yml@master + with: + image-tag: $(echo ${{ github.ref }} | sed 's/refs\/tags\///') + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsecrets + secrets: + kube-context: ${{ secrets.LTOPS_K8S_PRODUCTION_CONTEXT }} + image-repo-username: ${{ secrets.DOCKERHUB_USERNAME }} + image-repo-password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml new file mode 100644 index 0000000000..49dc259ee8 --- /dev/null +++ b/.github/workflows/staging.yml @@ -0,0 +1,19 @@ +name: Deploy to staging + +# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#on +on: + push: + branches: + - develop + +jobs: + staging: + # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_iduses + uses: sillsdev/web-languageforge/.github/workflows/integrate-and-deploy.yml@develop + with: + image-tag: develop-$(date +%Y%m%d)-${{ github.sha }} + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idsecrets + secrets: + kube-context: ${{ secrets.LTOPS_K8S_STAGING_CONTEXT }} + image-repo-username: ${{ secrets.DOCKERHUB_USERNAME }} + image-repo-password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1577f471a9..f1fec1331a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,16 +17,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.11 - - - name: Run Unit Tests + - + uses: actions/checkout@v2 + - + name: Run Unit Tests run: make unit-tests-ci - - - name: echo working directory contents - run: ls -l - - - name: Publish Test Results + - + name: Publish Test Results uses: docker://ghcr.io/enricomi/publish-unit-test-result-action:v1 if: always() with: @@ -38,13 +35,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: satackey/action-docker-layer-caching@v0.0.11 - - - name: Run E2E Tests + - + uses: actions/checkout@v2 + - + name: Build app + run: make build + - + name: Run E2E Tests run: make e2e-tests-ci - - - name: Publish Test Results + - + name: Publish Test Results uses: docker://ghcr.io/enricomi/publish-unit-test-result-action:v1 if: always() with: diff --git a/README.md b/README.md index c94cd5f113..0e4caa2a14 100644 --- a/README.md +++ b/README.md @@ -166,10 +166,6 @@ After a minute or two, your source or test changes should be applied and you sho 1. `make dev` will start the app in development mode, i.e. changes to source code will immediately be reflected in the locally running app. -### Building for deployment - -1. Refer to `/.github/workflows/build-and-deploy-images.yml` for production build commands and `/.github/workflows/deployment-staging.yml` for staging build commands. - ### Visual Studio Code ### Visual Studio Code is a simple, free, cross-platform code editor. You can download VS Code from [here](https://code.visualstudio.com/). @@ -237,24 +233,26 @@ To debug the tests: - To debug in VSCode, select the "Node debugger" debug configuration and run it. ## Application deployment ## -Language Forge is built to run in a containerized environment. For now, Kubernetes is the chosen runtime platform. Deployments are not currently automated and must be manually run with the appropriate credentials or from within our CD platform, TeamCity at this time. Deployment scripts for k8s can be found in `docker/deployment` +Language Forge is built to run in a containerized environment. For now, Kubernetes is the chosen runtime platform. Deployments are automated under the right circumstances using GitHub Actions. ### Staging (QA) ### -Staging deployments can be run with `VERSION= make deploy-staging`. +Staging deployments can be manually run with `VERSION= make deploy-staging`. Current workflow: -1. merge commits into or make commits on `develop` branch -1. this will kick off the GHA (`.github/workflows/deployment-staging.yml`) to build, publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags) and deploy to the staging environment. +1. merge PR into or make commits on `develop` branch +1. this will kick off the GHA (`.github/workflows/staging.yml`) to build, test and publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags) and deploy this code to the staging environment at https://qa.languageforge.org ### Production ### -Production deployments can be run with `VERSION= make deploy-prod`. +Production deployments can be manually run with `VERSION= make deploy-prod`. Current workflow: 1. merge from `develop` into `master` 1. "Draft a new release" on https://github.com/sillsdev/web-languageforge/releases with a `v#.#.#` tag format 1. "Publish" the new release -1. this will kick off the GHA (`.github/workflows/build-and-deploy-images.yml`) to build and publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags) -1. then the deployment scripts can be run either manually or via the TeamCity deploy job +1. this will kick off the GHA (`.github/workflows/production.yml`) to build, test and publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags) and deploy this code to the production environment at https://languageforge.org + +### Revert ### +Various tagged images are maintained in Docker Hub. If you need to revert to a previous version, you can do so by running the deployments scripts with the appropriate permissions or utilizing the Kubernetes UI to change the image of a deployment at any time. ### Backup/Restore ### Backups will be established automatically by LTOps and utilized by LF through the `storageClassName` property in a Persistent Volume Claim. This storage class provided by LTOps establishes both a frequency and retention for a backup. Any time a restoration is needed, the LF team will need to coordinate the effort with LTOps. The process of restoring from a point in time will require the application be brought down for maintenance. The process will roughly follow these steps: diff --git a/docker/Makefile b/docker/Makefile index 77d1de7967..752686c753 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -10,13 +10,13 @@ dev: start docker-compose up -d ui-builder .PHONY: e2e-tests -e2e-tests: build +e2e-tests: docker-compose build app-for-e2e test-e2e docker-compose restart app-for-e2e || docker-compose up -d app-for-e2e docker-compose run -e TEST_SPECS=$(TEST_SPECS) test-e2e .PHONY: e2e-tests-ci -e2e-tests-ci: build +e2e-tests-ci: docker-compose build app-for-e2e test-e2e # "-" means continue running commands even if they error (failed tests) -docker-compose run -e GITHUB_ACTIONS=1 --name e2etests test-e2e diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index eab952c17b..8282e9b5a1 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -53,6 +53,8 @@ FROM sillsdev/web-languageforge:base-php AS production-app RUN rm /usr/local/bin/install-php-extensions RUN apt-get remove -y gnupg2 RUN mv $PHP_INI_DIR/php.ini-production $PHP_INI_DIR/php.ini +# had to add /wait into prod image so the prod image could be run through E2E tests. +COPY --from=sillsdev/web-languageforge:wait-latest /wait /wait # DEVELOPMENT IMAGE FROM sillsdev/web-languageforge:base-php AS development-app @@ -62,8 +64,7 @@ RUN mv $PHP_INI_DIR/php.ini-development $PHP_INI_DIR/php.ini COPY --from=sillsdev/web-languageforge:wait-latest /wait /wait FROM ${ENVIRONMENT}-app AS languageforge-app -ARG BUILD_VERSION=9.9.9 -ENV BUILD_VERSION=${BUILD_VERSION} +ARG BUILD_VERSION=${BUILD_VERSION:-'9.9.9'} # copy app into image COPY src /var/www/html/ diff --git a/docker/deployment/.gitignore b/docker/deployment/.gitignore index 2de73f6da9..9259cfe2d8 100644 --- a/docker/deployment/.gitignore +++ b/docker/deployment/.gitignore @@ -1 +1,3 @@ *-current.yaml +all_projects +on_hold diff --git a/docker/deployment/Makefile b/docker/deployment/Makefile index 3a201e5bbd..6bb999818d 100644 --- a/docker/deployment/Makefile +++ b/docker/deployment/Makefile @@ -56,3 +56,11 @@ delete-app-assets: kubectl delete pvc lf-project-assets delete-app-sendreceive-data: kubectl delete pvc lfmerge-sendreceive-data + +APPPOD = $(shell kubectl get pods --selector='app=app' -o name | sed -e s'/pod\///') +lfmerge-copy-state: + rm -rf all_projects/*.state on_hold/*.state + kubectl cp $(APPPOD):/var/lib/languageforge/lexicon/sendreceive/state all_projects + grep -l HOLD all_projects/*.state | wc | awk '{printf $$1; }' && echo ' projects on HOLD' + mkdir -p on_hold + for f in `grep -l HOLD all_projects/*.state`; do mv $$f on_hold; done diff --git a/src/Site/views/languageforge/container/languageforge.html.twig b/src/Site/views/languageforge/container/languageforge.html.twig index 9b48bf6ad1..61aa16e198 100644 --- a/src/Site/views/languageforge/container/languageforge.html.twig +++ b/src/Site/views/languageforge/container/languageforge.html.twig @@ -81,7 +81,7 @@