Skip to content

fix: add and update fields in parquet export (#10) #35

fix: add and update fields in parquet export (#10)

fix: add and update fields in parquet export (#10) #35

name: Docker Compose Deployment CI
on:
push:
branches:
- main
- deploy-*
tags:
- v*.*.*
jobs:
deploy:
runs-on: ubuntu-latest
strategy:
matrix:
env:
- ${{ startsWith(github.ref, 'refs/tags/v') && 'off-exports-org' || 'off-exports-net' }}
environment: ${{ matrix.env }}
concurrency: ${{ matrix.env }}
steps:
- name: Set various variable for staging deployment
if: matrix.env == 'off-exports-net'
run: |
# configurations
echo "ENVIRONMENT=preprod" >> $GITHUB_ENV
echo "ENABLE_HF_PUSH=0" >> $GITHUB_ENV
echo "ENABLE_S3_PUSH=0" >> $GITHUB_ENV
# deploy target
echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
echo "SSH_USERNAME=off" >> $GITHUB_ENV
echo "SSH_PROXY_USERNAME=off" >> $GITHUB_ENV
echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
- name: Set various variable for production deployment
if: matrix.env == 'off-exports-org'
run: |
# configurations
echo "ENVIRONMENT=prod" >> $GITHUB_ENV
echo "ENABLE_HF_PUSH=1" >> $GITHUB_ENV
echo "ENABLE_S3_PUSH=0" >> $GITHUB_ENV
# deploy target
echo "SSH_PROXY_HOST=45.147.209.254" >> $GITHUB_ENV
echo "SSH_USERNAME=off" >> $GITHUB_ENV
echo "SSH_PROXY_USERNAME=off" >> $GITHUB_ENV
echo "SSH_PROTOCOL=tcp" >> $GITHUB_ENV
echo "SSH_HOST=10.3.0.200" >> $GITHUB_ENV
- name: Wait for container build workflow
uses: tomchv/[email protected]
id: wait-build
with:
token: ${{ secrets.GITHUB_TOKEN }}
checkName: build
ref: ${{ github.event.pull_request.head.sha || github.sha }}
intervalSeconds: 10
timeoutSeconds: 600 # 10m
- name: Do something if build isn't launched
if: steps.wait-build.outputs.conclusion == 'not found'
run: echo job does not exist && true
- name: Do something if build fail
if: steps.wait-build.outputs.conclusion == 'failure'
run: echo fail && false # fail if build fail
- name: Do something if build timeout
if: steps.wait-build.outputs.conclusion == 'timed_out'
run: echo Timeout && false # fail if build time out
- name: Checkout git repository
uses: appleboy/ssh-action@master
with:
host: ${{ env.SSH_HOST }}
username: ${{ env.SSH_USERNAME }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
proxy_host: ${{ env.SSH_PROXY_HOST }}
proxy_username: ${{ env.SSH_PROXY_USERNAME }}
proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
script_stop: false
script: |
# Clone Git repository if not already there
[ ! -d '${{ matrix.env }}' ] && git clone --depth 1 https://github.com/${{ github.repository }} ${{ matrix.env }} --no-single-branch 2>&1
# Go to repository directory
cd ${{ matrix.env }}
# Fetch newest commits (in case it wasn't freshly cloned)
git fetch --depth 1
# Checkout current commit SHA
git checkout -qf ${{ github.sha }}
- name: Set environment variables
uses: appleboy/ssh-action@master
with:
host: ${{ env.SSH_HOST }}
username: ${{ env.SSH_USERNAME }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
proxy_host: ${{ env.SSH_PROXY_HOST }}
proxy_username: ${{ env.SSH_PROXY_USERNAME }}
proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
script_stop: false
script: |
# Go to repository directory
cd ${{ matrix.env }}
# new env
rm .env
touch .env
# Set Docker Compose variables
echo "DOCKER_CLIENT_TIMEOUT=120" >> .env
echo "COMPOSE_HTTP_TIMEOUT=120" >> .env
echo "COMPOSE_PROJECT_NAME=off-exports" >> .env
echo "COMPOSE_PATH_SEPARATOR=;" >> .env
echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env
echo "TAG=sha-${{ github.sha }}" >> .env
echo "RESTART_POLICY=always" >> .env
# App environment variables
echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env
echo "REDIS_HOST=redis" >> .env
echo "NUM_RQ_WORKERS=4" >> .env
echo "ENABLE_HF_PUSH=${{ env.ENABLE_HF_PUSH }}" >> .env
echo "ENABLE_S3_PUSH=${{ env.ENABLE_S3_PUSH }}" >> .env
# Secrets
echo "SENTRY_DSN=${{ secrets.SENTRY_DSN }}" >> .env
echo "HF_TOKEN=${{ secrets.HF_TOKEN }}" >> .env
echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> .env
echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> .env
- name: Create Docker volumes
uses: appleboy/ssh-action@master
with:
host: ${{ env.SSH_HOST }}
username: ${{ env.SSH_USERNAME }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
proxy_host: ${{ env.SSH_PROXY_HOST }}
proxy_username: ${{ env.SSH_PROXY_USERNAME }}
proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
script_stop: false
script: |
cd ${{ matrix.env }}
make create_external_volumes
- name: Start services
uses: appleboy/ssh-action@master
with:
host: ${{ env.SSH_HOST }}
username: ${{ env.SSH_USERNAME }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
proxy_host: ${{ env.SSH_PROXY_HOST }}
proxy_username: ${{ env.SSH_PROXY_USERNAME }}
proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
script_stop: false
script: |
cd ${{ matrix.env }}
make up
- name: Cleanup obsolete Docker objects
uses: appleboy/ssh-action@master
if: ${{ always() }}
with:
host: ${{ env.SSH_HOST }}
username: ${{ env.SSH_USERNAME }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
proxy_host: ${{ env.SSH_PROXY_HOST }}
proxy_username: ${{ env.SSH_PROXY_USERNAME }}
proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
script_stop: false
script: |
cd ${{ matrix.env }}
make prune