main.yml

name: AUTO Build and Test

env:
  DOCKER_CLIENT_TIMEOUT: "300"
  COMPOSE_HTTP_TIMEOUT: "300"

on:
  push:
    paths:
      - "images/**"
      - "model/**"
      - "scripts/**"
      - "dodo.py"
      - ".github/workflows/main.yml"

  pull_request:
    branches: [ main ]
    paths:
      - "images/**"
      - "model/**"
      - "scripts/**"
      - "dodo.py"
      - ".github/workflows/main.yml"

  workflow_dispatch:

jobs:
  start-runner:
    name: Start self-hosted EC2 runner
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Start EC2 runner
        id: start-ec2-runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          ec2-image-id: ami-079a0c9425d45e778
          ec2-instance-type: t3.xlarge
          subnet-id: subnet-0c4a81ef57cf2ebe3
          security-group-id: sg-01cb8c0cb0de3ba00
  
  docker-pipeline:
    needs: start-runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-runner.outputs.label }}
    if: >
      !contains(github.event.head_commit.message , 'skip ci') &&
      !contains(github.event.pull_request.title, 'skip ci')
    steps:
      - name: Checkout after Free Space
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Check Free Space 0
        run: |
          echo "Free space:"
          df -h

      - name: Docker/ENV cleanup Cleanup
        run: |
          docker image prune -a -f
          docker container prune -f
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: Check Free Space 1
        run: |
          echo "Free space:"
          df -h

      - name: Install pip dependencies
        run: |
          sudo yum -y install python-pip
          python3 -m pip install -r scripts/requirements.txt
      
      - name: Install selenium dependecies for r studio
        run: |
          chmod +x ./scripts/selenium_setup.sh 
          ./scripts/selenium_setup.sh
          
      - name: Install pydoit fixed
        run: |
          git clone https://github.com/davidzyx/doit
          pip install ./doit
          rm -rf ./doit
          mkdir -p logs manifests
          touch logs/run.log

      - name: Set Python-specific environment variables
        run: |
          echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV
          echo "PYTHONUNBUFFERED=TRUE" >> $GITHUB_ENV
      
      - name: Setup artifacts
        env:
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_USER: ${{ github.actor }}
        run: |
          doit prep
          mkdir -p logs

      - name: Clone Wiki
        uses: actions/checkout@v3
        with:
          repository: ${{github.repository}}.wiki
          path: wiki

      - name: Check Free Space 2
        run: |
          echo "Free space:"
          df -h
      
      - name: Build stack
        env:
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_USER: ${{ github.actor }}
          GRPC_CERT: ${{ secrets.GRPC_CERT }}
          PREPUSH_IMAGES: '["ghcr.io/ucsd-ets/scipy-ml-notebook"]'
        run: |
          python3 scripts/main.py

      - name: Check Free Space 3
        run: |
          echo "Free space:"
          df -h

      - name: Push Wiki to GitHub
        if: >
          !contains(github.base_ref , 'test') &&
          !contains(github.base_ref , 'dev') &&
          !contains(github.event.head_commit.message , 'skip push') &&
          (
            github.ref == 'refs/heads/main'
          )
        uses: stefanzweifel/git-auto-commit-action@v4
        with:
          commit_message: "[ci skip] Automated publish for ${{github.sha}}"
          repository: wiki/

      - name: Archive artifacts and logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v3
        with:
          name: build-artifacts
          path: |
            artifacts
            logs
            manifests
  stop-runner:
    name: Stop self-hosted EC2 runner
    needs:
      - start-runner # required to get output from the start-runner job
      - docker-pipeline # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}