Update main.yml #1108
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AUTO Build and Test | |
| env: | |
| DOCKER_CLIENT_TIMEOUT: "300" | |
| COMPOSE_HTTP_TIMEOUT: "300" | |
| on: | |
| push: | |
| paths: | |
| - "images/**" | |
| - "model/**" | |
| - "scripts/**" | |
| - "dodo.py" | |
| - ".github/workflows/main.yml" | |
| pull_request: | |
| branches: [ main ] | |
| paths: | |
| - "images/**" | |
| - "model/**" | |
| - "scripts/**" | |
| - "dodo.py" | |
| - ".github/workflows/main.yml" | |
| workflow_dispatch: | |
| jobs: | |
| start-runner: | |
| name: Start self-hosted EC2 runner | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ami-079a0c9425d45e778 | |
| ec2-instance-type: t3.xlarge | |
| subnet-id: subnet-0c4a81ef57cf2ebe3 | |
| security-group-id: sg-01cb8c0cb0de3ba00 | |
| docker-pipeline: | |
| needs: start-runner # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-runner.outputs.label }} | |
| if: > | |
| !contains(github.event.head_commit.message , 'skip ci') && | |
| !contains(github.event.pull_request.title, 'skip ci') | |
| steps: | |
| - name: Checkout after Free Space | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check Free Space 0 | |
| run: | | |
| echo "Free space:" | |
| df -h | |
| - name: Docker/ENV cleanup Cleanup | |
| run: | | |
| docker image prune -a -f | |
| docker container prune -f | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Check Free Space 1 | |
| run: | | |
| echo "Free space:" | |
| df -h | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.8 | |
| - name: Install pip dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r scripts/requirements.txt | |
| - name: Install selenium dependecies for r studio | |
| run: | | |
| chmod +x ./scripts/selenium_setup.sh | |
| ./scripts/selenium_setup.sh | |
| - name: Install pydoit fixed | |
| run: | | |
| git clone https://github.com/davidzyx/doit | |
| pip install ./doit | |
| rm -rf ./doit | |
| mkdir -p logs manifests | |
| touch logs/run.log | |
| - name: Set Python-specific environment variables | |
| run: | | |
| echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV | |
| echo "PYTHONUNBUFFERED=TRUE" >> $GITHUB_ENV | |
| - name: Setup artifacts | |
| env: | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USER: ${{ github.actor }} | |
| run: | | |
| doit prep | |
| mkdir -p logs | |
| - name: Clone Wiki | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{github.repository}}.wiki | |
| path: wiki | |
| - name: Check Free Space 2 | |
| run: | | |
| echo "Free space:" | |
| df -h | |
| - name: Build stack | |
| env: | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USER: ${{ github.actor }} | |
| GRPC_CERT: ${{ secrets.GRPC_CERT }} | |
| PREPUSH_IMAGES: '["ghcr.io/ucsd-ets/scipy-ml-notebook"]' | |
| run: | | |
| python3 scripts/main.py | |
| - name: Check Free Space 3 | |
| run: | | |
| echo "Free space:" | |
| df -h | |
| - name: Push Wiki to GitHub | |
| if: > | |
| !contains(github.base_ref , 'test') && | |
| !contains(github.base_ref , 'dev') && | |
| !contains(github.event.head_commit.message , 'skip push') && | |
| ( | |
| github.ref == 'refs/heads/main' | |
| ) | |
| uses: stefanzweifel/git-auto-commit-action@v4 | |
| with: | |
| commit_message: "[ci skip] Automated publish for ${{github.sha}}" | |
| repository: wiki/ | |
| - name: Archive artifacts and logs | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifacts | |
| path: | | |
| artifacts | |
| logs | |
| manifests | |
| stop-runner: | |
| name: Stop self-hosted EC2 runner | |
| needs: | |
| - start-runner # required to get output from the start-runner job | |
| - docker-pipeline # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} |