Merge branch 'main' into scipy-ml-fixup #1355
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AUTO Build and Test | |
| env: | |
| DOCKER_CLIENT_TIMEOUT: "300" | |
| COMPOSE_HTTP_TIMEOUT: "300" | |
| on: | |
| push: | |
| paths: | |
| - "images/**" | |
| - "model/**" | |
| - "scripts/**" | |
| - "dodo.py" | |
| - ".github/workflows/main.yml" | |
| pull_request: | |
| branches: [ main ] | |
| paths: | |
| - "images/**" | |
| - "model/**" | |
| - "scripts/**" | |
| - "dodo.py" | |
| - ".github/workflows/main.yml" | |
| workflow_dispatch: | |
| jobs: | |
| start-runner: | |
| name: Start self-hosted EC2 runner | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| ec2-image-id: ami-079a0c9425d45e778 | |
| ec2-instance-type: t3.2xlarge | |
| subnet-id: subnet-0c4a81ef57cf2ebe3 | |
| security-group-id: sg-01cb8c0cb0de3ba00 | |
| docker-pipeline: | |
| needs: start-runner # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-runner.outputs.label }} | |
| if: > | |
| !contains(github.event.head_commit.message , 'skip ci') && | |
| !contains(github.event.pull_request.title, 'skip ci') | |
| steps: | |
| - name: Checkout after Free Space | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Docker/ENV cleanup Cleanup | |
| run: | | |
| docker image prune -a -f | |
| docker container prune -f | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Install pip dependencies | |
| run: | | |
| sudo dnf -y install python-pip | |
| sudo dnf remove python3-requests -y | |
| python3 -m pip install -r scripts/requirements.txt | |
| - name: Install selenium dependecies for r studio | |
| run: | | |
| chmod +x ./scripts/selenium_setup.sh | |
| ./scripts/selenium_setup.sh | |
| - name: Install pydoit fixed | |
| run: | | |
| git clone https://github.com/davidzyx/doit | |
| pip install ./doit | |
| rm -rf ./doit | |
| mkdir -p logs manifests | |
| touch logs/run.log | |
| - name: Set Python-specific environment variables | |
| run: | | |
| echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV | |
| echo "PYTHONUNBUFFERED=TRUE" >> $GITHUB_ENV | |
| - name: Setup artifacts | |
| env: | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USER: ${{ github.actor }} | |
| run: | | |
| doit prep | |
| mkdir -p logs | |
| - name: Clone Wiki | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{github.repository}}.wiki | |
| path: wiki | |
| - name: Build stack | |
| env: | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_USER: ${{ github.actor }} | |
| GRPC_CERT: ${{ secrets.GRPC_CERT }} | |
| PREPUSH_IMAGES: '["ghcr.io/ucsd-ets/scipy-ml-notebook"]' | |
| run: | | |
| python3 scripts/main.py | |
| - name: Check Free Space | |
| run: | | |
| echo "Free space:" | |
| df -h | |
| - name: Push Wiki to GitHub | |
| if: > | |
| !contains(github.base_ref , 'test') && | |
| !contains(github.base_ref , 'dev') && | |
| !contains(github.event.head_commit.message , 'skip push') && | |
| ( | |
| github.ref == 'refs/heads/main' | |
| ) | |
| uses: stefanzweifel/git-auto-commit-action@v4 | |
| with: | |
| commit_message: "[ci skip] Automated publish for ${{github.sha}}" | |
| repository: wiki/ | |
| - name: Archive artifacts and logs | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifacts | |
| path: | | |
| artifacts | |
| logs | |
| manifests | |
| stop-runner: | |
| name: Stop self-hosted EC2 runner | |
| needs: | |
| - start-runner # required to get output from the start-runner job | |
| - docker-pipeline # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} |