v1-nightly.yml

name: TorchBench V1 nightly
on:
  workflow_dispatch:
  schedule:
    - cron: '0 14 * * *' # run at 2 PM UTC

jobs:
  run-benchmark:
    env:
      TORCHBENCH_VER: "v1.1"
      CONFIG_VER: "v1"
      CONDA_ENV_NAME:  "torchbench-v1-nightly-ci"
      OUTPUT_DIR: ".torchbench/v1-nightly-ci"
      BISECTION_ROOT: ".torchbench/v1-bisection-ci"
      CUDA_VERSION: "cu102"
      SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      IS_GHA: 1
      AWS_DEFAULT_REGION: us-east-1
      BUILD_ENVIRONMENT: benchmark-nightly
    if: ${{ github.repository_owner == 'pytorch' }}
    runs-on: [self-hosted, bm-runner]
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          ref: v1.0
      - name: Create conda env
        run: |
          conda create -y -q --name "${CONDA_ENV_NAME}" python=3.7
      - name: Install PyTorch nightly
        run: |
          . activate "${CONDA_ENV_NAME}"
          # Install dependencies
          pip install requests bs4 argparse gitpython boto3
          # Check if nightly builds are available
          NIGHTLIES=$(python torchbenchmark/util/torch_nightly.py --packages torch)
          # If failed, the script will generate empty result
          if [ -z $NIGHTLIES ]; then
              echo "Torch nightly build failed. Cancel the workflow."
              exit 1
          fi
          # Install PyTorch nightly from pip
          pip install --pre torch torchtext torchvision \
          -f https://download.pytorch.org/whl/nightly/${CUDA_VERSION}/torch_nightly.html
      - name: Install other TorchBench dependencies
        run: |
          . activate "${CONDA_ENV_NAME}"
          python install.py
      - name: Run benchmark
        run: |
          . activate "${CONDA_ENV_NAME}"
          WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}"
          bash ./.github/scripts/run.sh "${WORKFLOW_HOME}"
      - name: Generate the bisection config
        run: |
          set -x
          . activate "${CONDA_ENV_NAME}"
          WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}"
          mkdir -p benchmark-output/
          # Update the self-hosted pytorch version
          pushd "${HOME}/pytorch"
          git fetch origin
          popd
          pip install gitpython pyyaml dataclasses argparse pathlib
          # Compare the result from yesterday and report any perf signals
          python ./.github/scripts/generate-abtest-config.py \
                 --pytorch-dir "${HOME}/pytorch" \
                 --github-issue "${WORKFLOW_HOME}/gh-issue.md" \
                 --benchmark-dir "${WORKFLOW_HOME}" \
                 --out "${WORKFLOW_HOME}/bisection.yaml"
          # Include in the GitHub artifact
          if [ -f "${WORKFLOW_HOME}/gh-issue.md" ]; then
            cp "${WORKFLOW_HOME}/bisection.yaml" ./benchmark-output/
            cp "${WORKFLOW_HOME}/gh-issue.md" ./benchmark-output/
            # Setup the bisection environment
            BISECTION_HOME="${HOME}/${{ env.BISECTION_ROOT }}/bisection-gh${GITHUB_RUN_ID}"
            mkdir -p "${BISECTION_HOME}"
            cp ./benchmark-output/bisection.yaml "${BISECTION_HOME}/config.yaml"
          fi
      - name: Dispatch the v1-bisection workflow
        if: env.TORCHBENCH_PERF_SIGNAL
        run: |
          # Get the workflow ID from
          # https://api.github.com/repos/pytorch/benchmark/actions/workflows
          curl -u xuzhao9:${{ secrets.TORCHBENCH_ACCESS_TOKEN }} \
            -X POST \
            -H "Accept: application/vnd.github.v3+json" \
             https://api.github.com/repos/pytorch/benchmark/actions/workflows/10693192/dispatches \
            -d '{"ref": "main", "inputs": {"issue_name": "bisection-gh'"${GITHUB_RUN_ID}"'" } }'
      - name: Create the github issue
        if: env.TORCHBENCH_PERF_SIGNAL
        uses: peter-evans/create-issue-from-file@v3
        with:
          title: Performance Signal Detected by TorchBench CI on ${{ env.TORCHBENCH_PERF_SIGNAL }}
          content-filepath: ./benchmark-output/gh-issue.md
          labels: |
            torchbench-perf-report
      - name: Copy artifact and upload to scribe
        run: |
          . activate "${CONDA_ENV_NAME}"
          LATEST_RESULT=$(find ${HOME}/${OUTPUT_DIR}/gh${GITHUB_RUN_ID} -name "*.json" | sort -r | head -1)
          echo "Benchmark result file: $LATEST_RESULT"
          TODAY=$(date "+%Y%m%d%H%M%S")
          CONFIG_DIR=torchbenchmark/score/configs/${CONFIG_VER}
          CONFIG_ENV=${CONFIG_DIR}/config-${CONFIG_VER}.env
          # Load environment variables
          set -a; source "${CONFIG_ENV}"; set +a
          CONFIG_NORM_FILE="${CONFIG_DIR}/${CONFIG_FILE}"
          SCORE_FILE="./benchmark-result-v1-score-${TODAY}.json"
          # Generate score file
          python compute_score.py --score_version v1 --benchmark_data_file "${LATEST_RESULT}" > "${SCORE_FILE}"
          # Enable OSS stat uploads
          wget -O scripts/scribe.py https://github.com/pytorch/pytorch/raw/master/tools/stats/scribe.py || echo "failed to copy oss stat utils"
          # Upload result to Scribe
          python scripts/upload_scribe.py --pytest_bench_json "${LATEST_RESULT}" --torchbench_score_file "${SCORE_FILE}"
          mkdir -p benchmark-output
          cp "${LATEST_RESULT}" ./benchmark-output/benchmark-result-v1-${TODAY}.json
      - name: Upload artifact
        uses: actions/upload-artifact@v2
        with:
          name: Benchmark result
          path: benchmark-output/
      - name: Destroy conda env
        run: |
          conda env remove --name "${CONDA_ENV_NAME}"