userbenchmark-a100.yml

name: TorchBench Userbenchmark on A100
on:
  schedule:
    - cron: '0 14 * * *' # run at 2 PM UTC
  workflow_dispatch:
    inputs:
      userbenchmark_name:
        description: "Name of the user benchmark to run"
      userbenchmark_options:
        description: "Option of the user benchmark to run"
env:
  PYTHON_VERSION: "3.8"
  TENSORRT_PYTHON_VERSION: "cp38"
  TENSORRT_VERSION: "8.2.4.2"
  CUDA_VERSION: "cu116"
  CONDA_ENV_NAME: "userbenchmarks-ci"
  MAGMA_VERSION: "magma-cuda116"
  PLATFORM_NAME: "gcp_a100"
  TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN }}
  SETUP_SCRIPT: "/workspace/setup_instance.sh"
jobs:
  run-userbenchmark:
    runs-on: [self-hosted, a100-runner]
    timeout-minutes: 1440 # 24 hours
    steps:
      - name: Checkout TorchBench
        uses: actions/checkout@v3
        with:
          path: benchmark
      - name: Create conda environment
        run: |
          . "${SETUP_SCRIPT}"
          conda create -y -q --name "${CONDA_ENV_NAME}" python="${PYTHON_VERSION}"
      - name: Tune Nvidia GPU
        run: |
          . "${SETUP_SCRIPT}"
          sudo LD_LIBRARY_PATH=/usr/local/nvidia/lib64:$LD_LIBRARY_PATH nvidia-smi -pm 1
          sudo LD_LIBRARY_PATH=/usr/local/nvidia/lib64:$LD_LIBRARY_PATH nvidia-smi -ac 1215,1410
          nvidia-smi
      - name: Install PyTorch nightly
        run: |
          . "${SETUP_SCRIPT}" && conda activate "${CONDA_ENV_NAME}"
          pushd benchmark
          # Install dependencies
          conda install -y numpy requests ninja pyyaml setuptools gitpython beautifulsoup4 regex
          conda install -y expecttest libglib pango -c conda-forge
          conda install -y -c pytorch "${MAGMA_VERSION}"
          pip install requests bs4 argparse gitpython boto3 pyyaml numpy regex psutil
          # Check if nightly builds are available
          NIGHTLIES=$(python torchbenchmark/util/torch_nightly.py --packages torch)
          # If failed, the script will generate empty result
          if [ -z $NIGHTLIES ]; then
              echo "Torch nightly build failed. Cancel the workflow."
              exit 1
          fi
          # Install PyTorch nightly and domain libraries from pip
          pip install --pre torch torchtext torchvision torchaudio \
            -f https://download.pytorch.org/whl/nightly/${CUDA_VERSION}/torch_nightly.html
          # make sure pytorch+cuda works
          python -c "import torch; torch.cuda.init()"
      - name: Install TorchBench
        run: |
          set -x
          . "${SETUP_SCRIPT}" && conda activate "${CONDA_ENV_NAME}"
          pushd benchmark
          python install.py
      - name: Run user benchmark
        run: |
          set -x
          . "${SETUP_SCRIPT}" && conda activate "${CONDA_ENV_NAME}"
          # remove old results
          if [ -d benchmark-output ]; then rm -Rf benchmark-output; fi
          pushd benchmark
          if [ -d .userbenchmark ]; then rm -Rf .userbenchmark; fi
          MANUAL_WORKFLOW="${{ github.event.inputs.userbenchmark_name }}"
          if [ -z "${MANUAL_WORKFLOW}" ]; then
            # Figure out what userbenchmarks we should run, and run it
            python ./.github/scripts/userbenchmark/schedule-benchmarks.py --platform ${PLATFORM_NAME}
            cp -r ./.userbenchmark ../benchmark-output
          else
            python run_benchmark.py "${{ github.event.inputs.userbenchmark_name }}" ${{ github.event.inputs.userbenchmark_options }}
            cp -r ./.userbenchmark/"${{ github.event.inputs.userbenchmark_name }}" ../benchmark-output
          fi
      - name: Upload result jsons to Scribe
        run: |
          . "${SETUP_SCRIPT}" && conda activate "${CONDA_ENV_NAME}"
          pushd benchmark
          RESULTS=($(find ${PWD}/../benchmark-output -name "metrics-*.json" -maxdepth 2 | sort -r))
          echo "Uploading result jsons: ${RESULTS}"
          for r in ${RESULTS[@]}; do
            python ./scripts/userbenchmark/upload_scribe.py --userbenchmark_json "${r}" --userbenchmark_platform "${PLATFORM_NAME}"
          done
      - name: Upload artifact
        uses: actions/upload-artifact@v2
        with:
          name: TorchBench result
          path: benchmark-output/
      - name: Remove conda environment
        run: |
          . "${SETUP_SCRIPT}"
          conda env remove --name "${CONDA_ENV_NAME}"