Rework GPU runtime system and copies #6632
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| push: | |
| branches: [ master ] | |
| jobs: | |
| build-linux-nix: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@v20 | |
| - uses: cachix/cachix-action@v12 | |
| with: | |
| name: futhark | |
| signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' | |
| - name: Build Futhark | |
| run: nix-build --argstr suffix nightly-linux-x86_64 --argstr commit $GITHUB_SHA | |
| - uses: actions/upload-artifact@v1 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| path: result/futhark-nightly-linux-x86_64.tar.xz | |
| build-linux-cabal: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Workaround runner image issue | |
| # https://github.com/actions/runner-images/issues/7061 | |
| run: sudo chown -R $USER /usr/local/.ghcup | |
| - uses: haskell/actions/setup@v2 | |
| with: | |
| ghc-version: '9.4.4' | |
| - uses: actions/cache@v3 | |
| name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle | |
| with: | |
| path: | | |
| ~/.cabal/packages | |
| ~/.cabal/store | |
| dist-newstyle | |
| key: ${{ runner.os }}-cabal-${{ hashFiles('futhark.cabal', 'cabal.project') }} | |
| - name: Build Futhark | |
| run: | | |
| cabal --version | |
| ghc --version | |
| make configure | |
| make build | |
| make install | |
| build-mac-cabal: | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| brew install cabal-install ghc@9 sphinx-doc | |
| echo "/usr/local/opt/sphinx-doc/bin" >> $GITHUB_PATH | |
| echo "/usr/local/opt/ghc@9/bin" >> $GITHUB_PATH | |
| - uses: actions/cache@v3 | |
| name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle | |
| with: | |
| path: | | |
| ~/.cabal/packages | |
| ~/.cabal/store | |
| dist-newstyle | |
| key: ${{ runner.os }}-cabal-${{ hashFiles('futhark.cabal', 'cabal.project') }} | |
| - name: Build | |
| run: | | |
| cp -r tools/release/skeleton futhark-nightly-macos-x86_64 | |
| mkdir -p futhark-nightly-macos-x86_64/bin | |
| cabal v2-update | |
| cabal install --install-method=copy --overwrite-policy=always --installdir=futhark-nightly-macos-x86_64/bin | |
| mkdir -p futhark-nightly-macos-x86_64/share/man/man1/ | |
| (cd docs; make man) | |
| cp -r docs/_build/man/* futhark-nightly-macos-x86_64/share/man/man1/ | |
| mkdir -p futhark-nightly-macos-x86_64/share/futhark | |
| cp LICENSE futhark-nightly-macos-x86_64/share/futhark/ | |
| echo "${GITHUB_SHA}" > futhark-nightly-macos-x86_64/commit-id | |
| tar -Jcf futhark-nightly-macos-x86_64.tar.xz futhark-nightly-macos-x86_64 | |
| - uses: actions/upload-artifact@v1 | |
| with: | |
| name: futhark-nightly-macos-x86_64.tar.xz | |
| path: futhark-nightly-macos-x86_64.tar.xz | |
| build-windows-cabal: | |
| runs-on: windows-2019 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: setup-haskell | |
| uses: haskell/actions/setup@v2 | |
| with: | |
| ghc-version: '9.4.4' | |
| - uses: actions/cache@v3 | |
| name: Cache cabal stuff | |
| with: | |
| path: | | |
| ${{ steps.setup-haskell.outputs.cabal-store }} | |
| dist-newstyle | |
| key: ${{ runner.os }}-cabal-${{ hashFiles('futhark.cabal', 'cabal.project') }} | |
| - name: Build | |
| shell: bash | |
| run: | | |
| cabal update | |
| try() { cabal install --install-method=copy --overwrite-policy=always --installdir=.; } | |
| try || try || try | |
| - uses: papeloto/[email protected] | |
| with: | |
| files: futhark.exe | |
| dest: futhark-nightly-windows-x86_64.zip | |
| - uses: actions/upload-artifact@v1 | |
| with: | |
| name: futhark-nightly-windows-x86_64.zip | |
| path: futhark-nightly-windows-x86_64.zip | |
| haskell-test-style: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@v20 | |
| - uses: actions/cache@v3 | |
| name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle | |
| with: | |
| path: | | |
| ~/.cabal/packages | |
| ~/.cabal/store | |
| dist-newstyle | |
| key: ${{ runner.os }}-${{ hashFiles('nix/sources.json') }}-style | |
| - name: Style check | |
| run: nix-shell --pure --run "make check" | |
| python-test-style: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@v20 | |
| - name: Style check | |
| run: nix-shell --pure --run "black --check ." | |
| - name: Type check | |
| run: nix-shell --pure --run "mypy ." | |
| # Fails mysteriously right now. | |
| # | |
| # build-docs: | |
| # runs-on: ubuntu-22.04 | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - name: Install Nix | |
| # uses: cachix/install-nix-action@v18 | |
| # - uses: actions/cache@v3 | |
| # name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle | |
| # with: | |
| # path: | | |
| # ~/.cabal/packages | |
| # ~/.cabal/store | |
| # dist-newstyle | |
| # key: ${{ runner.os }}-${{ hashFiles('nix/sources.json', 'futhark.cabal', 'cabal.project') }}-haddock | |
| # - name: Run haddock | |
| # run: | | |
| # nix-shell --pure --run "make configure" | |
| # nix-shell --pure --run "make docs" | |
| test-interpreter: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| futhark test -i tests | |
| test-c: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y python3-jsonschema | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| export CFLAGS="-fsanitize=undefined -fsanitize=address -fno-sanitize-recover -O" | |
| futhark test -c --backend=c tests --no-tuning | |
| make -C tests_lib/c -j | |
| test-multicore: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y python3-jsonschema | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| export CFLAGS="-fsanitize=undefined -fsanitize=address -fno-sanitize-recover -O" | |
| futhark test -c --backend=multicore tests --no-tuning | |
| make -C tests_lib/c FUTHARK_BACKEND=multicore | |
| test-ispc: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y python3-jsonschema | |
| - name: Install Nix | |
| uses: cachix/install-nix-action@v20 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| export CFLAGS="-fsanitize=undefined -fsanitize=address -fno-sanitize-recover -O" | |
| nix-shell --run 'futhark test -c --backend=ispc tests --no-tuning' | |
| nix-shell --run 'make -C tests_lib/c FUTHARK_BACKEND=ispc' | |
| test-python: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y python3-numpy | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| futhark test -c --no-terminal --no-tuning --backend=python --exclude=no_python --exclude=compiled tests | |
| make -C tests_lib/python -j | |
| test-oclgrind: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y opencl-headers oclgrind nvidia-cuda-dev nvidia-opencl-dev | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| futhark test tests -c --no-terminal --backend=opencl --exclude=compiled --cache-extension=cache --pass-option=--build-option=-O0 --runner=tools/oclgrindrunner.sh | |
| test-opencl: | |
| runs-on: opencl | |
| needs: [build-linux-nix] | |
| if: github.repository == 'diku-dk/futhark' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: | | |
| module load python/3.9.9 | |
| virtualenv virtualenv | |
| source virtualenv/bin/activate | |
| pip install jsonschema | |
| - uses: ./.github/actions/futhark-slurm | |
| with: | |
| script: | | |
| set -e | |
| futhark test tests \ | |
| --backend=opencl \ | |
| --cache-extension=cache | |
| source virtualenv/bin/activate | |
| FUTHARK_BACKEND=opencl make -C tests_lib/c -j | |
| slurm-options: -p gpu --time=0-01:00:00 --gres=gpu:1 --job-name=fut-opencl-test --exclude=hendrixgpu05fl,hendrixgpu06fl | |
| test-pyopencl: | |
| runs-on: opencl | |
| needs: [build-linux-nix] | |
| if: github.repository == 'diku-dk/futhark' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: | | |
| module load python/3.9.9 | |
| virtualenv virtualenv | |
| source virtualenv/bin/activate | |
| pip install numpy pyopencl jsonschema | |
| - uses: ./.github/actions/futhark-slurm | |
| with: | |
| script: | | |
| set -e | |
| source virtualenv/bin/activate | |
| futhark test tests --no-terminal --backend=pyopencl | |
| slurm-options: -p gpu --time=0-01:00:00 --gres=gpu:1 --job-name=fut-pyopencl-test --exclude=hendrixgpu05fl,hendrixgpu06fl | |
| test-cuda: | |
| runs-on: cuda | |
| needs: [build-linux-nix] | |
| if: github.repository == 'diku-dk/futhark' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: | | |
| module load python/3.9.9 | |
| virtualenv virtualenv | |
| source virtualenv/bin/activate | |
| pip install jsonschema | |
| - uses: ./.github/actions/futhark-slurm | |
| with: | |
| script: | | |
| set -e | |
| futhark test tests --no-terminal --backend=cuda --concurrency=8 | |
| source virtualenv/bin/activate | |
| FUTHARK_BACKEND=cuda make -C tests_lib/c -j | |
| slurm-options: -p gpu --time=0-01:00:00 --gres=gpu:1 --job-name=fut-cuda-test --exclude=hendrixgpu05fl,hendrixgpu06fl | |
| test-wasm: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: mymindstorm/setup-emsdk@v11 | |
| with: | |
| version: 2.0.18 | |
| actions-cache-folder: 'emsdk-cache' | |
| - uses: actions/[email protected] | |
| with: | |
| node-version: '16.x' | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Run tests | |
| run: | | |
| node --version | |
| export EMCFLAGS="-sINITIAL_MEMORY=2147418112 -O1" # 2gb - 64kb... largest value of memory | |
| futhark test \ | |
| -c \ | |
| --backend=wasm \ | |
| --runner=./tools/node-simd.sh \ | |
| --no-tuning \ | |
| --exclude=no_wasm tests | |
| test-wasm-multicore: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: mymindstorm/setup-emsdk@v11 | |
| with: | |
| version: 2.0.18 | |
| actions-cache-folder: 'emsdk-cache' | |
| - uses: actions/[email protected] | |
| with: | |
| node-version: '16.x' | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Run tests | |
| run: | | |
| node --version | |
| export EMCFLAGS="-sINITIAL_MEMORY=2147418112 -O1 -s PTHREAD_POOL_SIZE=12" # 2gb - 64kb... largest value of memory | |
| futhark test -c --backend=wasm-multicore --runner=./tools/node-threaded.sh --no-tuning --exclude=no_wasm tests | |
| test-wasm-lib: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: mymindstorm/setup-emsdk@v11 | |
| with: | |
| version: 2.0.18 | |
| actions-cache-folder: 'emsdk-cache' | |
| - uses: actions/[email protected] | |
| with: | |
| node-version: '16.x' | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Run tests | |
| run: | | |
| make -C tests_lib/javascript | |
| test-ir-parser: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| tools/testparser.sh tests | |
| test-tools: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix] | |
| steps: | |
| - name: Install OS dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ffmpeg | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - name: Install from nightly tarball | |
| run: | | |
| tar xvf futhark-nightly-linux-x86_64.tar.xz | |
| make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - run: | | |
| cd tests_pkg && sh test.sh | |
| - run: | | |
| cd tests_literate && sh test.sh | |
| - run: | | |
| cd tests_repl && sh test.sh | |
| - run: | | |
| cd tests_bench && sh test.sh | |
| - run: | | |
| cd tests_adhoc && sh test.sh | |
| - run: | | |
| futhark doc -o prelude-docs /dev/null | |
| tar -Jcf prelude-docs.tar.xz prelude-docs | |
| - uses: actions/upload-artifact@v1 | |
| with: | |
| name: prelude-docs.tar.xz | |
| path: prelude-docs.tar.xz | |
| deploy-nightly: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-linux-nix, build-mac-cabal, build-windows-cabal, test-tools] | |
| if: github.ref == 'refs/heads/master' | |
| steps: | |
| - name: Install SSH key | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.SSHKEY }} | |
| known_hosts: ${{ secrets.KNOWN_HOSTS }} | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-windows-x86_64.zip | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-macos-x86_64.tar.xz | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: futhark-nightly-linux-x86_64.tar.xz | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: prelude-docs.tar.xz | |
| - name: scp tarballs to server | |
| run: | | |
| scp -o StrictHostKeyChecking=no futhark-nightly-windows-x86_64.zip futhark-nightly-linux-x86_64.tar.xz futhark-nightly-macos-x86_64.tar.xz [email protected]:/var/www/htdocs/futhark-lang.org/releases | |
| - name: copy docs to server | |
| run: | | |
| tar -xf prelude-docs.tar.xz | |
| rsync -rv -e 'ssh -o "StrictHostKeyChecking no"' prelude-docs/* [email protected]:/var/www/htdocs/futhark-lang.org/docs/prelude/ | |
| - name: make nightly release | |
| uses: "marvinpinto/[email protected]" | |
| with: | |
| repo_token: "${{ secrets.GITHUB_TOKEN }}" | |
| automatic_release_tag: "nightly" | |
| prerelease: true | |
| title: "nightly" | |
| files: | | |
| futhark-nightly-windows-x86_64.zip | |
| futhark-nightly-linux-x86_64.tar.xz | |
| futhark-nightly-macos-x86_64.tar.xz |