main.yml

name: CI
on:
  push:
    branches: [main]
    tags-ignore: [dev]
    paths-ignore:
      - 'meetings/**'
  pull_request:
    branches: [main]
    paths-ignore:
      - 'meetings/**'
defaults:
  run:
    shell: bash

# Cancel any in-flight jobs for the same PR/branch so there's only one active
# at a time
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Check Code style quickly by running `rustfmt` over all code
  rustfmt:
    name: Rustfmt
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - uses: ./.github/actions/install-rust
    - run: rustup component add rustfmt
    - run: cargo fmt --all -- --check

  # Lint dependency graph for security advisories, duplicate versions, and
  # incompatible licences
  cargo_deny:
    name: Cargo deny
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - uses: ./.github/actions/install-rust
    - run: |
        set -e
        curl -L https://github.com/EmbarkStudios/cargo-deny/releases/download/0.8.5/cargo-deny-0.8.5-x86_64-unknown-linux-musl.tar.gz | tar xzf -
        mv cargo-deny-*-x86_64-unknown-linux-musl/cargo-deny cargo-deny
        echo `pwd` >> $GITHUB_PATH
    - run: cargo deny check

  doc:
    name: Doc build
    runs-on: ubuntu-latest
    env:
      CARGO_MDBOOK_VERSION: 0.4.8
      RUSTDOCFLAGS: -Dbroken_intra_doc_links --cfg nightlydoc
      OPENVINO_SKIP_LINKING: 1
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - uses: ./.github/actions/install-rust
      with:
        toolchain: nightly-2021-07-18

    # Build C API documentation
    - run: sudo apt-get update -y && sudo apt-get install -y libclang1-9 libclang-cpp9
    - run: curl -L https://doxygen.nl/files/doxygen-1.9.1.linux.bin.tar.gz | tar xzf -
    - run: echo "`pwd`/doxygen-1.9.1/bin" >> $GITHUB_PATH
    - run: cd crates/c-api && doxygen doxygen.conf

    # install mdbook, build the docs, and test the docs
    - uses: actions/cache@v2
      with:
        path: ${{ runner.tool_cache }}/mdbook
        key: cargo-mdbook-bin-${{ env.CARGO_MDBOOK_VERSION }}
    - run: |
        echo "${{ runner.tool_cache }}/mdbook/bin" >> $GITHUB_PATH
        cargo install --root ${{ runner.tool_cache }}/mdbook --version ${{ env.CARGO_MDBOOK_VERSION }} mdbook
    - run: (cd docs && mdbook build)
    - run: cargo build -p wasmtime-wasi --features wasmtime/wat,wasmtime/cranelift
    - run: (cd docs && mdbook test -L ../target/debug/deps)

    # Build Rust API documentation
    - run: |
        cargo doc --no-deps --workspace \
          --exclude wasmtime-cli \
          --exclude test-programs \
          --exclude cranelift-codegen-meta \
          --exclude 'peepmatic*'
    - run: cargo doc --package cranelift-codegen-meta --document-private-items

    # Assemble the documentation, and always upload it as an artifact for
    # inspection on PRs and such.
    - run: |
        mv docs/book gh-pages
        mv crates/c-api/html gh-pages/c-api
        mv target/doc gh-pages/api
        tar czf gh-pages.tar.gz gh-pages
    - uses: actions/upload-artifact@v2
      with:
        name: gh-pages
        path: gh-pages.tar.gz

    # If this is a push to the main branch push to the `gh-pages` using a
    # deploy key. Note that a deploy key is necessary for now because otherwise
    # using the default token for github actions doesn't actually trigger a page
    # rebuild.
    - name: Push to gh-pages
      run: curl -LsSf https://git.io/fhJ8n | rustc - && (cd gh-pages && ../rust_out)
      env:
        GITHUB_DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
        BUILD_REPOSITORY_ID: ${{ github.repository }}
        BUILD_SOURCEVERSION: ${{ github.sha }}
      if: github.event_name == 'push' && github.ref == 'refs/heads/main'

  # Quick checks of various feature combinations and whether things
  # compile. The goal here isn't to run tests, mostly just serve as a
  # double-check that Rust code compiles and is likely to work everywhere else.
  checks:
    name: Check
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - uses: ./.github/actions/install-rust

    # Check some feature combinations of the `wasmtime` crate
    - run: cargo check -p wasmtime --no-default-features
    - run: cargo check -p wasmtime --no-default-features --features wat
    - run: cargo check -p wasmtime --no-default-features --features lightbeam
    - run: cargo check -p wasmtime --no-default-features --features jitdump
    - run: cargo check -p wasmtime --no-default-features --features vtune
    - run: cargo check -p wasmtime --no-default-features --features cache
    - run: cargo check -p wasmtime --no-default-features --features async
    - run: cargo check -p wasmtime --no-default-features --features uffd
    - run: cargo check -p wasmtime --no-default-features --features cranelift
    - run: cargo check -p wasmtime --no-default-features --features cranelift,wat,async,cache

    # Check some feature combinations of the `wasmtime-c-api` crate
    - run: cargo check -p wasmtime-c-api --no-default-features
    - run: cargo check -p wasmtime-c-api --no-default-features --features wat
    - run: cargo check -p wasmtime-c-api --no-default-features --features wasi

    # Check a few builds of the cranelift backend
    # - only x86 backend support,
    # - only arm64 backend support,
    # - experimental arm32 support,
    # - no debug_assertions.
    - run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util --no-default-features --features=cranelift-codegen/arm64
    - run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util --no-default-features --features=cranelift-codegen/x86
    - run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util --no-default-features --features=cranelift-codegen/arm32
    - run: cargo check --manifest-path=./cranelift/Cargo.toml --bin clif-util
      env:
        CARGO_PROFILE_DEV_DEBUG_ASSERTIONS: false

    # Check whether `crates/wasi-common` cross-compiles to the following targets:
    # * wasm32-unknown-emscripten
    # * armv7-unknown-linux-gnueabihf
    - run: |
        rustup target add wasm32-unknown-emscripten
        rustup target add armv7-unknown-linux-gnueabihf
    - run: cargo check --target wasm32-unknown-emscripten -p wasi-common
    - run: cargo check --target armv7-unknown-linux-gnueabihf -p wasi-common

    # Check that codegen and wasm crates typecheck on 1.43.0; this is required
    # for Firefox.
    - run: rustup install 1.43.0
    - run: cargo +1.43.0 check -p cranelift-codegen
    - run: cargo +1.43.0 check -p cranelift-wasm

  fuzz_targets:
    name: Fuzz Targets
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    # Note that building with fuzzers requires nightly since it uses unstable
    # flags to rustc.
    - uses: ./.github/actions/install-rust
      with:
        toolchain: nightly-2021-07-18
    - run: cargo install cargo-fuzz --vers "^0.8"
    # Install OCaml packages necessary for 'differential_spec' fuzz target.
    - run: sudo apt install -y ocaml-nox ocamlbuild
    - run: cargo fetch
      working-directory: ./fuzz
    - run: cargo fuzz build --dev

  rebuild_peephole_optimizers:
    name: Rebuild Peephole Optimizers
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - run: rustup update stable && rustup default stable
    - name: Test `peepmatic`
      run: cargo test --package 'peepmatic*'
    - name: Rebuild Peepmatic-based peephole optimizers
      run: |
        cargo test \
          --features 'enable-peepmatic rebuild-peephole-optimizers' \
          peepmatic
      working-directory: ./cranelift/codegen
    - name: Upload rebuilt peephole optimizers
      uses: actions/upload-artifact@v2
      with:
        name: peephole-optimizers
        path: cranelift/codegen/src/preopt.serialized
    - name: Check that built peephole optimizers are up to date
      run: git diff --exit-code
    - name: Test with Peepmatic-based peephole optimizers
      run: cargo test --features 'enable-peepmatic'
      working-directory: ./cranelift

  # Perform all tests (debug mode) for `wasmtime`. This runs stable/beta/nightly
  # channels of Rust as well as macOS/Linux/Windows.
  test:
    name: Test
    runs-on: ${{ matrix.os }}
    env:
      QEMU_BUILD_VERSION: 6.0.0
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
          - os: macos-latest
          - os: windows-latest
          - os: windows-latest
            target: x86_64-pc-windows-gnu
          - os: ubuntu-latest
            target: aarch64-unknown-linux-gnu
            gcc_package: gcc-aarch64-linux-gnu
            gcc: aarch64-linux-gnu-gcc
            qemu: qemu-aarch64 -L /usr/aarch64-linux-gnu
            qemu_target: aarch64-linux-user
            # FIXME(#3183) shouldn't be necessary to specify this
            qemu_flags: -cpu cortex-a72
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - uses: ./.github/actions/install-rust

    # Install targets in order to build various tests throughout the repo
    - run: rustup target add wasm32-wasi wasm32-unknown-unknown ${{ matrix.target }}
    - run: echo CARGO_BUILD_TARGET=${{ matrix.target }} >> $GITHUB_ENV
      if: matrix.target != ''

    # Fix an ICE for now in gcc when compiling zstd with debuginfo (??)
    - run: echo CFLAGS=-g0 >> $GITHUB_ENV
      if: matrix.target == 'x86_64-pc-windows-gnu'

    - run: cargo fetch --locked
    - run: cargo fetch --locked --manifest-path crates/test-programs/wasi-tests/Cargo.toml

    - uses: actions/cache@v2
      with:
        path: ${{ runner.tool_cache }}/qemu
        key: qemu-${{ matrix.target }}-${{ env.QEMU_BUILD_VERSION }}
      if: matrix.target != '' && matrix.os == 'ubuntu-latest'
    - name: Install cross-compilation tools
      run: |
        set -ex
        sudo apt-get update
        sudo apt-get install -y ${{ matrix.gcc_package }} ninja-build

        # Configure Cargo for cross compilation and tell it how it can run
        # cross executables
        upcase=$(echo ${{ matrix.target }} | awk '{ print toupper($0) }' | sed 's/-/_/g')
        echo CARGO_TARGET_${upcase}_RUNNER=${{ runner.tool_cache }}/qemu/bin/${{ matrix.qemu }} ${{ matrix.qemu_flags }} >> $GITHUB_ENV
        echo CARGO_TARGET_${upcase}_LINKER=${{ matrix.gcc }} >> $GITHUB_ENV

        # QEMU emulation is not always the speediest, so total testing time
        # goes down if we build the libs in release mode when running tests.
        echo CARGO_PROFILE_DEV_OPT_LEVEL=2 >> $GITHUB_ENV

        # See comments in the source for why we enable this during QEMU
        # emulation.
        echo WASMTIME_TEST_NO_HOG_MEMORY=1 >> $GITHUB_ENV

        # See if qemu is already in the cache
        if [ -f ${{ runner.tool_cache }}/qemu/built ]; then
          exit 0
        fi

        # Download and build qemu from source since the most recent release is
        # way faster at arm emulation than the current version github actions'
        # ubuntu image uses. Disable as much as we can to get it to build
        # quickly.
        curl https://download.qemu.org/qemu-$QEMU_BUILD_VERSION.tar.xz | tar xJf -
        cd qemu-$QEMU_BUILD_VERSION
        ./configure --target-list=${{ matrix.qemu_target }} --prefix=${{ runner.tool_cache}}/qemu --disable-tools --disable-slirp --disable-fdt --disable-capstone --disable-docs
        ninja -C build install
        touch ${{ runner.tool_cache }}/qemu/built
      if: matrix.gcc != ''

    # Ensure all our examples build and execute
    - run: cargo run -p run-examples
      env:
        RUST_BACKTRACE: 1
      if: matrix.target == ''

    # Build and test all features except for lightbeam
    - run: ./ci/run-tests.sh --locked
      env:
        RUST_BACKTRACE: 1

    # Test debug (DWARF) related functionality.
    - run: |
        sudo apt-get update && sudo apt-get install -y gdb lldb llvm
        cargo test test_debug_dwarf -- --ignored --test-threads 1
      if: matrix.os == 'ubuntu-latest' && matrix.target == ''
      env:
        RUST_BACKTRACE: 1

    # Test uffd functionality on Linux
    - run: |
        cargo test --features uffd -p wasmtime-runtime instance::allocator::pooling
        cargo test --features uffd -p wasmtime-cli pooling_allocator
        cargo test --features uffd -p wasmtime-cli wast::Cranelift
      if: matrix.os == 'ubuntu-latest' && matrix.target == ''
      env:
        RUST_BACKTRACE: 1

    # Build and test lightbeam. Note that
    # Lightbeam tests fail right now, but we don't want to block on that.
    - run: cargo build --package lightbeam
      if: matrix.target != 'aarch64-unknown-linux-gnu'
    - run: cargo test --package lightbeam
      if: matrix.target != 'aarch64-unknown-linux-gnu'
      continue-on-error: true
      env:
        RUST_BACKTRACE: 1

  # Perform all tests (debug mode) for `wasmtime` with the old x86 backend.
  test_x86:
    name: Test old x86 backend
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - uses: ./.github/actions/install-rust
      with:
        toolchain: stable

    # Install wasm32 targets in order to build various tests throughout the
    # repo.
    - run: rustup target add wasm32-wasi
    - run: rustup target add wasm32-unknown-unknown

    # Run the old x86 backend CI (we will eventually remove this).
    - run: ./ci/run-tests.sh --features old-x86-backend --locked
      env:
        RUST_BACKTRACE: 1

  # Build and test the wasi-nn module.
  test_wasi_nn:
    name: Test wasi-nn module
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - uses: ./.github/actions/install-rust
      - run: rustup target add wasm32-wasi
      - uses: ./.github/actions/install-openvino
      - run: ./ci/run-wasi-nn-example.sh
        env:
          RUST_BACKTRACE: 1

  # Build and test the wasi-crypto module.
  test_wasi_crypto:
    name: Test wasi-crypto module
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - run: rustup target add wasm32-wasi
    - name: Install Rust
      run: rustup update stable && rustup default stable
    - run: ./ci/run-wasi-crypto-example.sh
      env:
        RUST_BACKTRACE: 1

  bench:
    name: Run benchmarks
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - run: rustup target add wasm32-wasi
    - name: Install Rust
      run: rustup update stable && rustup default stable
    - run: cargo bench
    - run: cargo bench --features uffd

  # Verify that cranelift's code generation is deterministic
  meta_determinist_check:
    name: Meta deterministic check
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Install Rust
      run: rustup update stable && rustup default stable
    - run: cd cranelift/codegen && cargo build --features all-arch
    - run: ci/ensure_deterministic_build.sh

  # Perform release builds of `wasmtime` and `libwasmtime.so`. Builds on
  # Windows/Mac/Linux, and artifacts are uploaded after the build is finished.
  # Note that we also run tests here to test exactly what we're deploying.
  build:
    name: Build wasmtime
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
        - build: x86_64-linux
          os: ubuntu-latest
        - build: x86_64-macos
          os: macos-latest
        - build: x86_64-windows
          os: windows-latest
        - build: x86_64-mingw
          os: windows-latest
          target: x86_64-pc-windows-gnu
        - build: aarch64-linux
          os: ubuntu-latest
          target: aarch64-unknown-linux-gnu
          gcc_package: gcc-aarch64-linux-gnu
          gcc: aarch64-linux-gnu-gcc
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - uses: ./.github/actions/install-rust
    - uses: ./.github/actions/binary-compatible-builds
      if: matrix.target == ''

    - name: Install cross-compilation tools
      run: |
        set -ex
        sudo apt-get update
        sudo apt-get install -y ${{ matrix.gcc_package }}
        upcase=$(echo ${{ matrix.target }} | awk '{ print toupper($0) }' | sed 's/-/_/g')
        echo CARGO_TARGET_${upcase}_LINKER=${{ matrix.gcc }} >> $GITHUB_ENV
      if: matrix.target != '' && matrix.os == 'ubuntu-latest'
    - run: |
        echo CARGO_BUILD_TARGET=${{ matrix.target }} >> $GITHUB_ENV
        rustup target add ${{ matrix.target }}
      if: matrix.target != ''

    # Build `wasmtime` and executables
    - run: $CENTOS cargo build --release --bin wasmtime

    # Build `libwasmtime.so`
    - run: $CENTOS cargo build --release --manifest-path crates/c-api/Cargo.toml

    # Assemble release artifats appropriate for this platform, then upload them
    # unconditionally to this workflow's files so we have a copy of them.
    - run: ./ci/build-tarballs.sh "${{ matrix.build }}" "${{ matrix.target }}"
    - uses: actions/upload-artifact@v1
      with:
        name: bins-${{ matrix.build }}
        path: dist

    # ... and if this was an actual push (tag or `main`) then we publish a
    # new release. This'll automatically publish a tag release or update `dev`
    # with this `sha`
    - run: cd .github/actions/github-release && npm install --production
    - name: Publish Release
      uses: ./.github/actions/github-release
      if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
      with:
        files: "dist/*"
        token: ${{ secrets.GITHUB_TOKEN }}

  cargo-audit:
    env:
      CARGO_AUDIT_VERSION: 0.11.2
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/cache@v1
      with:
        path: ${{ runner.tool_cache }}/cargo-audit
        key: cargo-audit-bin-${{ env.CARGO_AUDIT_VERSION }}
    - run: echo "${{ runner.tool_cache }}/cargo-audit/bin" >> $GITHUB_PATH
    - run: |
        cargo install --root ${{ runner.tool_cache }}/cargo-audit --version ${{ env.CARGO_AUDIT_VERSION }} cargo-audit
        cargo audit

  verify-publish:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - run: rustup update stable && rustup default stable
    - run: |
        cd ${{ runner.tool_cache }}
        curl -L https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz | tar xzf -
        echo "`pwd`/sccache-0.2.13-x86_64-unknown-linux-musl" >> $GITHUB_PATH
        echo RUSTC_WRAPPER=sccache >> $GITHUB_ENV
    - run: |
        rustc scripts/publish.rs
        ./publish verify