From ba8b0ae88e137b7c04b6dcd1d9ac1137366a9eec Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 27 Apr 2021 22:13:47 -0500 Subject: [PATCH] Compile windows runners in linux --- .github/workflows/release.yml | 51 +++++++++++++--- Cargo.lock | 16 +---- build.rs | 111 ++++++++++++++++++++++++---------- 3 files changed, 122 insertions(+), 56 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bcd7686..b8ec0b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,12 +6,45 @@ env: CARGO_TERM_COLOR: always jobs: + compile_windows: + runs-on: ubuntu-latest + + steps: + - run: curl https://get.wasmer.io -sSfL | sh + + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - run: aws s3 sync s3://${{ secrets.S3_BUCKET_PUBLIC }}/lang-runners lang-runners + + - name: Compile windows runners + run: | + source ~/.wasmer/wasmer.sh + for runner in lang-runners/*.wasm; do + wasmer compile --target x86_64-pc-windows-msvc --llvm "$runner" -o "${runner/.wasm/.wjit}" + rm "$runner" + done + + - uses: actions/upload-artifact@v2 + with: + name: windows-runners + path: lang-runners/ + build: + needs: compile_windows runs-on: ${{ matrix.os }}-latest strategy: matrix: - os: [ubuntu, macos] #, windows] + include: + - os: ubuntu + cargoargs: --features build-llvm + - os: macos + cargoargs: --features build-llvm + - os: windows + cargoargs: '' steps: - uses: actions/checkout@v2 @@ -41,14 +74,12 @@ jobs: - run: aws s3 sync s3://${{ secrets.S3_BUCKET_BUILD }}/cli-assets dist - if: matrix.os == 'windows' - name: Install LLVM on windows - shell: bash - run: | - cargo install llvmenv - llvmenv init - llvmenv build-entry 11.0.0 -j4 - llvmenv global 11.0.0 - echo "LLVM_SYS_110_PREFIX=$(llvmenv prefix)" >> $GITHUB_ENV + uses: actions/download-artifact@v2 + with: + name: windows-runners + path: compiled_runners + - if: matrix.os == 'windows' + run: echo "COMPILED_RUNNERS=compiled_runners" >>$GITHUB_ENV - if: matrix.os == 'macos' run: | brew install llvm @@ -56,7 +87,7 @@ jobs: - if: matrix.os == 'ubuntu' run: sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" - - run: cargo build --release --no-default-features --features build-llvm + - run: cargo build --release --no-default-features ${{matrix.cargoargs}} env: SENTRY_DSN: ${{ secrets.SENTRY_DSN }} diff --git a/Cargo.lock b/Cargo.lock index 9abaa7a..6627acb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -620,17 +620,6 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" -[[package]] -name = "futures-executor" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - [[package]] name = "futures-io" version = "0.3.14" @@ -2775,12 +2764,11 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasi-process" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc20fe23489c06613c2fe78f08c3fb8d9bd0d90929b6daecd33da9ddc789ce7c" +checksum = "f8b20032ff6501c2abbdd000cda737a65f71dfcfaffc9c2f14ac1d6feae84c93" dependencies = [ "bytes", - "futures-executor", "parking_lot", "serde", "tokio", diff --git a/build.rs b/build.rs index d16a61e..1840eba 100644 --- a/build.rs +++ b/build.rs @@ -7,64 +7,111 @@ use wasmer_compiler_cranelift::Cranelift as Compiler; #[cfg(feature = "build-llvm")] use wasmer_compiler_llvm::LLVM as Compiler; -use wasmer_engine::Artifact; +enum CompilationSource { + Precompiled(PathBuf), + #[cfg(any(feature = "build-cranelift", feature = "build-llvm"))] + Compiler { + engine: wasmer::JITEngine, + runners_dir: PathBuf, + jit_ext: &'static str, + tunables: wasmer::BaseTunables, + }, +} fn main() { - let mut features = wasmer::CpuFeature::set(); - for feat in env::var("CARGO_CFG_TARGET_FEATURE").unwrap().split(',') { - if let Ok(feat) = feat.parse() { - features.insert(feat); + let source = match env::var_os("COMPILED_RUNNERS") { + Some(dir) => CompilationSource::Precompiled(dir.into()), + #[cfg(not(any(feature = "build-cranelift", feature = "build-llvm")))] + None => { + panic!("need build-cranelift or build-llvm or the COMPILED_RUNNERS env var") } - } - let target = wasmer::Target::new(env::var("TARGET").unwrap().parse().unwrap(), features); - let tunables = wasmer::BaseTunables::for_target(&target); - let jit_ext = wasmer::JITArtifact::get_default_extension(target.triple()); - let engine = wasmer::JIT::new(Compiler::new()).target(target).engine(); + #[cfg(any(feature = "build-cranelift", feature = "build-llvm"))] + None => { + let mut features = wasmer::CpuFeature::set(); + for feat in env::var("CARGO_CFG_TARGET_FEATURE").unwrap().split(',') { + if let Ok(feat) = feat.parse() { + features.insert(feat); + } + } + let target = + wasmer::Target::new(env::var("TARGET").unwrap().parse().unwrap(), features); + let tunables = wasmer::BaseTunables::for_target(&target); + let jit_ext = wasmer::JITArtifact::get_default_extension(target.triple()); + let engine = wasmer::JIT::new(Compiler::new()).target(target).engine(); + + let runners_dir = fs::canonicalize("../logic/wasm-dist/lang-runners") + .expect("need to run logic/build-wasm.sh"); + + CompilationSource::Compiler { + engine, + runners_dir, + jit_ext, + tunables, + } + } + }; let lang_runners = [("Python", "pyrunner"), ("Javascript", "jsrunner")]; - let runners_dir = fs::canonicalize("../logic/wasm-dist/lang-runners") - .expect("need to run logic/build-wasm.sh"); let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); let mut match_lang = fs::File::create(out_dir.join("lang_runners.rs")).unwrap(); writeln!(match_lang, "match lang {{").unwrap(); for (lang, runner) in &lang_runners { - let mut src = runners_dir.join(runner); - src.set_extension("wasm"); - let mut dst = out_dir.join(runner); - dst.set_extension(jit_ext); + let (path, include_bin) = match &source { + CompilationSource::Precompiled(dir) => { + let mut wjit = dir.join(runner); + wjit.set_extension("wjit"); + (wjit, true) + } + #[cfg(any(feature = "build-cranelift", feature = "build-llvm"))] + CompilationSource::Compiler { + engine, + runners_dir, + jit_ext, + tunables, + } => { + use wasmer_engine::Artifact; - println!("compiling {}", runner); + let mut src = runners_dir.join(runner); + src.set_extension("wasm"); + let mut dst = out_dir.join(runner); + dst.set_extension(*jit_ext); - println!("cargo:rerun-if-changed={}", src.display()); + println!("compiling {}", runner); - let needs_updating = src - .metadata() - .and_then(|m| Ok((m, dst.metadata()?))) - .and_then(|(src, dst)| Ok(src.modified()? > dst.modified()?)) - .unwrap_or(true); + println!("cargo:rerun-if-changed={}", src.display()); - if needs_updating { - let wasm_source = fs::read(&src).unwrap(); - let artifact = wasmer::JITArtifact::new(&engine, &wasm_source, &tunables).unwrap(); + let needs_updating = src + .metadata() + .and_then(|m| Ok((m, dst.metadata()?))) + .and_then(|(src, dst)| Ok(src.modified()? > dst.modified()?)) + .unwrap_or(true); - fs::write(&dst, artifact.serialize().unwrap()).unwrap(); - } + if needs_updating { + let wasm_source = fs::read(&src).unwrap(); + let artifact = + wasmer::JITArtifact::new(engine, &wasm_source, tunables).unwrap(); + + fs::write(&dst, artifact.serialize().unwrap()).unwrap(); + } + + (dst, cfg!(feature = "build-llvm")) + } + }; writeln!( match_lang, " Lang::{} => lang_runner!({}({:?}){}),", lang, - // - if cfg!(feature = "build-llvm") { + if include_bin { "include_bytes!" } else { "&std::fs::read" }, - dst, - if cfg!(feature = "build-llvm") { + path, + if include_bin { "" } else { r#".expect("should compile with --features=build-llvm when distributing")"#