From e6b4c6753a694bafd801b42a69d7f5035c87e66e Mon Sep 17 00:00:00 2001 From: David Ellis Date: Wed, 13 Nov 2024 14:45:56 -0600 Subject: [PATCH 1/3] Get WebGPU integration testing working again --- .github/workflows/rust.yml | 2 ++ alan/src/compile/integration_tests.rs | 12 +++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0d7d7c5be..84e4e98b6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -42,6 +42,8 @@ jobs: runs-on: [self-hosted, macOS] steps: - uses: actions/checkout@v4 + - name: Node deps + run: yarn - name: Build run: cargo build --verbose - if: ${{ github.ref_name == 'main' }} diff --git a/alan/src/compile/integration_tests.rs b/alan/src/compile/integration_tests.rs index a5f3e0160..f46ca7184 100644 --- a/alan/src/compile/integration_tests.rs +++ b/alan/src/compile/integration_tests.rs @@ -228,16 +228,11 @@ macro_rules! test_gpgpu { // My playwright scripts only work on Linux and MacOS, though, so that reduces it // to just MacOS to test this on. // if cfg!(windows) || cfg!(macos) { - // TODO: This apparently wasn't working at all because the `macos` cfg keyword was - // deprecated at some point and the new version of Rust finally told me? In any - // case, fixing this will be in a follow-up PR - /* if cfg!(target_os = "macos") { crate::program::Program::set_target_lang_js(); - { - let mut program = crate::program::Program::get_program().lock().unwrap(); - program.env.insert("ALAN_TARGET".to_string(), "test".to_string()); - } + let mut program = crate::program::Program::get_program(); + program.env.insert("ALAN_TARGET".to_string(), "test".to_string()); + crate::program::Program::return_program(program); match crate::compile::web(filename.to_string()) { Ok(_) => { /* Do nothing */ } Err(e) => { @@ -303,7 +298,6 @@ macro_rules! test_gpgpu { Err(e) => Err(format!("Could not remove the generated HTML file {:?}", e)), }?; } - */ std::fs::remove_file(&filename)?; Ok(()) } From a72127d64e2f3287fbdc61ad75b237d08998b9b3 Mon Sep 17 00:00:00 2001 From: David Ellis Date: Thu, 14 Nov 2024 08:25:47 -0600 Subject: [PATCH 2/3] Get WebGPU integration tests working after debugging a lot of binding issues (and a couple of problems in the JS stdlib) --- .github/workflows/rust.yml | 5 +++++ alan/src/compile/integration_tests.rs | 11 ++--------- alan/src/std/root.ln | 24 ++++++++++++++++++------ chrome_console.js | 2 +- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 84e4e98b6..0cf4febff 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -44,6 +44,8 @@ jobs: - uses: actions/checkout@v4 - name: Node deps run: yarn + - name: Start web server + run: yarn start-server - name: Build run: cargo build --verbose - if: ${{ github.ref_name == 'main' }} @@ -52,6 +54,9 @@ jobs: - if: ${{ github.ref_name != 'main' }} name: Run tests run: cargo test --verbose + - if: always() + name: Stop web server + run: yarn stop-server test-arm-linux: runs-on: [self-hosted, linux, ARM64] diff --git a/alan/src/compile/integration_tests.rs b/alan/src/compile/integration_tests.rs index f46ca7184..5d190e828 100644 --- a/alan/src/compile/integration_tests.rs +++ b/alan/src/compile/integration_tests.rs @@ -273,21 +273,14 @@ macro_rules! test_gpgpu { return Err(format!("Failed to create temporary HTML file {:?}", e).into()); } }; - match std::process::Command::new("bash").arg("-c").arg("yarn start-server").output() { - Ok(a) => Ok(a), - Err(e) => Err(format!("Could not start test web server {:?}", e)), - }?; + // We're assuming an HTTP server is already running let run = match std::process::Command::new("bash") .arg("-c") - .arg(format!("yarn chrome-console http://localhost:8080/alan/{}.html", stringify!($rule))) + .arg(format!("yarn -s chrome-console http://localhost:8080/alan/{}.html", stringify!($rule))) .output() { Ok(a) => Ok(a), Err(e) => Err(format!("Could not run the test JS code {:?}", e)), }?; - match std::process::Command::new("bash").arg("-c").arg("yarn stop-server").output() { - Ok(a) => Ok(a), - Err(e) => Err(format!("Could not start test web server {:?}", e)), - }?; $( $type!($test_val, false, &run); )+ match std::fs::remove_file(&jsfile) { Ok(a) => Ok(a), diff --git a/alan/src/std/root.ln b/alan/src/std/root.ln index 8f46702d3..5af7833a9 100644 --- a/alan/src/std/root.ln +++ b/alan/src/std/root.ln @@ -187,7 +187,7 @@ export const tau = 6.283185307179586; /// Functions for (potentially) every type export fn{Rs} clone{T} (v: T) -> T = {Method{"clone"} :: T -> T}(v); // TODO: This needs to be turned into a JS function that's bound -export fn{Js} clone{T} (v: T) -> T = {"(function clone(t) { if (t instanceof Array) { return t.map(clone); } else if (t.build instanceof Function) { return t.build(t.val); } else { return structuredClone(t) } })" :: T -> T}(v); +export fn{Js} clone{T} (v: T) -> T = {"(function clone(t) { if (t instanceof Array) { return t.map(clone); } else if (t.build instanceof Function) { return t.build(t.val); } else if (t instanceof Set) { return t.union(new Set()); } else { return structuredClone(t); } })" :: T -> T}(v); // TODO: The "proper" way to hash this consistently for all types is to decompose the input type // into the various primitive types of Alan and then have hashing rules for each of them, which // may themselves decompose, etc. This might be doable in Alan code on top of specialized hashing @@ -1110,7 +1110,8 @@ export fn{Rs} repeat (a: string, n: i64) = {Method{"repeat"} :: (string, Deref{B a, {Cast{"usize"} :: Deref{i64} -> Binds{"usize"}}(n)); export fn{Js} repeat "((s, n) => new alan_std.Str(s.val.repeat(Number(n.val))))" <- RootBacking :: (string, i64) -> string; -export fn replace Method{"replace"} :: (string, string, string) -> string; +export fn{Rs} replace Method{"replace"} :: (string, string, string) -> string; +export fn{Js} replace "((s, o, n) => new alan_std.Str(s.valueOf().replaceAll(o.valueOf(), n.valueOf())))" :: (string, string, string) -> string; export fn{Rs} split "alan_std::splitstring" <- RootBacking :: (string, string) -> string[]; export fn{Js} split "((a, b) => a.val.split(b.val).map(v => new alan_std.Str(v)))" <- RootBacking :: (string, string) -> string[]; export fn{Rs} len (s: string) = {Cast{"i64"} :: Deref{Binds{"usize"}} -> i64}( @@ -1176,6 +1177,7 @@ export fn{Js} concat{T} (a: T[], b: T[]) -> T[] = {Method{"concat"} :: (T[], T[] export fn{Rs} append{T} "alan_std::append" <- RootBacking :: (Mut{T[]}, T[]); export fn{Js} append{T} "((a, b) => { for (let v in b) { a.push(v); } })" :: (Mut{T[]}, T[]); export fn{Rs} filled{T} "alan_std::filled" <- RootBacking :: (T, i64) -> T[]; +export fn{Js} filled{T} "((v, c) => { let out = []; for (let i = 0n; i < c; i++) { out.push(v); } return out; })" :: (T, i64) -> T[]; export fn{Rs} has{T} (a: T[], v: T) = {Method{"contains"} :: (T[], T) -> bool}(a, v); export fn{Js} has{T} (a: T[], v: T) = a.reduce(false, fn (out: bool, t: T) = if(out, true, t == v)); export fn{Rs} has{T} "alan_std::hasfnarray" <- RootBacking :: (T[], T -> bool) -> bool; @@ -1474,7 +1476,7 @@ export fn{Rs} f64 Method{"as_secs_f64"} :: Duration -> f64; /// Uuid-related bindings export fn{Rs} uuid "alan_std::Uuid::new_v4" <- RootBacking :: () -> uuid; -export fn{Js} uuid "alan_std::uuidv4" <- RootBacking :: () -> uuid; +export fn{Js} uuid "alan_std.uuidv4" <- RootBacking :: () -> uuid; export fn{Rs} string "format!" :: ("{}", uuid) -> string; export fn{Js} string "new alan_std.Str" :: uuid -> string; @@ -1495,10 +1497,10 @@ export fn{Js} mapWriteBuffer "alan_std.mapWriteBufferType" <- RootBacking :: () export fn{Rs} storageBuffer "alan_std::storage_buffer_type" <- RootBacking :: () -> BufferUsages; export fn{Js} storageBuffer "alan_std.storageBufferType" <- RootBacking :: () -> BufferUsages; export fn{Rs} GBuffer{T}(bu: BufferUsages, arr: T[]) = {"alan_std::create_buffer_init" <- RootBacking :: (BufferUsages, T[], i8) -> GBuffer}(bu, arr, {Size{T}}().i8); -export fn{Js} GBuffer{T}(bu: BufferUsages, arr: T[]) = {"alan_std.createBufferInit" <- RootBacking :: (BufferUsages, T[], i8) -> GBuffer}(bu, arr, {Size{T}}().i8); +export fn{Js} GBuffer{T}(bu: BufferUsages, arr: T[]) = {"alan_std.createBufferInit" <- RootBacking :: (BufferUsages, T[]) -> GBuffer}(bu, arr); export fn{Rs} GBuffer{T}(bu: BufferUsages, size: i64) = {"alan_std::create_empty_buffer" <- RootBacking :: (BufferUsages, i64, i8) -> GBuffer}(bu, size, {Size{T}}().i8); // TODO: Get the type into JS -export fn{Js} GBuffer{T}(bu: BufferUsages, size: i64) = {"alan_std.createEmptyBuffer" <- RootBacking :: (BufferUsages, i64) -> GBuffer}(bu, size); +export fn{Js} GBuffer{T}(bu: BufferUsages, size: i64) = {"alan_std.createEmptyBuffer" <- RootBacking :: (BufferUsages, i32) -> GBuffer}(bu, size.i32); export fn GBuffer{T}(vals: T[]) = GBuffer{T}(storageBuffer(), vals); export fn GBuffer{T}(size: i64) = GBuffer{T}(storageBuffer(), size); export fn{Rs} len "alan_std::bufferlen" <- RootBacking :: GBuffer -> i64; @@ -1506,7 +1508,7 @@ export fn{Js} len "alan_std.bufferlen" <- RootBacking :: GBuffer -> i64; export fn{Rs} id "alan_std::buffer_id" <- RootBacking :: GBuffer -> string; export fn{Js} id "alan_std.bufferid" <- RootBacking :: GBuffer -> string; export fn{Rs} GPGPU "alan_std::GPGPU::new" <- RootBacking :: (Own{string}, Own{Array{Array{GBuffer}}}, Deref{i64[3]}) -> GPGPU; -export fn{Js} GPGPU "new alan_std.GPGPU" <- RootBacking :: (string, Array{Array{GBuffer}}, i64[3]) -> GPGPU; +export fn{Js} GPGPU (src: string, gbuffers: Array{Array{GBuffer}}, idx: i64[3]) = {"new alan_std.GPGPU" <- RootBacking :: (string, Array{Array{GBuffer}}, i32[3]) -> GPGPU}(src, gbuffers, idx.map(i32)); export fn GPGPU(src: string, buf: GBuffer) -> GPGPU { // In order to support larger arrays, we need to split the buffer length across them. Each of // indices is allowed to be up to 65535 (yes, a 16-bit integer) leading to a maximum length of @@ -4793,6 +4795,15 @@ export fn map(gb: GBuffer, f: gf32 -> gf32) { compute.build.run; return out; } +export fn{Js} map(gb: GBuffer, f: gf32 -> gf32) { // TODO: Get rid of this jank + let idx = gFor(gb.len); + let val = gb[idx].asF32; + let out = GBuffer{f32}(gb.len); // The JS binding currently ignores this + {"((b) => { b.ValKind = alan_std.F32; })" :: GBuffer}(out); // Trick to get it to work for now + let compute = out[idx].store(f(val).asI32); + compute.build.run; + return out; +} // This one technically only works for i32/u32/f32 by chance. Once the issue above is fixed, this // one can be fixed, too. export fn map{G, G2}(gb: GBuffer, f: (G, gu32) -> G2) { @@ -4819,6 +4830,7 @@ export fn ExitCode(e: i64) = ExitCode(e.u8); // TODO: Rework this to just print anything that can be converted to `string` via interfaces export fn{Rs} print{T}(v: T) = {"println!" :: ("{}", string)}(v.string); export fn{Js} print{T}(v: T) = {"console.log" :: T}(v); +export fn{Js} print{T}(v: T[]) = {"((vs) => console.log(vs.map((v) => v.valueOf())))" :: T[]}(v); export fn{Js} print "((s) => console.log(s.val))" :: string; export fn{Js} print (b: bool) = b.string.print; export fn{Js} print (i: i8) = i.string.print; diff --git a/chrome_console.js b/chrome_console.js index 2e6a61990..94c997d23 100644 --- a/chrome_console.js +++ b/chrome_console.js @@ -6,7 +6,7 @@ import { chromium } from 'playwright'; const page = await context.newPage(); page.on('console', msg => console.log(msg.text())); await page.goto(process.argv.pop()); - await new Promise((r) => setTimeout(r, 1000)); // TODO: Better way to determine completion + await new Promise((r) => setTimeout(r, 2000)); // TODO: Better way to determine completion await context.close(); await browser.close(); })(); From 80aa19414731c79ffeaa0607c1e250d321244fe6 Mon Sep 17 00:00:00 2001 From: David Ellis Date: Thu, 14 Nov 2024 09:50:54 -0600 Subject: [PATCH 3/3] The tests cut off in CI, so let's increase the timeout and hope this fixes it. --- chrome_console.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chrome_console.js b/chrome_console.js index 94c997d23..078a4e0c3 100644 --- a/chrome_console.js +++ b/chrome_console.js @@ -6,7 +6,7 @@ import { chromium } from 'playwright'; const page = await context.newPage(); page.on('console', msg => console.log(msg.text())); await page.goto(process.argv.pop()); - await new Promise((r) => setTimeout(r, 2000)); // TODO: Better way to determine completion + await new Promise((r) => setTimeout(r, 5000)); // TODO: Better way to determine completion await context.close(); await browser.close(); })();