Test wasm32-emscripten wheels

.github/workflows/CI.yml

@@ -223,7 +223,7 @@ jobs:
           name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }}
           path: dist
 
-  build-wasm32-emscripten:
+  build-wasm32-emscripten-pyodide:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -245,12 +245,23 @@ jobs:
 
       - name: Install Emscripten
         uses: mymindstorm/setup-emsdk@v14
+        with:
+          # This needs to match the exact expected version pyodide expects...seems a bit brittle TBH, maybe I'm missing something.
+          # Discover by updating pyodide in package.json and re-running 'npm run test'; it'll spit out the error of 
+          # the expected vs actual versions.
+          version: '3.1.58'
 
       - name: Build
         run: |
           pip install maturin
           maturin build --release -i python${{ matrix.python }} --features wasm32-compat --target wasm32-unknown-emscripten -o ./dist
 
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - run: npm install
+      - run: npm run test
+
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }}

.gitignore

@@ -1,3 +1,6 @@
+node_modules/
+package-lock.json
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

package.json

@@ -0,0 +1,26 @@
+{
+  "name": "cramjam",
+  "version": "1.0.0",
+  "description": "for running wasm tests.",
+  "author": "Miles Granger",
+  "license": "MIT",
+  "homepage": "https://github.com/milesgranger/cramjam#readme",
+  "main": "tests/test_cramjam_pyodide.js",
+  "dependencies": {
+    "prettier": "^2.7.1",
+    "pyodide": "^0.26.2"
+  },
+  "scripts": {
+    "test": "node tests/test_cramjam_pyodide.js",
+    "format": "prettier --write 'tests/test_cramjam_pyodide.js'",
+    "lint": "prettier --check 'tests/test_cramjam_pyodide.js'"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "printWidth": 119,
+    "bracketSpacing": false,
+    "arrowParens": "avoid"
+  }
+}

tests/test_cramjam_pyodide.js

@@ -0,0 +1,123 @@
+// Credit: https://github.com/pydantic/pydantic-core/blob/d6e7890b36ef21cb28180a7f5b1479da2319012d/tests/emscripten_runner.js
+
+const {opendir} = require('node:fs/promises');
+const {loadPyodide} = require('pyodide');
+const path = require('path');
+
+async function find_wheel(dist_dir) {
+  const dir = await opendir(dist_dir);
+  for await (const dirent of dir) {
+    if (dirent.name.endsWith('.whl')) {
+      return path.join(dist_dir, dirent.name);
+    }
+  }
+}
+
+function make_tty_ops(stream) {
+  return {
+    // get_char has 3 particular return values:
+    // a.) the next character represented as an integer
+    // b.) undefined to signal that no data is currently available
+    // c.) null to signal an EOF
+    get_char(tty) {
+      if (!tty.input.length) {
+        let result = null;
+        const BUFSIZE = 256;
+        const buf = Buffer.alloc(BUFSIZE);
+        const bytesRead = fs.readSync(process.stdin.fd, buf, 0, BUFSIZE, -1);
+        if (bytesRead === 0) {
+          return null;
+        }
+        result = buf.slice(0, bytesRead);
+        tty.input = Array.from(result);
+      }
+      return tty.input.shift();
+    },
+    put_char(tty, val) {
+      try {
+        if (val !== null) {
+          tty.output.push(val);
+        }
+        if (val === null || val === 10) {
+          process.stdout.write(Buffer.from(tty.output));
+          tty.output = [];
+        }
+      } catch (e) {
+        console.warn(e);
+      }
+    },
+    fsync(tty) {
+      if (!tty.output || tty.output.length === 0) {
+        return;
+      }
+      stream.write(Buffer.from(tty.output));
+      tty.output = [];
+    },
+  };
+}
+
+function setupStreams(FS, TTY) {
+  let mytty = FS.makedev(FS.createDevice.major++, 0);
+  let myttyerr = FS.makedev(FS.createDevice.major++, 0);
+  TTY.register(mytty, make_tty_ops(process.stdout));
+  TTY.register(myttyerr, make_tty_ops(process.stderr));
+  FS.mkdev('/dev/mytty', mytty);
+  FS.mkdev('/dev/myttyerr', myttyerr);
+  FS.unlink('/dev/stdin');
+  FS.unlink('/dev/stdout');
+  FS.unlink('/dev/stderr');
+  FS.symlink('/dev/mytty', '/dev/stdin');
+  FS.symlink('/dev/mytty', '/dev/stdout');
+  FS.symlink('/dev/myttyerr', '/dev/stderr');
+  FS.closeStream(0);
+  FS.closeStream(1);
+  FS.closeStream(2);
+  FS.open('/dev/stdin', 0);
+  FS.open('/dev/stdout', 1);
+  FS.open('/dev/stderr', 1);
+}
+
+async function main() {
+  const root_dir = path.resolve(__dirname, '..');
+  const wheel_path = await find_wheel(path.join(root_dir, 'dist'));
+  let errcode = 1;
+  try {
+    const pyodide = await loadPyodide();
+    const FS = pyodide.FS;
+    setupStreams(FS, pyodide._module.TTY);
+    FS.mkdir('/test_dir');
+    FS.mount(FS.filesystems.NODEFS, {root: path.join(root_dir, 'tests')}, '/test_dir');
+    FS.chdir('/test_dir');
+    await pyodide.loadPackage(['micropip', 'pytest', 'pytz']);
+    // language=python
+    errcode = await pyodide.runPythonAsync(`
+import micropip
+import importlib
+
+# ugly hack to get tests to work on arm64 (my m1 mac)
+# see https://github.com/pyodide/pyodide/issues/2840
+# import sys; sys.setrecursionlimit(200)
+
+await micropip.install([
+    'file:${wheel_path}',
+])
+importlib.invalidate_caches()
+
+print('installed packages:', micropip.list())
+
+import cramjam
+
+data = b"bytes"
+compressed = cramjam.snappy.compress(data)
+decompressed = cramjam.snappy.decompress(compressed)
+assert bytes(decompressed) == data
+
+`);
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+  process.exit(errcode);
+}
+
+main();