Add wasm32-emscripten builds

.github/workflows/CI.yml

@@ -223,6 +223,51 @@ jobs:
           name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }}
           path: dist
 
+  build-wasm32-emscripten-pyodide:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python:
+          - "3.12"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: nightly
+          target: wasm32-unknown-emscripten
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install Emscripten
+        uses: mymindstorm/setup-emsdk@v14
+        with:
+          # This needs to match the exact expected version pyodide expects...seems a bit brittle TBH, maybe I'm missing something.
+          # Discover by updating pyodide in package.json and re-running 'npm run test'; it'll spit out the error of 
+          # the expected vs actual versions.
+          version: '3.1.58'
+
+      - name: Build
+        run: |
+          pip install maturin
+          maturin build --release -i python${{ matrix.python }} --features wasm32-compat --target wasm32-unknown-emscripten -o ./dist
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - run: npm install
+      - run: npm run test
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }}
+        with:
+          name: wasm32-unknown-emscripten-python${{ matrix.python }}
+          path: dist
+
   build-sdist:
     name: Build sdists
     runs-on: ubuntu-latest
@@ -258,6 +303,8 @@ jobs:
         merge-multiple: true
     - name: List artifacts
       run: ls -lhs
+    - name: Remove wasm32 wheels  # TODO: https://discuss.python.org/t/support-wasm-wheels-on-pypi/21924
+      run: rm ./*wasm32.whl
     - name: Publish package distributions to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:

.gitignore

@@ -1,3 +1,6 @@
+node_modules/
+package-lock.json
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cramjam-python"
-version = "2.8.4-rc4"
+version = "2.8.4-rc5"
 authors = ["Miles Granger <[email protected]>"]
 edition = "2021"
 license = "MIT"
@@ -42,10 +42,12 @@ blosc2-shared            = ["libcramjam/blosc2-shared"]
 use-system-blosc2-static = ["libcramjam/use-system-blosc2", "libcramjam/blosc2-static"]
 use-system-blosc2-shared = ["libcramjam/use-system-blosc2", "libcramjam/blosc2-shared"]
 
+wasm32-compat            = ["libcramjam/wasm32-compat"]
+
 
 [dependencies]
 pyo3 = { version = "^0.22", default-features = false, features = ["macros"] }
-libcramjam = { version = "0.4.5", default-features = false }
+libcramjam = { version = "0.4.6", default-features = false }
 
 [build-dependencies]
 pyo3-build-config = "^0.22"

package.json

@@ -0,0 +1,26 @@
+{
+  "name": "cramjam",
+  "version": "1.0.0",
+  "description": "for running wasm tests.",
+  "author": "Miles Granger",
+  "license": "MIT",
+  "homepage": "https://github.com/milesgranger/cramjam#readme",
+  "main": "tests/test_cramjam_pyodide.js",
+  "dependencies": {
+    "prettier": "^2.7.1",
+    "pyodide": "^0.26.2"
+  },
+  "scripts": {
+    "test": "node tests/test_cramjam_pyodide.js",
+    "format": "prettier --write 'tests/test_cramjam_pyodide.js'",
+    "lint": "prettier --check 'tests/test_cramjam_pyodide.js'"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "printWidth": 119,
+    "bracketSpacing": false,
+    "arrowParens": "avoid"
+  }
+}

tests/test_cramjam_pyodide.js

@@ -0,0 +1,123 @@
+// Credit: https://github.com/pydantic/pydantic-core/blob/d6e7890b36ef21cb28180a7f5b1479da2319012d/tests/emscripten_runner.js
+
+const {opendir} = require('node:fs/promises');
+const {loadPyodide} = require('pyodide');
+const path = require('path');
+
+async function find_wheel(dist_dir) {
+  const dir = await opendir(dist_dir);
+  for await (const dirent of dir) {
+    if (dirent.name.endsWith('.whl')) {
+      return path.join(dist_dir, dirent.name);
+    }
+  }
+}
+
+function make_tty_ops(stream) {
+  return {
+    // get_char has 3 particular return values:
+    // a.) the next character represented as an integer
+    // b.) undefined to signal that no data is currently available
+    // c.) null to signal an EOF
+    get_char(tty) {
+      if (!tty.input.length) {
+        let result = null;
+        const BUFSIZE = 256;
+        const buf = Buffer.alloc(BUFSIZE);
+        const bytesRead = fs.readSync(process.stdin.fd, buf, 0, BUFSIZE, -1);
+        if (bytesRead === 0) {
+          return null;
+        }
+        result = buf.slice(0, bytesRead);
+        tty.input = Array.from(result);
+      }
+      return tty.input.shift();
+    },
+    put_char(tty, val) {
+      try {
+        if (val !== null) {
+          tty.output.push(val);
+        }
+        if (val === null || val === 10) {
+          process.stdout.write(Buffer.from(tty.output));
+          tty.output = [];
+        }
+      } catch (e) {
+        console.warn(e);
+      }
+    },
+    fsync(tty) {
+      if (!tty.output || tty.output.length === 0) {
+        return;
+      }
+      stream.write(Buffer.from(tty.output));
+      tty.output = [];
+    },
+  };
+}
+
+function setupStreams(FS, TTY) {
+  let mytty = FS.makedev(FS.createDevice.major++, 0);
+  let myttyerr = FS.makedev(FS.createDevice.major++, 0);
+  TTY.register(mytty, make_tty_ops(process.stdout));
+  TTY.register(myttyerr, make_tty_ops(process.stderr));
+  FS.mkdev('/dev/mytty', mytty);
+  FS.mkdev('/dev/myttyerr', myttyerr);
+  FS.unlink('/dev/stdin');
+  FS.unlink('/dev/stdout');
+  FS.unlink('/dev/stderr');
+  FS.symlink('/dev/mytty', '/dev/stdin');
+  FS.symlink('/dev/mytty', '/dev/stdout');
+  FS.symlink('/dev/myttyerr', '/dev/stderr');
+  FS.closeStream(0);
+  FS.closeStream(1);
+  FS.closeStream(2);
+  FS.open('/dev/stdin', 0);
+  FS.open('/dev/stdout', 1);
+  FS.open('/dev/stderr', 1);
+}
+
+async function main() {
+  const root_dir = path.resolve(__dirname, '..');
+  const wheel_path = await find_wheel(path.join(root_dir, 'dist'));
+  let errcode = 1;
+  try {
+    const pyodide = await loadPyodide();
+    const FS = pyodide.FS;
+    setupStreams(FS, pyodide._module.TTY);
+    FS.mkdir('/test_dir');
+    FS.mount(FS.filesystems.NODEFS, {root: path.join(root_dir, 'tests')}, '/test_dir');
+    FS.chdir('/test_dir');
+    await pyodide.loadPackage(['micropip', 'pytest', 'pytz']);
+    // language=python
+    errcode = await pyodide.runPythonAsync(`
+import micropip
+import importlib
+
+# ugly hack to get tests to work on arm64 (my m1 mac)
+# see https://github.com/pyodide/pyodide/issues/2840
+# import sys; sys.setrecursionlimit(200)
+
+await micropip.install([
+    'file:${wheel_path}',
+])
+importlib.invalidate_caches()
+
+print('installed packages:', micropip.list())
+
+import cramjam
+
+data = b"bytes"
+compressed = cramjam.snappy.compress(data)
+decompressed = cramjam.snappy.decompress(compressed)
+assert bytes(decompressed) == data
+
+`);
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+  process.exit(errcode);
+}
+
+main();