diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0219cdbb..de7fec34 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -223,6 +223,51 @@ jobs: name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }} path: dist + build-wasm32-emscripten-pyodide: + runs-on: ubuntu-latest + strategy: + matrix: + python: + - "3.12" + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly + target: wasm32-unknown-emscripten + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install Emscripten + uses: mymindstorm/setup-emsdk@v14 + with: + # This needs to match the exact expected version pyodide expects...seems a bit brittle TBH, maybe I'm missing something. + # Discover by updating pyodide in package.json and re-running 'npm run test'; it'll spit out the error of + # the expected vs actual versions. + version: '3.1.58' + + - name: Build + run: | + pip install maturin + maturin build --release -i python${{ matrix.python }} --features wasm32-compat --target wasm32-unknown-emscripten -o ./dist + + - uses: actions/setup-node@v3 + with: + node-version: '18' + - run: npm install + - run: npm run test + + - name: Upload wheels + uses: actions/upload-artifact@v4 + if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} + with: + name: wasm32-unknown-emscripten-python${{ matrix.python }} + path: dist + build-sdist: name: Build sdists runs-on: ubuntu-latest @@ -258,6 +303,8 @@ jobs: merge-multiple: true - name: List artifacts run: ls -lhs + - name: Remove wasm32 wheels # TODO: https://discuss.python.org/t/support-wasm-wheels-on-pypi/21924 + run: rm ./*wasm32.whl - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.gitignore b/.gitignore index f5e37467..b22a1c41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +node_modules/ +package-lock.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Cargo.toml b/Cargo.toml index bfa98e26..fad22b4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cramjam-python" -version = "2.8.4-rc4" +version = "2.8.4-rc5" authors = ["Miles Granger "] edition = "2021" license = "MIT" @@ -42,10 +42,12 @@ blosc2-shared = ["libcramjam/blosc2-shared"] use-system-blosc2-static = ["libcramjam/use-system-blosc2", "libcramjam/blosc2-static"] use-system-blosc2-shared = ["libcramjam/use-system-blosc2", "libcramjam/blosc2-shared"] +wasm32-compat = ["libcramjam/wasm32-compat"] + [dependencies] pyo3 = { version = "^0.22", default-features = false, features = ["macros"] } -libcramjam = { version = "0.4.5", default-features = false } +libcramjam = { version = "0.4.6", default-features = false } [build-dependencies] pyo3-build-config = "^0.22" diff --git a/package.json b/package.json new file mode 100644 index 00000000..314851b1 --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "cramjam", + "version": "1.0.0", + "description": "for running wasm tests.", + "author": "Miles Granger", + "license": "MIT", + "homepage": "https://github.com/milesgranger/cramjam#readme", + "main": "tests/test_cramjam_pyodide.js", + "dependencies": { + "prettier": "^2.7.1", + "pyodide": "^0.26.2" + }, + "scripts": { + "test": "node tests/test_cramjam_pyodide.js", + "format": "prettier --write 'tests/test_cramjam_pyodide.js'", + "lint": "prettier --check 'tests/test_cramjam_pyodide.js'" + }, + "prettier": { + "singleQuote": true, + "trailingComma": "all", + "tabWidth": 2, + "printWidth": 119, + "bracketSpacing": false, + "arrowParens": "avoid" + } +} diff --git a/tests/test_cramjam_pyodide.js b/tests/test_cramjam_pyodide.js new file mode 100644 index 00000000..e72366f5 --- /dev/null +++ b/tests/test_cramjam_pyodide.js @@ -0,0 +1,123 @@ +// Credit: https://github.com/pydantic/pydantic-core/blob/d6e7890b36ef21cb28180a7f5b1479da2319012d/tests/emscripten_runner.js + +const {opendir} = require('node:fs/promises'); +const {loadPyodide} = require('pyodide'); +const path = require('path'); + +async function find_wheel(dist_dir) { + const dir = await opendir(dist_dir); + for await (const dirent of dir) { + if (dirent.name.endsWith('.whl')) { + return path.join(dist_dir, dirent.name); + } + } +} + +function make_tty_ops(stream) { + return { + // get_char has 3 particular return values: + // a.) the next character represented as an integer + // b.) undefined to signal that no data is currently available + // c.) null to signal an EOF + get_char(tty) { + if (!tty.input.length) { + let result = null; + const BUFSIZE = 256; + const buf = Buffer.alloc(BUFSIZE); + const bytesRead = fs.readSync(process.stdin.fd, buf, 0, BUFSIZE, -1); + if (bytesRead === 0) { + return null; + } + result = buf.slice(0, bytesRead); + tty.input = Array.from(result); + } + return tty.input.shift(); + }, + put_char(tty, val) { + try { + if (val !== null) { + tty.output.push(val); + } + if (val === null || val === 10) { + process.stdout.write(Buffer.from(tty.output)); + tty.output = []; + } + } catch (e) { + console.warn(e); + } + }, + fsync(tty) { + if (!tty.output || tty.output.length === 0) { + return; + } + stream.write(Buffer.from(tty.output)); + tty.output = []; + }, + }; +} + +function setupStreams(FS, TTY) { + let mytty = FS.makedev(FS.createDevice.major++, 0); + let myttyerr = FS.makedev(FS.createDevice.major++, 0); + TTY.register(mytty, make_tty_ops(process.stdout)); + TTY.register(myttyerr, make_tty_ops(process.stderr)); + FS.mkdev('/dev/mytty', mytty); + FS.mkdev('/dev/myttyerr', myttyerr); + FS.unlink('/dev/stdin'); + FS.unlink('/dev/stdout'); + FS.unlink('/dev/stderr'); + FS.symlink('/dev/mytty', '/dev/stdin'); + FS.symlink('/dev/mytty', '/dev/stdout'); + FS.symlink('/dev/myttyerr', '/dev/stderr'); + FS.closeStream(0); + FS.closeStream(1); + FS.closeStream(2); + FS.open('/dev/stdin', 0); + FS.open('/dev/stdout', 1); + FS.open('/dev/stderr', 1); +} + +async function main() { + const root_dir = path.resolve(__dirname, '..'); + const wheel_path = await find_wheel(path.join(root_dir, 'dist')); + let errcode = 1; + try { + const pyodide = await loadPyodide(); + const FS = pyodide.FS; + setupStreams(FS, pyodide._module.TTY); + FS.mkdir('/test_dir'); + FS.mount(FS.filesystems.NODEFS, {root: path.join(root_dir, 'tests')}, '/test_dir'); + FS.chdir('/test_dir'); + await pyodide.loadPackage(['micropip', 'pytest', 'pytz']); + // language=python + errcode = await pyodide.runPythonAsync(` +import micropip +import importlib + +# ugly hack to get tests to work on arm64 (my m1 mac) +# see https://github.com/pyodide/pyodide/issues/2840 +# import sys; sys.setrecursionlimit(200) + +await micropip.install([ + 'file:${wheel_path}', +]) +importlib.invalidate_caches() + +print('installed packages:', micropip.list()) + +import cramjam + +data = b"bytes" +compressed = cramjam.snappy.compress(data) +decompressed = cramjam.snappy.decompress(compressed) +assert bytes(decompressed) == data + +`); + } catch (e) { + console.error(e); + process.exit(1); + } + process.exit(errcode); +} + +main();