diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 91d5561a..a48ce0f6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -223,7 +223,7 @@ jobs: name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }} path: dist - build-wasm32-emscripten: + build-wasm32-emscripten-pyodide: runs-on: ubuntu-latest strategy: matrix: @@ -245,12 +245,23 @@ jobs: - name: Install Emscripten uses: mymindstorm/setup-emsdk@v14 + with: + # This needs to match the exact expected version pyodide expects...seems a bit brittle TBH, maybe I'm missing something. + # Discover by updating pyodide in package.json and re-running 'npm run test'; it'll spit out the error of + # the expected vs actual versions. + version: '3.1.58' - name: Build run: | pip install maturin maturin build --release -i python${{ matrix.python }} --features wasm32-compat --target wasm32-unknown-emscripten -o ./dist + - uses: actions/setup-node@v3 + with: + node-version: '18' + - run: npm install + - run: npm run test + - name: Upload wheels uses: actions/upload-artifact@v4 if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} diff --git a/.gitignore b/.gitignore index f5e37467..b22a1c41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +node_modules/ +package-lock.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/package.json b/package.json new file mode 100644 index 00000000..314851b1 --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "cramjam", + "version": "1.0.0", + "description": "for running wasm tests.", + "author": "Miles Granger", + "license": "MIT", + "homepage": "https://github.com/milesgranger/cramjam#readme", + "main": "tests/test_cramjam_pyodide.js", + "dependencies": { + "prettier": "^2.7.1", + "pyodide": "^0.26.2" + }, + "scripts": { + "test": "node tests/test_cramjam_pyodide.js", + "format": "prettier --write 'tests/test_cramjam_pyodide.js'", + "lint": "prettier --check 'tests/test_cramjam_pyodide.js'" + }, + "prettier": { + "singleQuote": true, + "trailingComma": "all", + "tabWidth": 2, + "printWidth": 119, + "bracketSpacing": false, + "arrowParens": "avoid" + } +} diff --git a/tests/test_cramjam_pyodide.js b/tests/test_cramjam_pyodide.js new file mode 100644 index 00000000..e72366f5 --- /dev/null +++ b/tests/test_cramjam_pyodide.js @@ -0,0 +1,123 @@ +// Credit: https://github.com/pydantic/pydantic-core/blob/d6e7890b36ef21cb28180a7f5b1479da2319012d/tests/emscripten_runner.js + +const {opendir} = require('node:fs/promises'); +const {loadPyodide} = require('pyodide'); +const path = require('path'); + +async function find_wheel(dist_dir) { + const dir = await opendir(dist_dir); + for await (const dirent of dir) { + if (dirent.name.endsWith('.whl')) { + return path.join(dist_dir, dirent.name); + } + } +} + +function make_tty_ops(stream) { + return { + // get_char has 3 particular return values: + // a.) the next character represented as an integer + // b.) undefined to signal that no data is currently available + // c.) null to signal an EOF + get_char(tty) { + if (!tty.input.length) { + let result = null; + const BUFSIZE = 256; + const buf = Buffer.alloc(BUFSIZE); + const bytesRead = fs.readSync(process.stdin.fd, buf, 0, BUFSIZE, -1); + if (bytesRead === 0) { + return null; + } + result = buf.slice(0, bytesRead); + tty.input = Array.from(result); + } + return tty.input.shift(); + }, + put_char(tty, val) { + try { + if (val !== null) { + tty.output.push(val); + } + if (val === null || val === 10) { + process.stdout.write(Buffer.from(tty.output)); + tty.output = []; + } + } catch (e) { + console.warn(e); + } + }, + fsync(tty) { + if (!tty.output || tty.output.length === 0) { + return; + } + stream.write(Buffer.from(tty.output)); + tty.output = []; + }, + }; +} + +function setupStreams(FS, TTY) { + let mytty = FS.makedev(FS.createDevice.major++, 0); + let myttyerr = FS.makedev(FS.createDevice.major++, 0); + TTY.register(mytty, make_tty_ops(process.stdout)); + TTY.register(myttyerr, make_tty_ops(process.stderr)); + FS.mkdev('/dev/mytty', mytty); + FS.mkdev('/dev/myttyerr', myttyerr); + FS.unlink('/dev/stdin'); + FS.unlink('/dev/stdout'); + FS.unlink('/dev/stderr'); + FS.symlink('/dev/mytty', '/dev/stdin'); + FS.symlink('/dev/mytty', '/dev/stdout'); + FS.symlink('/dev/myttyerr', '/dev/stderr'); + FS.closeStream(0); + FS.closeStream(1); + FS.closeStream(2); + FS.open('/dev/stdin', 0); + FS.open('/dev/stdout', 1); + FS.open('/dev/stderr', 1); +} + +async function main() { + const root_dir = path.resolve(__dirname, '..'); + const wheel_path = await find_wheel(path.join(root_dir, 'dist')); + let errcode = 1; + try { + const pyodide = await loadPyodide(); + const FS = pyodide.FS; + setupStreams(FS, pyodide._module.TTY); + FS.mkdir('/test_dir'); + FS.mount(FS.filesystems.NODEFS, {root: path.join(root_dir, 'tests')}, '/test_dir'); + FS.chdir('/test_dir'); + await pyodide.loadPackage(['micropip', 'pytest', 'pytz']); + // language=python + errcode = await pyodide.runPythonAsync(` +import micropip +import importlib + +# ugly hack to get tests to work on arm64 (my m1 mac) +# see https://github.com/pyodide/pyodide/issues/2840 +# import sys; sys.setrecursionlimit(200) + +await micropip.install([ + 'file:${wheel_path}', +]) +importlib.invalidate_caches() + +print('installed packages:', micropip.list()) + +import cramjam + +data = b"bytes" +compressed = cramjam.snappy.compress(data) +decompressed = cramjam.snappy.decompress(compressed) +assert bytes(decompressed) == data + +`); + } catch (e) { + console.error(e); + process.exit(1); + } + process.exit(errcode); +} + +main();