Add WASM/JS bindings

.changeset/config.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://unpkg.com/@changesets/[email protected]/schema.json",
+  "changelog": "@changesets/cli/changelog",
+  "commit": false,
+  "fixed": [],
+  "linked": [],
+  "access": "restricted",
+  "baseBranch": "main",
+  "updateInternalDependencies": "patch",
+  "ignore": []
+}

.github/workflows/build_js.yml

@@ -0,0 +1,23 @@
+name: Build and Test JS/WASM
+
+on: [push, pull_request, workflow_dispatch]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_js:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 18
+          registry-url: "https://registry.npmjs.org"
+          cache: yarn
+      - name: Install
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+      - run: yarn install --frozen-lockfile
+      - run: yarn run build
+      - run: yarn run test

.github/workflows/publish.yml

@@ -0,0 +1,30 @@
+name: Publish JS
+on:
+  push:
+    branches:
+      - "main"
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          registry-url: "https://registry.npmjs.org"
+          cache: yarn
+      - name: Install
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+      - run: yarn install --frozen-lockfile
+      - name: Create Release Pull Request or Publish
+        id: changesets
+        uses: changesets/action@v1
+        with:
+          publish: yarn run publish
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -41,3 +41,7 @@ htmlcov
 
 Cargo.lock
 target/
+
+ranks/
+node_modules
+.turbo

Cargo.toml

@@ -1,21 +1,15 @@
-[package]
-name = "tiktoken"
-version = "0.5.1"
-edition = "2021"
-rust-version = "1.57.0"
+[workspace]
 
-[lib]
-name = "_tiktoken"
-crate-type = ["cdylib"]
-
-[dependencies]
-pyo3 = { version = "0.19.0", features = ["extension-module"] }
-
-# tiktoken dependencies
-fancy-regex = "0.11.0"
-regex = "1.8.3"
-rustc-hash = "1.1.0"
-bstr = "1.5.0"
+members = [
+    "core",
+    "python",
+    "wasm"
+]
 
 [profile.release]
 incremental = true
+opt-level = 's'     # Optimize for size
+lto = true          # Enable link-time optimization
+codegen-units = 1   # Reduce number of codegen units to increase optimizations
+panic = 'abort'     # Abort on panic
+strip = true        # Strip symbols from binary*

MANIFEST.in

@@ -1,8 +1,18 @@
 include *.svg
 include *.toml
 include *.md
+exclude yarn.lock
 include Makefile
 global-include py.typed
 recursive-include scripts *.py
 recursive-include tests *.py
-recursive-include src *.rs
+recursive-include core *.rs *.toml
+recursive-include python *.rs *.toml
+recursive-exclude jni *
+recursive-exclude java *
+recursive-exclude js *
+recursive-exclude wasm *
+recursive-exclude static *
+recursive-exclude .changeset *
+recursive-exclude scripts *.ts *.json
+include tiktoken *.json