diff --git a/package-lock.json b/package-lock.json index c0691ce8..603954ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,9 +12,10 @@ "@deck.gl/layers": "^8.9.33", "@deck.gl/react": "^8.9.33", "@geoarrow/deck.gl-layers": "^0.3.0-beta.8", - "apache-arrow": "^14.0.2", + "apache-arrow": "^15.0.0", + "arrow-js-ffi": "^0.4.1", "maplibre-gl": "^3.5.2", - "parquet-wasm": "0.5.0", + "parquet-wasm": "^0.6.0-beta.2", "react": "^18.2.0", "react-dom": "^18.2.0", "react-map-gl": "^7.1.7", @@ -1611,6 +1612,14 @@ "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true }, + "node_modules/@swc/helpers": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.3.tgz", + "integrity": "sha512-FaruWX6KdudYloq1AHD/4nU+UsMTdNE8CKyrseXWEcgjDAbvkwJg2QGPAnfIJLIWsjZOSPLOAykK6fuYp4vp4A==", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@turf/boolean-clockwise": { "version": "5.1.5", "resolved": "https://registry.npmjs.org/@turf/boolean-clockwise/-/boolean-clockwise-5.1.5.tgz", @@ -1678,9 +1687,9 @@ } }, "node_modules/@types/command-line-args": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz", - "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==" + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", + "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==" }, "node_modules/@types/command-line-usage": { "version": "5.0.2", @@ -1742,20 +1751,18 @@ } }, "node_modules/@types/node": { - "version": "20.3.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", - "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==" + "version": "20.11.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.13.tgz", + "integrity": "sha512-5G4zQwdiQBSWYTDAH1ctw2eidqdhMJaNsiIDKHFr55ihz5Trl2qqR8fdrT732yPBho5gkNxXm67OxWFBqX9aPg==", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/offscreencanvas": { "version": "2019.7.3", "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz", "integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==" }, - "node_modules/@types/pad-left": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz", - "integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==" - }, "node_modules/@types/pbf": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/pbf/-/pbf-3.0.5.tgz", @@ -1956,23 +1963,22 @@ } }, "node_modules/apache-arrow": { - "version": "14.0.2", - "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz", - "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==", - "dependencies": { - "@types/command-line-args": "5.2.0", - "@types/command-line-usage": "5.0.2", - "@types/node": "20.3.0", - "@types/pad-left": "2.1.1", - "command-line-args": "5.2.1", - "command-line-usage": "7.0.1", - "flatbuffers": "23.5.26", + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-15.0.0.tgz", + "integrity": "sha512-e6aunxNKM+woQf137ny3tp/xbLjFJS2oGQxQhYGqW6dGeIwNV1jOeEAeR6sS2jwAI2qLO83gYIP2MBz02Gw5Xw==", + "dependencies": { + "@swc/helpers": "^0.5.2", + "@types/command-line-args": "^5.2.1", + "@types/command-line-usage": "^5.0.2", + "@types/node": "^20.6.0", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.1", + "flatbuffers": "^23.5.26", "json-bignum": "^0.0.3", - "pad-left": "^2.1.0", - "tslib": "^2.5.3" + "tslib": "^2.6.2" }, "bin": { - "arrow2csv": "bin/arrow2csv.js" + "arrow2csv": "bin/arrow2csv.cjs" } }, "node_modules/argparse": { @@ -2000,6 +2006,14 @@ "node": ">=6" } }, + "node_modules/arrow-js-ffi": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/arrow-js-ffi/-/arrow-js-ffi-0.4.1.tgz", + "integrity": "sha512-gBE63pHv1lhScKImDKE9JhKyUflGe+1JfZUPNyyVSxpI0P9f2Rxs2NeiQ/Bz9gIjUgk5cjEaDFKc59LC7ER1RQ==", + "peerDependencies": { + "apache-arrow": ">=15" + } + }, "node_modules/assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", @@ -3314,21 +3328,10 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/pad-left": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz", - "integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==", - "dependencies": { - "repeat-string": "^1.5.4" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/parquet-wasm": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/parquet-wasm/-/parquet-wasm-0.5.0.tgz", - "integrity": "sha512-XN3EW+3xf915QgpxCvfCdVYsxDLXz+BckKLDlMo41cbipSYD8x+F/3lGcrWtxdY6uJgAmtb+SX3tS+9PPsplQA==" + "version": "0.6.0-beta.2", + "resolved": "https://registry.npmjs.org/parquet-wasm/-/parquet-wasm-0.6.0-beta.2.tgz", + "integrity": "sha512-RPJxap4+8nhKENDGtD+UAT62tD7bPZi1JbmUtYaqyDu2J2WBGdW2w+HMGo+WY7YKtF0/BD3ZTOU44FpuPr3e2w==" }, "node_modules/path-browserify": { "version": "1.0.1", @@ -3584,14 +3587,6 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" }, - "node_modules/repeat-string": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", - "engines": { - "node": ">=0.10" - } - }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -4084,6 +4079,11 @@ "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", "dev": true }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", diff --git a/package.json b/package.json index 15e03af3..ca328e8b 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,10 @@ "@deck.gl/layers": "^8.9.33", "@deck.gl/react": "^8.9.33", "@geoarrow/deck.gl-layers": "^0.3.0-beta.8", - "apache-arrow": "^14.0.2", + "apache-arrow": "^15.0.0", + "arrow-js-ffi": "^0.4.1", "maplibre-gl": "^3.5.2", - "parquet-wasm": "0.5.0", + "parquet-wasm": "^0.6.0-beta.2", "react": "^18.2.0", "react-dom": "^18.2.0", "react-map-gl": "^7.1.7", diff --git a/src/parquet.ts b/src/parquet.ts index 3e270363..c0ca286a 100644 --- a/src/parquet.ts +++ b/src/parquet.ts @@ -1,11 +1,12 @@ import { useEffect, useState } from "react"; -import _initParquetWasm, { readParquet } from "parquet-wasm/esm/arrow2"; +import _initParquetWasm, { readParquet, wasmMemory } from "parquet-wasm"; import * as arrow from "apache-arrow"; +import { parseTable } from "arrow-js-ffi"; // NOTE: this version must be synced exactly with the parquet-wasm version in // use. -const PARQUET_WASM_VERSION = "0.5.0"; -const PARQUET_WASM_CDN_URL = `https://cdn.jsdelivr.net/npm/parquet-wasm@${PARQUET_WASM_VERSION}/esm/arrow2_bg.wasm`; +const PARQUET_WASM_VERSION = "0.6.0-beta.2"; +const PARQUET_WASM_CDN_URL = `https://cdn.jsdelivr.net/npm/parquet-wasm@${PARQUET_WASM_VERSION}/esm/parquet_wasm_bg.wasm`; let WASM_READY: boolean = false; export async function initParquetWasm() { @@ -28,10 +29,18 @@ export function parseParquet(dataView: DataView): arrow.Table { console.time("readParquet"); // TODO: use arrow-js-ffi for more memory-efficient wasm --> js transfer - const arrowIPCBuffer = readParquet( - new Uint8Array(dataView.buffer), - ).intoIPCStream(); - const arrowTable = arrow.tableFromIPC(arrowIPCBuffer); + const WASM_MEMORY = wasmMemory(); + const wasmTable = readParquet(new Uint8Array(dataView.buffer)); + const wasmFFITable = wasmTable.intoFFI(); + + console.time("arrow js ffi parseTable"); + const arrowTable = parseTable( + WASM_MEMORY.buffer, + Array.from(wasmFFITable.arrayAddrs()), + wasmFFITable.schemaAddr(), + true, + ); + console.timeEnd("arrow js ffi parseTable"); console.timeEnd("readParquet");