diff --git a/package-lock.json b/package-lock.json index 1883341..de8140d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "arquero", - "version": "8.0.0", + "version": "8.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "arquero", - "version": "8.0.0", + "version": "8.0.1", "license": "BSD-3-Clause", "dependencies": { "@uwdata/flechette": "^1.1.2", @@ -17,10 +17,10 @@ "@rollup/plugin-terser": "^0.4.4", "@types/punycode": "^2.1.4", "apache-arrow": "19.0.0", - "eslint": "^9.19.0", + "eslint": "^9.20.1", "mocha": "^11.1.0", "rimraf": "^6.0.1", - "rollup": "^4.34.4", + "rollup": "^4.34.7", "rollup-plugin-bundle-size": "^1.0.3", "tape": "^5.9.0", "typescript": "^5.7.3" @@ -110,9 +110,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.19.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.19.0.tgz", - "integrity": "sha512-rbq9/g38qjfqFLOVPvwjIvFFdNziEC5S65jmjPw5r6A//QH+W91akh9irMwjDN8zKUTak6W9EsAv4m/7Wnw0UQ==", + "version": "9.20.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.20.0.tgz", + "integrity": "sha512-iZA07H9io9Wn836aVTytRaNqh00Sad+EamwOVJT12GTLw1VGMFV/4JaME+JjLtr9fiGaoWgYnS54wrfWsSs4oQ==", "dev": true, "license": "MIT", "engines": { @@ -408,10 +408,38 @@ } } }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.34.7.tgz", + "integrity": "sha512-l6CtzHYo8D2TQ3J7qJNpp3Q1Iye56ssIAtqbM2H8axxCEEwvN7o8Ze9PuIapbxFL3OHrJU2JBX6FIIVnP/rYyw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.34.7.tgz", + "integrity": "sha512-KvyJpFUueUnSp53zhAa293QBYqwm94TgYTIfXyOTtidhm5V0LbLCJQRGkQClYiX3FXDQGSvPxOTD/6rPStMMDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.34.4", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.34.4.tgz", - "integrity": "sha512-drHl+4qhFj+PV/jrQ78p9ch6A0MfNVZScl/nBps5a7u01aGf/GuBRrHnRegA9bP222CBDfjYbFdjkIJ/FurvSQ==", + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.34.7.tgz", + "integrity": "sha512-jq87CjmgL9YIKvs8ybtIC98s/M3HdbqXhllcy9EdLV0yMg1DpxES2gr65nNy7ObNo/vZ/MrOTxt0bE5LinL6mA==", "cpu": [ "arm64" ], @@ -422,6 +450,230 @@ "darwin" ] }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.34.7.tgz", + "integrity": "sha512-rSI/m8OxBjsdnMMg0WEetu/w+LhLAcCDEiL66lmMX4R3oaml3eXz3Dxfvrxs1FbzPbJMaItQiksyMfv1hoIxnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.34.7.tgz", + "integrity": "sha512-oIoJRy3ZrdsXpFuWDtzsOOa/E/RbRWXVokpVrNnkS7npz8GEG++E1gYbzhYxhxHbO2om1T26BZjVmdIoyN2WtA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.34.7.tgz", + "integrity": "sha512-X++QSLm4NZfZ3VXGVwyHdRf58IBbCu9ammgJxuWZYLX0du6kZvdNqPwrjvDfwmi6wFdvfZ/s6K7ia0E5kI7m8Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.34.7.tgz", + "integrity": "sha512-Z0TzhrsNqukTz3ISzrvyshQpFnFRfLunYiXxlCRvcrb3nvC5rVKI+ZXPFG/Aa4jhQa1gHgH3A0exHaRRN4VmdQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.34.7.tgz", + "integrity": "sha512-nkznpyXekFAbvFBKBy4nNppSgneB1wwG1yx/hujN3wRnhnkrYVugMTCBXED4+Ni6thoWfQuHNYbFjgGH0MBXtw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.34.7.tgz", + "integrity": "sha512-KCjlUkcKs6PjOcxolqrXglBDcfCuUCTVlX5BgzgoJHw+1rWH1MCkETLkLe5iLLS9dP5gKC7mp3y6x8c1oGBUtA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.34.7.tgz", + "integrity": "sha512-uFLJFz6+utmpbR313TTx+NpPuAXbPz4BhTQzgaP0tozlLnGnQ6rCo6tLwaSa6b7l6gRErjLicXQ1iPiXzYotjw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loongarch64-gnu": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.34.7.tgz", + "integrity": "sha512-ws8pc68UcJJqCpneDFepnwlsMUFoWvPbWXT/XUrJ7rWUL9vLoIN3GAasgG+nCvq8xrE3pIrd+qLX/jotcLy0Qw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.34.7.tgz", + "integrity": "sha512-vrDk9JDa/BFkxcS2PbWpr0C/LiiSLxFbNOBgfbW6P8TBe9PPHx9Wqbvx2xgNi1TOAyQHQJ7RZFqBiEohm79r0w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.34.7.tgz", + "integrity": "sha512-rB+ejFyjtmSo+g/a4eovDD1lHWHVqizN8P0Hm0RElkINpS0XOdpaXloqM4FBkF9ZWEzg6bezymbpLmeMldfLTw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.34.7.tgz", + "integrity": "sha512-nNXNjo4As6dNqRn7OrsnHzwTgtypfRA3u3AKr0B3sOOo+HkedIbn8ZtFnB+4XyKJojIfqDKmbIzO1QydQ8c+Pw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.34.7.tgz", + "integrity": "sha512-9kPVf9ahnpOMSGlCxXGv980wXD0zRR3wyk8+33/MXQIpQEOpaNe7dEHm5LMfyRZRNt9lMEQuH0jUKj15MkM7QA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.34.7.tgz", + "integrity": "sha512-7wJPXRWTTPtTFDFezA8sle/1sdgxDjuMoRXEKtx97ViRxGGkVQYovem+Q8Pr/2HxiHp74SSRG+o6R0Yq0shPwQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.34.7.tgz", + "integrity": "sha512-MN7aaBC7mAjsiMEZcsJvwNsQVNZShgES/9SzWp1HC9Yjqb5OpexYnRjF7RmE4itbeesHMYYQiAtUAQaSKs2Rfw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.34.7.tgz", + "integrity": "sha512-aeawEKYswsFu1LhDM9RIgToobquzdtSc4jSVqHV8uApz4FVvhFl/mKh92wc8WpFc6aYCothV/03UjY6y7yLgbg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.34.7.tgz", + "integrity": "sha512-4ZedScpxxIrVO7otcZ8kCX1mZArtH2Wfj3uFCxRJ9NO80gg1XV0U/b2f/MKaGwj2X3QopHfoWiDQ917FRpwY3w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@swc/helpers": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.13.tgz", @@ -1405,18 +1657,18 @@ } }, "node_modules/eslint": { - "version": "9.19.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.19.0.tgz", - "integrity": "sha512-ug92j0LepKlbbEv6hD911THhoRHmbdXt2gX+VDABAW/Ir7D3nqKdv5Pf5vtlyY6HQMTEP2skXY43ueqTCWssEA==", + "version": "9.20.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.20.1.tgz", + "integrity": "sha512-m1mM33o6dBUjxl2qb6wv6nGNwCAsns1eKtaQ4l/NPHeTvhiUPbtdfMyktxN4B3fgHIgsYh1VT3V9txblpQHq+g==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", "@eslint/config-array": "^0.19.0", - "@eslint/core": "^0.10.0", + "@eslint/core": "^0.11.0", "@eslint/eslintrc": "^3.2.0", - "@eslint/js": "9.19.0", + "@eslint/js": "9.20.0", "@eslint/plugin-kit": "^0.2.5", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -1491,6 +1743,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/@eslint/core": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.11.0.tgz", + "integrity": "sha512-DWUB2pksgNEb6Bz2fggIy1wh6fGgZP4Xyy/Mt0QZPiloKKXerbqq9D3SBQTlCRYOrcRPu4vuz+CGjwdfqxnoWA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, "node_modules/eslint/node_modules/chalk": { "version": "4.1.2", "dev": true, @@ -3365,9 +3630,9 @@ } }, "node_modules/rollup": { - "version": "4.34.4", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.34.4.tgz", - "integrity": "sha512-spF66xoyD7rz3o08sHP7wogp1gZ6itSq22SGa/IZTcUDXDlOyrShwMwkVSB+BUxFRZZCUYqdb3KWDEOMVQZxuw==", + "version": "4.34.7", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.34.7.tgz", + "integrity": "sha512-8qhyN0oZ4x0H6wmBgfKxJtxM7qS98YJ0k0kNh5ECVtuchIJ7z9IVVvzpmtQyT10PXKMtBxYr1wQ5Apg8RS8kXQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3381,25 +3646,25 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.34.4", - "@rollup/rollup-android-arm64": "4.34.4", - "@rollup/rollup-darwin-arm64": "4.34.4", - "@rollup/rollup-darwin-x64": "4.34.4", - "@rollup/rollup-freebsd-arm64": "4.34.4", - "@rollup/rollup-freebsd-x64": "4.34.4", - "@rollup/rollup-linux-arm-gnueabihf": "4.34.4", - "@rollup/rollup-linux-arm-musleabihf": "4.34.4", - "@rollup/rollup-linux-arm64-gnu": "4.34.4", - "@rollup/rollup-linux-arm64-musl": "4.34.4", - "@rollup/rollup-linux-loongarch64-gnu": "4.34.4", - "@rollup/rollup-linux-powerpc64le-gnu": "4.34.4", - "@rollup/rollup-linux-riscv64-gnu": "4.34.4", - "@rollup/rollup-linux-s390x-gnu": "4.34.4", - "@rollup/rollup-linux-x64-gnu": "4.34.4", - "@rollup/rollup-linux-x64-musl": "4.34.4", - "@rollup/rollup-win32-arm64-msvc": "4.34.4", - "@rollup/rollup-win32-ia32-msvc": "4.34.4", - "@rollup/rollup-win32-x64-msvc": "4.34.4", + "@rollup/rollup-android-arm-eabi": "4.34.7", + "@rollup/rollup-android-arm64": "4.34.7", + "@rollup/rollup-darwin-arm64": "4.34.7", + "@rollup/rollup-darwin-x64": "4.34.7", + "@rollup/rollup-freebsd-arm64": "4.34.7", + "@rollup/rollup-freebsd-x64": "4.34.7", + "@rollup/rollup-linux-arm-gnueabihf": "4.34.7", + "@rollup/rollup-linux-arm-musleabihf": "4.34.7", + "@rollup/rollup-linux-arm64-gnu": "4.34.7", + "@rollup/rollup-linux-arm64-musl": "4.34.7", + "@rollup/rollup-linux-loongarch64-gnu": "4.34.7", + "@rollup/rollup-linux-powerpc64le-gnu": "4.34.7", + "@rollup/rollup-linux-riscv64-gnu": "4.34.7", + "@rollup/rollup-linux-s390x-gnu": "4.34.7", + "@rollup/rollup-linux-x64-gnu": "4.34.7", + "@rollup/rollup-linux-x64-musl": "4.34.7", + "@rollup/rollup-win32-arm64-msvc": "4.34.7", + "@rollup/rollup-win32-ia32-msvc": "4.34.7", + "@rollup/rollup-win32-x64-msvc": "4.34.7", "fsevents": "~2.3.2" } }, diff --git a/package.json b/package.json index 93053a5..f8b5ecb 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "arquero", "type": "module", - "version": "8.0.0", + "version": "8.0.1", "description": "Query processing and transformation of array-backed data tables.", "keywords": [ "data", @@ -45,10 +45,10 @@ "@rollup/plugin-terser": "^0.4.4", "@types/punycode": "^2.1.4", "apache-arrow": "19.0.0", - "eslint": "^9.19.0", + "eslint": "^9.20.1", "mocha": "^11.1.0", "rimraf": "^6.0.1", - "rollup": "^4.34.4", + "rollup": "^4.34.7", "rollup-plugin-bundle-size": "^1.0.3", "tape": "^5.9.0", "typescript": "^5.7.3" diff --git a/src/format/stream/collect.js b/src/format/stream/collect.js index 46aeae6..609d103 100644 --- a/src/format/stream/collect.js +++ b/src/format/stream/collect.js @@ -1,5 +1,6 @@ import { isReadableStream } from '../../util/is-readable-stream.js'; import { isString } from '../../util/is-string.js'; +import { streamIterator } from './stream-iterator.js'; /** * @param {ReadableStream} input @@ -8,7 +9,7 @@ import { isString } from '../../util/is-string.js'; export async function collectBytes(input) { const bytes = []; let size = 0; - for await (const chunk of input) { + for await (const chunk of streamIterator(input)) { size += chunk.length; bytes.push(chunk); } @@ -41,7 +42,7 @@ export async function collectJSON(input) { export async function collectText(input) { if (isString(input)) return input; let text = ''; - for await (const chunk of input) { + for await (const chunk of streamIterator(input)) { text += chunk; } return text; diff --git a/src/format/stream/line-filter-stream.js b/src/format/stream/line-filter-stream.js index 7eb7952..d7735de 100644 --- a/src/format/stream/line-filter-stream.js +++ b/src/format/stream/line-filter-stream.js @@ -22,19 +22,6 @@ function shouldDrop(skip, comment, accessor) { : null; } -// /** -// * @template T -// * @extends {TransformStream} -// */ -// export class LineFilterStream extends TransformStream { -// /** -// * @param {(line: T, index: number) => boolean} drop -// */ -// constructor(drop) { -// super(lineFilterTransformer(drop)); -// } -// } - /** * Returns a new line filter stream transformer. * @template T diff --git a/src/format/stream/parse-json-rows.js b/src/format/stream/parse-json-rows.js index b8c26cb..79a221b 100644 --- a/src/format/stream/parse-json-rows.js +++ b/src/format/stream/parse-json-rows.js @@ -1,4 +1,5 @@ import { pipelineStream, pipelineSync } from './pipeline.js'; +import { streamIterator } from './stream-iterator.js'; export function parseJSONSync(input, columns, transformers) { /** @type {string[][]} */ @@ -10,7 +11,7 @@ export function parseJSONSync(input, columns, transformers) { export async function parseJSONStream(input, columns, transformers) { const stream = pipelineStream(input, transformers); - const iter = stream[Symbol.asyncIterator](); + const iter = streamIterator(stream); let first; do { first = (await iter.next()).value; } while (first.length === 0); const { names, values } = parseJSONColumns(first, columns); diff --git a/src/format/stream/parse-text-rows.js b/src/format/stream/parse-text-rows.js index 36f4c2a..442b945 100644 --- a/src/format/stream/parse-text-rows.js +++ b/src/format/stream/parse-text-rows.js @@ -4,6 +4,7 @@ import { repeat } from '../../util/repeat.js'; import { parseValues } from '../../util/parse-values.js'; import { pipelineStream, pipelineSync } from './pipeline.js'; import { ColumnTable } from '../../table/ColumnTable.js'; +import { streamIterator } from './stream-iterator.js'; /** * Create a table from column names and value arrays. @@ -43,7 +44,7 @@ export async function parseTextRowsStream(input, transformers, options = {}) { // pull batches from stream to meet type inference needs const automax = +options.autoMax || 1000; - const iter = stream[Symbol.asyncIterator](); + const iter = streamIterator(stream); const init = []; let size = 0; while (size < automax) { diff --git a/src/format/stream/stream-iterator.js b/src/format/stream/stream-iterator.js new file mode 100644 index 0000000..2e5eb35 --- /dev/null +++ b/src/format/stream/stream-iterator.js @@ -0,0 +1,24 @@ +import { isFunction } from '../../util/is-function.js'; + +/** + * Return an async iterator for a stream. + * This method is needed to deal with Safari. + * @template T + * @param {ReadableStream} stream + * @returns {AsyncIterator & AsyncIterable} + */ +export function streamIterator(stream) { + if (isFunction(stream[Symbol.asyncIterator])) { + return stream[Symbol.asyncIterator](); + } else { + const reader = stream.getReader(); + return { + next() { + return /** @type {Promise>} */ (reader.read()); + }, + [Symbol.asyncIterator]() { + return this; + } + }; + } +}