diff --git a/package.json b/package.json index 480cb10..34d8719 100644 --- a/package.json +++ b/package.json @@ -32,12 +32,12 @@ "src/" ], "peerDependencies": { - "apache-arrow": ">=14" + "apache-arrow": ">=15" }, "devDependencies": { "@rollup/plugin-terser": "^0.4.3", "@rollup/plugin-typescript": "^11.1.2", - "apache-arrow": "^14", + "apache-arrow": "^15", "prettier": "^3.1.0", "rollup": "^4.1.5", "rollup-plugin-dts": "^6.1.0", diff --git a/src/field.ts b/src/field.ts index a519f07..7a97015 100644 --- a/src/field.ts +++ b/src/field.ts @@ -2,7 +2,7 @@ import * as arrow from "apache-arrow"; import { assert } from "./vector"; -import { LargeBinary, LargeList, LargeUtf8 } from "./types"; +import { LargeList } from "./types"; interface Flags { nullable: boolean; @@ -27,9 +27,9 @@ const formatMapping: Record = { f: new arrow.Float32(), g: new arrow.Float64(), z: new arrow.Binary(), - Z: new LargeBinary(), + Z: new arrow.LargeBinary(), u: new arrow.Utf8(), - U: new LargeUtf8(), + U: new arrow.LargeUtf8(), tdD: new arrow.DateDay(), tdm: new arrow.DateMillisecond(), tts: new arrow.TimeSecond(), diff --git a/src/types.ts b/src/types.ts index ce97e76..787e3eb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,11 +21,11 @@ export enum Type { FixedSizeList = 16 /** Fixed-size list. Each value occupies the same number of bytes */, Map = 17 /** Map of named logical types */, Duration = 18 /** Measure of elapsed time in either seconds, milliseconds, microseconds or nanoseconds. */, + LargeBinary = 19 /** Large variable-length bytes (no guarantee of UTF8-ness) */, + LargeUtf8 = 20 /** Large variable-length string as List */, - // These 3 are not included in the upstream enum + // Not yet included in the upstream enum LargeList = 30, - LargeBinary = 31, - LargeUtf8 = 32, Dictionary = -1 /** Dictionary aka Category type */, Int8 = -2, @@ -61,13 +61,10 @@ export class LargeList extends DataType< { [0]: T } > { constructor(child: Field) { - super(); + super(Type.LargeList); this.children = [child]; } public declare readonly children: Field[]; - public get typeId() { - return Type.LargeList as Type.LargeList; // Type.List as Type.List; - } public toString() { return `LargeList<${this.valueType}>`; } @@ -86,48 +83,6 @@ export class LargeList extends DataType< })(LargeList.prototype); } -// @ts-expect-error Type 'Type.LargeBinary' does not satisfy the constraint 'Type' -export class LargeBinary extends DataType { - constructor() { - super(); - } - public get typeId() { - return Type.LargeBinary as Type.LargeBinary; - } - public toString() { - return `Binary`; - } - protected static [Symbol.toStringTag] = ((proto: LargeBinary) => { - (proto).ArrayType = Uint8Array; - return (proto[Symbol.toStringTag] = "LargeBinary"); - })(LargeBinary.prototype); -} - -// @ts-expect-error Type 'Type.LargeUtf8' does not satisfy the constraint 'Type' -export class LargeUtf8 extends DataType { - constructor() { - super(); - } - public get typeId() { - return Type.LargeUtf8 as Type.LargeUtf8; - } - public toString() { - return `LargeUtf8`; - } - protected static [Symbol.toStringTag] = ((proto: LargeUtf8) => { - (proto).ArrayType = Uint8Array; - return (proto[Symbol.toStringTag] = "LargeUtf8"); - })(LargeUtf8.prototype); -} - export function isLargeList(x: any): x is LargeList { return x?.typeId === Type.LargeList; } - -export function isLargeBinary(x: any): x is LargeBinary { - return x?.typeId === Type.LargeBinary; -} - -export function isLargeUtf8(x: any): x is LargeUtf8 { - return x?.typeId === Type.LargeUtf8; -} diff --git a/src/vector.ts b/src/vector.ts index 34c8fbf..431f5f7 100644 --- a/src/vector.ts +++ b/src/vector.ts @@ -1,6 +1,6 @@ import * as arrow from "apache-arrow"; import { DataType } from "apache-arrow"; -import { LargeList, isLargeBinary, isLargeList, isLargeUtf8 } from "./types"; +import { LargeList, isLargeList } from "./types"; type NullBitmap = Uint8Array | null | undefined; @@ -340,7 +340,7 @@ export function parseData( }); } - if (isLargeBinary(dataType)) { + if (DataType.isLargeBinary(dataType)) { const [validityPtr, offsetsPtr, dataPtr] = bufferPtrs; const nullBitmap = parseNullBitmap( dataView.buffer, @@ -420,7 +420,7 @@ export function parseData( }); } - if (isLargeUtf8(dataType)) { + if (DataType.isLargeUtf8(dataType)) { const [validityPtr, offsetsPtr, dataPtr] = bufferPtrs; const nullBitmap = parseNullBitmap( dataView.buffer, diff --git a/tests/ffi.test.ts b/tests/ffi.test.ts index a48668e..e54654f 100644 --- a/tests/ffi.test.ts +++ b/tests/ffi.test.ts @@ -95,7 +95,7 @@ describe("primitive types non-null", (t) => { expect(field.name, "col1"); expect(field.typeId).toStrictEqual(fixture.dataType.typeId); - expect(field.nullable).toBeFalsy(); + expect(field.nullable).toBeTruthy(); const arrayPtr = ffiTable.arrayAddr(0, 0); const wasmVector = parseVector( diff --git a/yarn.lock b/yarn.lock index a5e47d6..7046ea1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -463,6 +463,15 @@ __metadata: languageName: node linkType: hard +"@swc/helpers@npm:^0.5.2": + version: 0.5.3 + resolution: "@swc/helpers@npm:0.5.3" + dependencies: + tslib: "npm:^2.4.0" + checksum: 7003b9dafad38e27a4474a3dcf78d2aee85ce4bb7ee171ecc2d5b533768fb4f8fc9dbd6a1c3a9df5f8001e61ff416b93681e0a7dcbdc0df109b3e09c50cd4615 + languageName: node + linkType: hard + "@tootallnate/once@npm:2": version: 2.0.0 resolution: "@tootallnate/once@npm:2.0.0" @@ -514,17 +523,17 @@ __metadata: languageName: node linkType: hard -"@types/command-line-args@npm:5.2.0": - version: 5.2.0 - resolution: "@types/command-line-args@npm:5.2.0" - checksum: 82a9308c9219fb19619dba302a4c3ccd039c8974b4b2d46265b9c99333168207743cba9c02fd85178d2f7f9a1e6d76c472e3ac45b1616b1f5926877c95ce0169 +"@types/command-line-args@npm:^5.2.1": + version: 5.2.3 + resolution: "@types/command-line-args@npm:5.2.3" + checksum: 3a9bc58fd26e546391f6369dd28c03d59349dc4ac39eada1a5c39cc3578e02e4aac222615170e0db79b198ffba2af84fdbdda46e08c6edc4da42bc17ea85200f languageName: node linkType: hard -"@types/command-line-usage@npm:5.0.2": - version: 5.0.2 - resolution: "@types/command-line-usage@npm:5.0.2" - checksum: 5553012915df5fcf95e1ef5363fa2d0bc8543d80aa972916cb8e0ab99c0581e573af6e5e9426560367353e86bea7ff11c73d7e1d979190fe8799f6229b6d1e32 +"@types/command-line-usage@npm:^5.0.2": + version: 5.0.4 + resolution: "@types/command-line-usage@npm:5.0.4" + checksum: 67840ebf4bcfee200c07d978669ad596fe2adc350fd5c19d44ec2248623575d96ec917f513d1d59453f8f57e879133861a4cc41c20045c07f6c959f1fcaac7ad languageName: node linkType: hard @@ -542,17 +551,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:20.3.0": - version: 20.3.0 - resolution: "@types/node@npm:20.3.0" - checksum: f717d92c29c4877db394b604771b3734216f013312f93252f72c2018aabe8083be905fbcf0644c859938c8183b6e0245faaeaab94c9e78268b87a449bc6ef4aa - languageName: node - linkType: hard - -"@types/pad-left@npm:2.1.1": - version: 2.1.1 - resolution: "@types/pad-left@npm:2.1.1" - checksum: 134a50786e5515e78b7b6a69150e768e877c6986ef637f921b9602d95cc73c0bb44ddbb160d7eb96706be792dc643e8b82617ef9760f4a5fa7f99cd06df03e50 +"@types/node@npm:^20.6.0": + version: 20.11.5 + resolution: "@types/node@npm:20.11.5" + dependencies: + undici-types: "npm:~5.26.4" + checksum: 6d18cec852f5cfbed3ec42b5c01c026e7a3f9da540d6e3d6738d4cee9979fb308cf27b6df7ba40a6553e7bc82e678f0ef53ba6e6ad52e5b86bd97b7783c2a42c languageName: node linkType: hard @@ -717,23 +721,22 @@ __metadata: languageName: node linkType: hard -"apache-arrow@npm:^14": - version: 14.0.1 - resolution: "apache-arrow@npm:14.0.1" +"apache-arrow@npm:^15": + version: 15.0.0 + resolution: "apache-arrow@npm:15.0.0" dependencies: - "@types/command-line-args": "npm:5.2.0" - "@types/command-line-usage": "npm:5.0.2" - "@types/node": "npm:20.3.0" - "@types/pad-left": "npm:2.1.1" - command-line-args: "npm:5.2.1" - command-line-usage: "npm:7.0.1" - flatbuffers: "npm:23.5.26" + "@swc/helpers": "npm:^0.5.2" + "@types/command-line-args": "npm:^5.2.1" + "@types/command-line-usage": "npm:^5.0.2" + "@types/node": "npm:^20.6.0" + command-line-args: "npm:^5.2.1" + command-line-usage: "npm:^7.0.1" + flatbuffers: "npm:^23.5.26" json-bignum: "npm:^0.0.3" - pad-left: "npm:^2.1.0" - tslib: "npm:^2.5.3" + tslib: "npm:^2.6.2" bin: - arrow2csv: bin/arrow2csv.js - checksum: 9049c6fab8f5a0a7da737f6ce1c9e13dd5d97830e89c13f6d2a65b8b1b99bb57d7c678d580f4ca2c0ecb7ee13119f7d3c6b72f80415e4ecf36de5323e822ddc1 + arrow2csv: bin/arrow2csv.cjs + checksum: 706921d06c89841d1973a0a7d53150862d065f503091d824173535d0d58322e7b2600932fd3bfeb62be99d94e91acfef6961161776fcf51a18a9cc23172ca243 languageName: node linkType: hard @@ -781,7 +784,7 @@ __metadata: dependencies: "@rollup/plugin-terser": "npm:^0.4.3" "@rollup/plugin-typescript": "npm:^11.1.2" - apache-arrow: "npm:^14" + apache-arrow: "npm:^15" prettier: "npm:^3.1.0" rollup: "npm:^4.1.5" rollup-plugin-dts: "npm:^6.1.0" @@ -790,7 +793,7 @@ __metadata: typescript: "npm:^5.2.2" vitest: "npm:^0.34.6" peerDependencies: - apache-arrow: ">=14" + apache-arrow: ">=15" languageName: unknown linkType: soft @@ -970,7 +973,7 @@ __metadata: languageName: node linkType: hard -"command-line-args@npm:5.2.1, command-line-args@npm:^5.2.1": +"command-line-args@npm:^5.2.1": version: 5.2.1 resolution: "command-line-args@npm:5.2.1" dependencies: @@ -982,7 +985,7 @@ __metadata: languageName: node linkType: hard -"command-line-usage@npm:7.0.1, command-line-usage@npm:^7.0.0": +"command-line-usage@npm:^7.0.0, command-line-usage@npm:^7.0.1": version: 7.0.1 resolution: "command-line-usage@npm:7.0.1" dependencies: @@ -1233,7 +1236,7 @@ __metadata: languageName: node linkType: hard -"flatbuffers@npm:23.5.26": +"flatbuffers@npm:^23.5.26": version: 23.5.26 resolution: "flatbuffers@npm:23.5.26" checksum: 69e0903caa3aa8d2bdbf1557995835fd097806d413abc9b4b7440301a8ae2bd427220be1d613af80609c38d38cdbdeb459706903080433b509ceb64faf91b711 @@ -1889,15 +1892,6 @@ __metadata: languageName: node linkType: hard -"pad-left@npm:^2.1.0": - version: 2.1.0 - resolution: "pad-left@npm:2.1.0" - dependencies: - repeat-string: "npm:^1.5.4" - checksum: 446c858bf3f7171cd164569aec074f3fe05ed9bcd03bc4016bb02b14733060344e54965e61667b46b67242ff77ae13d2ef9c35b6b039aed658ef5e9d1872f5d4 - languageName: node - linkType: hard - "path-is-absolute@npm:^1.0.0": version: 1.0.1 resolution: "path-is-absolute@npm:1.0.1" @@ -2036,13 +2030,6 @@ __metadata: languageName: node linkType: hard -"repeat-string@npm:^1.5.4": - version: 1.6.1 - resolution: "repeat-string@npm:1.6.1" - checksum: 87fa21bfdb2fbdedc44b9a5b118b7c1239bdd2c2c1e42742ef9119b7d412a5137a1d23f1a83dc6bb686f4f27429ac6f542e3d923090b44181bafa41e8ac0174d - languageName: node - linkType: hard - "resolve@npm:^1.22.1": version: 1.22.3 resolution: "resolve@npm:1.22.3" @@ -2513,7 +2500,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.5.3": +"tslib@npm:^2.4.0, tslib@npm:^2.6.2": version: 2.6.2 resolution: "tslib@npm:2.6.2" checksum: e03a8a4271152c8b26604ed45535954c0a45296e32445b4b87f8a5abdb2421f40b59b4ca437c4346af0f28179780d604094eb64546bee2019d903d01c6c19bdb @@ -2568,6 +2555,13 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~5.26.4": + version: 5.26.5 + resolution: "undici-types@npm:5.26.5" + checksum: bb673d7876c2d411b6eb6c560e0c571eef4a01c1c19925175d16e3a30c4c428181fb8d7ae802a261f283e4166a0ac435e2f505743aa9e45d893f9a3df017b501 + languageName: node + linkType: hard + "unique-filename@npm:^3.0.0": version: 3.0.0 resolution: "unique-filename@npm:3.0.0"