Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

upgrade to apache-arrow v13 #1396

Merged
merged 1 commit into from
Sep 4, 2023
Merged

upgrade to apache-arrow v13 #1396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/benchmarks/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"private": true,
"dependencies": {
"@duckdb/duckdb-wasm": "file:../duckdb-wasm",
"apache-arrow": "^11.0.0",
"apache-arrow": "^13.0.0",
"apache-arrow-3": "npm:apache-arrow@^3.0.0",
"arquero": "^5.2.0",
"buffalo-bench": "^2.0.0",
Expand Down
2 changes: 1 addition & 1 deletion packages/duckdb-wasm-app/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"dependencies": {
"@duckdb/duckdb-wasm": "file:../duckdb-wasm",
"@duckdb/duckdb-wasm-shell": "file:../duckdb-wasm-shell",
"apache-arrow": "^11.0.0",
"apache-arrow": "^13.0.0",
"bootstrap": "^5.3.0",
"classnames": "^2.3.2",
"framer-motion": "^10.16.1",
Expand Down
2 changes: 1 addition & 1 deletion packages/duckdb-wasm/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"csv"
],
"dependencies": {
"apache-arrow": "^12.0.0"
"apache-arrow": "^13.0.0"
},
"devDependencies": {
"@types/emscripten": "^1.39.6",
Expand Down
6 changes: 3 additions & 3 deletions packages/duckdb-wasm/test/regression/github_477.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export function test477(db: () => duckdb.AsyncDuckDB): void {
});
describe('GitHub issues', () => {
it('477', async () => {
// Baseline without cast: we expect the negative decimal values to not work accurately
// Baseline without cast: we expect decimal values to not handle fractional parts correctly
await db().open({
path: ':memory:',
query: {},
Expand All @@ -27,8 +27,8 @@ export function test477(db: () => duckdb.AsyncDuckDB): void {
const resultWithoutCast = await conn.query(`SELECT (-1.9)::DECIMAL(2,1) as decimal`);
expect(resultWithoutCast.schema.fields[0].type.scale).toEqual(1);
expect(resultWithoutCast.schema.fields[0].type.precision).toEqual(2);
// If this assertion breaks, arrow JS was likely updated to handle negative values
expect(resultWithoutCast.toArray()[0]?.decimal == -19).toBe(false);
// Arrow JS now handles negative decimals, but not the fractional part.
expect(resultWithoutCast.toArray()[0]?.decimal == -19).toBe(true);

// Using castDecimalToDouble we force decimals to be cast to doubles, note the inevitable imprecision.
await db().open({
Expand Down
108 changes: 88 additions & 20 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@
# yarn lockfile v1


"@75lb/deep-merge@^1.1.1":
version "1.1.1"
resolved "https://registry.yarnpkg.com/@75lb/deep-merge/-/deep-merge-1.1.1.tgz#3b06155b90d34f5f8cc2107d796f1853ba02fd6d"
integrity sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==
dependencies:
lodash.assignwith "^4.2.0"
typical "^7.1.1"

"@aashutoshrathi/word-wrap@^1.2.3":
version "1.2.6"
resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf"
Expand Down Expand Up @@ -460,7 +468,7 @@
"@duckdb/duckdb-wasm@file:packages/duckdb-wasm":
version "1.11.0"
dependencies:
apache-arrow "^12.0.0"
apache-arrow "^13.0.0"

"@emotion/is-prop-valid@^0.8.2":
version "0.8.8"
Expand Down Expand Up @@ -1009,16 +1017,16 @@
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.5.3.tgz#fa52c147f405d56b2f1dd8780d840aa87ddff629"
integrity sha512-ITI7rbWczR8a/S6qjAW7DMqxqFMjjTo61qZVWJ1ubPvbIQsL5D/TvwjYEalM8Kthpe3hTzOGrF2TGbAu2uyqeA==

"@types/[email protected]":
version "18.14.5"
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.14.5.tgz#4a13a6445862159303fc38586598a9396fc408b3"
integrity sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==

"@types/[email protected]":
version "18.7.23"
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.23.tgz#75c580983846181ebe5f4abc40fe9dfb2d65665f"
integrity sha512-DWNcCHolDq0ZKGizjx2DZjR/PqsYwAcYUJmfMWqtVU2MBMG5Mo+xFZrhGId5r/O5HOuMPyQEcM6KUBp5lBZZBg==

"@types/[email protected]":
version "20.3.0"
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.3.0.tgz#719498898d5defab83c3560f45d8498f58d11938"
integrity sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==

"@types/node@^12.0.4":
version "12.20.27"
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.27.tgz#4141fcad57c332a120591de883e26fe4bb14aaea"
Expand Down Expand Up @@ -1628,21 +1636,21 @@ apache-arrow@^11.0.0:
pad-left "^2.1.0"
tslib "^2.4.0"

apache-arrow@^12.0.0:
version "12.0.1"
resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-12.0.1.tgz#dffd865850d1d94896f1e1aa8332d586fb9e7de1"
integrity sha512-g17ARsc/KEAzViy8PEFsDBlL4ZLx3BesgQCplDLgUWtY0aFWNdEmfaZsbbXVRDfQ21D7vbUKtu0ZWNgcbxDrig==
apache-arrow@^13.0.0:
version "13.0.0"
resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-13.0.0.tgz#8c8c69ca8ec53b804b4e10fe6b57483bda46dfbd"
integrity sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==
dependencies:
"@types/command-line-args" "5.2.0"
"@types/command-line-usage" "5.0.2"
"@types/node" "18.14.5"
"@types/node" "20.3.0"
"@types/pad-left" "2.1.1"
command-line-args "5.2.1"
command-line-usage "6.1.3"
flatbuffers "23.3.3"
command-line-usage "7.0.1"
flatbuffers "23.5.26"
json-bignum "^0.0.3"
pad-left "^2.1.0"
tslib "^2.5.0"
tslib "^2.5.3"

append-transform@^2.0.0:
version "2.0.0"
Expand Down Expand Up @@ -1702,6 +1710,11 @@ array-back@^4.0.1, array-back@^4.0.2:
resolved "https://registry.yarnpkg.com/array-back/-/array-back-4.0.2.tgz#8004e999a6274586beeb27342168652fdb89fa1e"
integrity sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==

array-back@^6.2.2:
version "6.2.2"
resolved "https://registry.yarnpkg.com/array-back/-/array-back-6.2.2.tgz#f567d99e9af88a6d3d2f9dfcc21db6f9ba9fd157"
integrity sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==

array-buffer-byte-length@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz#fabe8bc193fea865f317fe7807085ee0dee5aead"
Expand Down Expand Up @@ -2049,6 +2062,13 @@ caniuse-lite@^1.0.30001286:
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001311.tgz#682ef3f4e617f1a177ad943de59775ed3032e511"
integrity sha512-mleTFtFKfykEeW34EyfhGIFjGCqzhh38Y0LhdQ9aWF+HorZTtdgKV/1hEE0NlFkG2ubvisPV6l400tlbPys98A==

chalk-template@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/chalk-template/-/chalk-template-0.4.0.tgz#692c034d0ed62436b9062c1707fadcd0f753204b"
integrity sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==
dependencies:
chalk "^4.1.2"

chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2:
version "2.4.2"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
Expand Down Expand Up @@ -2242,7 +2262,7 @@ [email protected]:
lodash.camelcase "^4.3.0"
typical "^2.6.1"

[email protected]:
[email protected], command-line-args@^5.2.1:
version "5.2.1"
resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-5.2.1.tgz#c44c32e437a57d7c51157696893c5909e9cec42e"
integrity sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==
Expand Down Expand Up @@ -2272,6 +2292,16 @@ [email protected]:
table-layout "^1.0.2"
typical "^5.2.0"

[email protected], command-line-usage@^7.0.0:
version "7.0.1"
resolved "https://registry.yarnpkg.com/command-line-usage/-/command-line-usage-7.0.1.tgz#e540afef4a4f3bc501b124ffde33956309100655"
integrity sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==
dependencies:
array-back "^6.2.2"
chalk-template "^0.4.0"
table-layout "^3.0.0"
typical "^7.1.1"

commander@^2.20.0:
version "2.20.3"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
Expand Down Expand Up @@ -3510,10 +3540,10 @@ [email protected]:
resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-2.0.4.tgz#034456e29ec480de48bad34f7fc18c03f20c9768"
integrity sha512-4rUFVDPjSoP0tOII34oQf+72NKU7E088U5oX7kwICahft0UB2kOQ9wUzzCp+OHxByERIfxRDCgX5mP8Pjkfl0g==

flatbuffers@23.3.3:
version "23.3.3"
resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-23.3.3.tgz#23654ba7a98d4b866a977ae668fe4f8969f34a66"
integrity sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g==
flatbuffers@23.5.26:
version "23.5.26"
resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-23.5.26.tgz#01358e272a61239f0faf3bfbe4e014f3ace9d746"
integrity sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==

flatted@^3.1.0:
version "3.2.2"
Expand Down Expand Up @@ -5012,6 +5042,11 @@ locate-path@^6.0.0:
dependencies:
p-locate "^5.0.0"

lodash.assignwith@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz#127a97f02adc41751a954d24b0de17e100e038eb"
integrity sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==

lodash.camelcase@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
Expand Down Expand Up @@ -6859,6 +6894,11 @@ std-env@^3.0.1:
resolved "https://registry.yarnpkg.com/std-env/-/std-env-3.0.1.tgz#bc4cbc0e438610197e34c2d79c3df30b491f5182"
integrity sha512-mC1Ps9l77/97qeOZc+HrOL7TIaOboHqMZ24dGVQrlxFcpPpfCHpH+qfUT7Dz+6mlG8+JPA1KfBQo19iC/+Ngcw==

stream-read-all@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/stream-read-all/-/stream-read-all-3.0.1.tgz#60762ae45e61d93ba0978cda7f3913790052ad96"
integrity sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==

streamroller@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/streamroller/-/streamroller-3.0.2.tgz#30418d0eee3d6c93ec897f892ed098e3a81e68b7"
Expand Down Expand Up @@ -7072,6 +7112,19 @@ table-layout@^1.0.2:
typical "^5.2.0"
wordwrapjs "^4.0.0"

table-layout@^3.0.0:
version "3.0.2"
resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-3.0.2.tgz#69c2be44388a5139b48c59cf21e73b488021769a"
integrity sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==
dependencies:
"@75lb/deep-merge" "^1.1.1"
array-back "^6.2.2"
command-line-args "^5.2.1"
command-line-usage "^7.0.0"
stream-read-all "^3.0.1"
typical "^7.1.1"
wordwrapjs "^5.1.0"

tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0:
version "2.2.1"
resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0"
Expand Down Expand Up @@ -7266,11 +7319,16 @@ tslib@^1.12.0:
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==

tslib@^2.0.1, tslib@^2.0.3, tslib@^2.3.0, tslib@^2.4.0, tslib@^2.5.0:
tslib@^2.0.1, tslib@^2.0.3, tslib@^2.3.0, tslib@^2.4.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3"
integrity sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA==

tslib@^2.5.3:
version "2.6.2"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==

[email protected]:
version "1.0.6"
resolved "https://registry.yarnpkg.com/tsscmp/-/tsscmp-1.0.6.tgz#85b99583ac3589ec4bfef825b5000aa911d605eb"
Expand Down Expand Up @@ -7406,6 +7464,11 @@ typical@^5.2.0:
resolved "https://registry.yarnpkg.com/typical/-/typical-5.2.0.tgz#4daaac4f2b5315460804f0acf6cb69c52bb93066"
integrity sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==

typical@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/typical/-/typical-7.1.1.tgz#ba177ab7ab103b78534463ffa4c0c9754523ac1f"
integrity sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==

ua-parser-js@^0.7.30:
version "0.7.30"
resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.30.tgz#4cf5170e8b55ac553fe8b38df3a82f0669671f0b"
Expand Down Expand Up @@ -7910,6 +7973,11 @@ wordwrapjs@^4.0.0:
reduce-flatten "^2.0.0"
typical "^5.2.0"

wordwrapjs@^5.1.0:
version "5.1.0"
resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-5.1.0.tgz#4c4d20446dcc670b14fa115ef4f8fd9947af2b3a"
integrity sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==

"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
Expand Down