diff --git a/api/deno.lock b/api/deno.lock index 7b27500db5..ff1fb7d8c1 100644 --- a/api/deno.lock +++ b/api/deno.lock @@ -4015,6 +4015,9 @@ } } }, + "redirects": { + "https://esm.sh/v135/@types/stream-json@~1/filters/Pick~.d.ts": "https://esm.sh/v135/@types/stream-json@1.7.8/filters/Pick~.d.ts" + }, "remote": { "https://cdn.sheetjs.com/xlsx-0.20.3/package/xlsx.mjs": "1a0fb062ee9781b13f6687371b202aaefc53b6ce55b530c027e01f9c087b77db", "https://deno.land/std@0.214.0/assert/assert.ts": "bec068b2fccdd434c138a555b19a2c2393b71dfaada02b7d568a01541e67cdc5", @@ -4152,6 +4155,7 @@ "https://deno.land/std@0.224.0/assert/unimplemented.ts": "8c55a5793e9147b4f1ef68cd66496b7d5ba7a9e7ca30c6da070c1a58da723d73", "https://deno.land/std@0.224.0/assert/unreachable.ts": "5ae3dbf63ef988615b93eb08d395dda771c96546565f9e521ed86f6510c29e19", "https://deno.land/std@0.224.0/async/delay.ts": "f90dd685b97c2f142b8069082993e437b1602b8e2561134827eeb7c12b95c499", + "https://deno.land/std@0.224.0/bytes/concat.ts": "86161274b5546a02bdb3154652418efe7af8c9310e8d54107a68aaa148e0f5ed", "https://deno.land/std@0.224.0/collections/_utils.ts": "b2ec8ada31b5a72ebb1d99774b849b4c09fe4b3a38d07794bd010bd218a16e0b", "https://deno.land/std@0.224.0/collections/aggregate_groups.ts": "c63a57a16e87537ee71df1174a111e2c4cd05f0d4be83db701c8740e81bd157c", "https://deno.land/std@0.224.0/collections/associate_by.ts": "8c09c6a66769480f3181bd7b582796aed85fa71e2595c1b3d73684b9caae33b1", @@ -4204,18 +4208,46 @@ "https://deno.land/std@0.224.0/crypto/mod.ts": "9148fb70ca3d64977e9487b2002d3b1026e8ad8a2078774b807586ba3c77e3bb", "https://deno.land/std@0.224.0/crypto/timing_safe_equal.ts": "bc3622b5aec05e2d8b735bf60633425c34333c06cfb6c4a9f102e4a0f3931ced", "https://deno.land/std@0.224.0/crypto/unstable_keystack.ts": "c2a6f6ed67a4e78745e3c9b490ebb7c12f6066f5c2fe0c69d353961909dc82dd", + "https://deno.land/std@0.224.0/csv/_io.ts": "568718b7eb4cf3ba140b0690a00ef5a8416ceae9a4116f386114c83717bd8a9e", + "https://deno.land/std@0.224.0/csv/csv_parse_stream.ts": "f06696ed1fbda5cb75426dff735b9cc91227eb0fa5367591b98bf88b99ae3048", + "https://deno.land/std@0.224.0/csv/csv_stringify_stream.ts": "b564e43b884eb9ab8e1efee8c30a0868a1a9328a33c32a7dd436dc26f63954ac", + "https://deno.land/std@0.224.0/csv/mod.ts": "289ecf79b3609f63dfc1d39a7c791f8f14b73b29fb65bbb6b3e3e2bd0cddc75d", + "https://deno.land/std@0.224.0/csv/parse.ts": "e2d5e82e19b71a968f5fcd3b76b438ee6f646fe8e948c4c2d3b00ad9cd0a659a", + "https://deno.land/std@0.224.0/csv/stringify.ts": "9e0badcbc32fc63ee089d2dfbc5721650d5766ef0a554f07243ab8f03f7c3127", "https://deno.land/std@0.224.0/datetime/constants.ts": "5df80a84e301da6db5793804122c034d2d090da31f1e1c5fdaa831fc70a45da7", "https://deno.land/std@0.224.0/datetime/difference.ts": "8da08374910cae7d316cd7669912b70465f36d4e159e2c9f525a441f91fa9c7a", "https://deno.land/std@0.224.0/encoding/_util.ts": "beacef316c1255da9bc8e95afb1fa56ed69baef919c88dc06ae6cb7a6103d376", "https://deno.land/std@0.224.0/encoding/base64.ts": "dd59695391584c8ffc5a296ba82bcdba6dd8a84d41a6a539fbee8e5075286eaf", "https://deno.land/std@0.224.0/encoding/base64url.ts": "ef40e0f18315ab539f17cebcc32839779e018d86dea9df39d94d302f342a1713", "https://deno.land/std@0.224.0/fmt/colors.ts": "508563c0659dd7198ba4bbf87e97f654af3c34eb56ba790260f252ad8012e1c5", + "https://deno.land/std@0.224.0/fs/_create_walk_entry.ts": "5d9d2aaec05bcf09a06748b1684224d33eba7a4de24cf4cf5599991ca6b5b412", + "https://deno.land/std@0.224.0/fs/_get_file_info_type.ts": "da7bec18a7661dba360a1db475b826b18977582ce6fc9b25f3d4ee0403fe8cbd", + "https://deno.land/std@0.224.0/fs/_is_same_path.ts": "709c95868345fea051c58b9e96af95cff94e6ae98dfcff2b66dee0c212c4221f", + "https://deno.land/std@0.224.0/fs/_is_subdir.ts": "c68b309d46cc8568ed83c000f608a61bbdba0943b7524e7a30f9e450cf67eecd", + "https://deno.land/std@0.224.0/fs/_to_path_string.ts": "29bfc9c6c112254961d75cbf6ba814d6de5349767818eb93090cecfa9665591e", + "https://deno.land/std@0.224.0/fs/copy.ts": "7ab12a16adb65d155d4943c88081ca16ce3b0b5acada64c1ce93800653678039", + "https://deno.land/std@0.224.0/fs/empty_dir.ts": "e400e96e1d2c8c558a5a1712063bd43939e00619c1d1cc29959babc6f1639418", + "https://deno.land/std@0.224.0/fs/ensure_dir.ts": "51a6279016c65d2985f8803c848e2888e206d1b510686a509fa7cc34ce59d29f", + "https://deno.land/std@0.224.0/fs/ensure_file.ts": "67608cf550529f3d4aa1f8b6b36bf817bdc40b14487bf8f60e61cbf68f507cf3", + "https://deno.land/std@0.224.0/fs/ensure_link.ts": "5c98503ebfa9cc05e2f2efaa30e91e60b4dd5b43ebbda82f435c0a5c6e3ffa01", + "https://deno.land/std@0.224.0/fs/ensure_symlink.ts": "cafe904cebacb9a761977d6dbf5e3af938be946a723bb394080b9a52714fafe4", + "https://deno.land/std@0.224.0/fs/eol.ts": "18c4ac009d0318504c285879eb7f47942643f13619e0ff070a0edc59353306bd", "https://deno.land/std@0.224.0/fs/exists.ts": "3d38cb7dcbca3cf313be343a7b8af18a87bddb4b5ca1bd2314be12d06533b50f", + "https://deno.land/std@0.224.0/fs/expand_glob.ts": "2e428d90acc6676b2aa7b5c78ef48f30641b13f1fe658e7976c9064fb4b05309", + "https://deno.land/std@0.224.0/fs/mod.ts": "c25e6802cbf27f3050f60b26b00c2d8dba1cb7fcdafe34c66006a7473b7b34d4", + "https://deno.land/std@0.224.0/fs/move.ts": "ca205d848908d7f217353bc5c623627b1333490b8b5d3ef4cab600a700c9bd8f", + "https://deno.land/std@0.224.0/fs/walk.ts": "cddf87d2705c0163bff5d7767291f05b0f46ba10b8b28f227c3849cace08d303", "https://deno.land/std@0.224.0/internal/diff.ts": "6234a4b493ebe65dc67a18a0eb97ef683626a1166a1906232ce186ae9f65f4e6", "https://deno.land/std@0.224.0/internal/format.ts": "0a98ee226fd3d43450245b1844b47003419d34d210fa989900861c79820d21c2", "https://deno.land/std@0.224.0/internal/mod.ts": "534125398c8e7426183e12dc255bb635d94e06d0f93c60a297723abe69d3b22e", "https://deno.land/std@0.224.0/io/types.ts": "acecb3074c730b5ff487ba4fe9ce51e67bd982aa07c95e5f5679b7b2f24ad129", "https://deno.land/std@0.224.0/io/write_all.ts": "24aac2312bb21096ae3ae0b102b22c26164d3249dff96dbac130958aa736f038", + "https://deno.land/std@0.224.0/json/_common.ts": "6444c6ea166514eb379f778d9216c1d8eb159f97c753aeeb1fc44cd091e30544", + "https://deno.land/std@0.224.0/json/common.ts": "33f1a4f39a45e2f9f357823fd0b5cf88b63fb4784b8c9a28f8120f70a20b23e9", + "https://deno.land/std@0.224.0/json/concatenated_json_parse_stream.ts": "13a707615e03e5ea93ac81e5f00420e3b7764c4e3fa88043e63bbac87ebe62ce", + "https://deno.land/std@0.224.0/json/json_parse_stream.ts": "2740652ea73726cd0f2edc89188b35d64a1ec15dc8cf7fd87db52a0170bc182c", + "https://deno.land/std@0.224.0/json/json_stringify_stream.ts": "269633e63d4e38ab3ba31e76a4292d11b9eb3151e26ff4f49dee1c264b4878fa", + "https://deno.land/std@0.224.0/json/mod.ts": "acd3e39a6c68c70ee7fa93991a8a8f02d799f7394db3cef09594e2dd8fd69814", "https://deno.land/std@0.224.0/log/_config.ts": "489e11b6d3c917bf5fc954c5e914c095d3480efd924d1e85f2fc576468581c54", "https://deno.land/std@0.224.0/log/_state.ts": "314c0c31ab9c8f4fb33326ad446757d35f75e5bb21746b7720ed4e3f3a939da1", "https://deno.land/std@0.224.0/log/base_handler.ts": "f03f013dac9c1a226aab60c6f5751b3131cc4f2808720715285e0dab5697a54e", @@ -4247,7 +4279,17 @@ "https://deno.land/std@0.224.0/path/_common/strip_trailing_separators.ts": "7024a93447efcdcfeaa9339a98fa63ef9d53de363f1fbe9858970f1bba02655a", "https://deno.land/std@0.224.0/path/_common/to_file_url.ts": "7f76adbc83ece1bba173e6e98a27c647712cab773d3f8cbe0398b74afc817883", "https://deno.land/std@0.224.0/path/_interface.ts": "8dfeb930ca4a772c458a8c7bbe1e33216fe91c253411338ad80c5b6fa93ddba0", + "https://deno.land/std@0.224.0/path/_os.ts": "8fb9b90fb6b753bd8c77cfd8a33c2ff6c5f5bc185f50de8ca4ac6a05710b2c15", + "https://deno.land/std@0.224.0/path/basename.ts": "7ee495c2d1ee516ffff48fb9a93267ba928b5a3486b550be73071bc14f8cc63e", + "https://deno.land/std@0.224.0/path/constants.ts": "0c206169ca104938ede9da48ac952de288f23343304a1c3cb6ec7625e7325f36", + "https://deno.land/std@0.224.0/path/dirname.ts": "85bd955bf31d62c9aafdd7ff561c4b5fb587d11a9a5a45e2b01aedffa4238a7c", + "https://deno.land/std@0.224.0/path/from_file_url.ts": "911833ae4fd10a1c84f6271f36151ab785955849117dc48c6e43b929504ee069", + "https://deno.land/std@0.224.0/path/glob_to_regexp.ts": "7f30f0a21439cadfdae1be1bf370880b415e676097fda584a63ce319053b5972", + "https://deno.land/std@0.224.0/path/is_absolute.ts": "4791afc8bfd0c87f0526eaa616b0d16e7b3ab6a65b62942e50eac68de4ef67d7", "https://deno.land/std@0.224.0/path/is_glob.ts": "a65f6195d3058c3050ab905705891b412ff942a292bcbaa1a807a74439a14141", + "https://deno.land/std@0.224.0/path/join.ts": "ae2ec5ca44c7e84a235fd532e4a0116bfb1f2368b394db1c4fb75e3c0f26a33a", + "https://deno.land/std@0.224.0/path/join_globs.ts": "5b3bf248b93247194f94fa6947b612ab9d3abd571ca8386cf7789038545e54a0", + "https://deno.land/std@0.224.0/path/normalize.ts": "4155743ccceeed319b350c1e62e931600272fad8ad00c417b91df093867a8352", "https://deno.land/std@0.224.0/path/posix/_util.ts": "1e3937da30f080bfc99fe45d7ed23c47dd8585c5e473b2d771380d3a6937cf9d", "https://deno.land/std@0.224.0/path/posix/basename.ts": "d2fa5fbbb1c5a3ab8b9326458a8d4ceac77580961b3739cd5bfd1d3541a3e5f0", "https://deno.land/std@0.224.0/path/posix/common.ts": "26f60ccc8b2cac3e1613000c23ac5a7d392715d479e5be413473a37903a2b5d4", @@ -4269,6 +4311,23 @@ "https://deno.land/std@0.224.0/path/posix/resolve.ts": "08b699cfeee10cb6857ccab38fa4b2ec703b0ea33e8e69964f29d02a2d5257cf", "https://deno.land/std@0.224.0/path/posix/to_file_url.ts": "7aa752ba66a35049e0e4a4be5a0a31ac6b645257d2e031142abb1854de250aaf", "https://deno.land/std@0.224.0/path/posix/to_namespaced_path.ts": "28b216b3c76f892a4dca9734ff1cc0045d135532bfd9c435ae4858bfa5a2ebf0", + "https://deno.land/std@0.224.0/path/resolve.ts": "a6f977bdb4272e79d8d0ed4333e3d71367cc3926acf15ac271f1d059c8494d8d", + "https://deno.land/std@0.224.0/path/windows/_util.ts": "d5f47363e5293fced22c984550d5e70e98e266cc3f31769e1710511803d04808", + "https://deno.land/std@0.224.0/path/windows/basename.ts": "6bbc57bac9df2cec43288c8c5334919418d784243a00bc10de67d392ab36d660", + "https://deno.land/std@0.224.0/path/windows/constants.ts": "5afaac0a1f67b68b0a380a4ef391bf59feb55856aa8c60dfc01bd3b6abb813f5", + "https://deno.land/std@0.224.0/path/windows/dirname.ts": "33e421be5a5558a1346a48e74c330b8e560be7424ed7684ea03c12c21b627bc9", + "https://deno.land/std@0.224.0/path/windows/from_file_url.ts": "ced2d587b6dff18f963f269d745c4a599cf82b0c4007356bd957cb4cb52efc01", + "https://deno.land/std@0.224.0/path/windows/glob_to_regexp.ts": "e45f1f89bf3fc36f94ab7b3b9d0026729829fabc486c77f414caebef3b7304f8", + "https://deno.land/std@0.224.0/path/windows/is_absolute.ts": "4a8f6853f8598cf91a835f41abed42112cebab09478b072e4beb00ec81f8ca8a", + "https://deno.land/std@0.224.0/path/windows/join.ts": "8d03530ab89195185103b7da9dfc6327af13eabdcd44c7c63e42e27808f50ecf", + "https://deno.land/std@0.224.0/path/windows/join_globs.ts": "a9475b44645feddceb484ee0498e456f4add112e181cb94042cdc6d47d1cdd25", + "https://deno.land/std@0.224.0/path/windows/normalize.ts": "78126170ab917f0ca355a9af9e65ad6bfa5be14d574c5fb09bb1920f52577780", + "https://deno.land/std@0.224.0/path/windows/normalize_glob.ts": "9c87a829b6c0f445d03b3ecadc14492e2864c3ebb966f4cea41e98326e4435c6", + "https://deno.land/std@0.224.0/path/windows/resolve.ts": "8dae1dadfed9d46ff46cc337c9525c0c7d959fb400a6308f34595c45bdca1972", + "https://deno.land/std@0.224.0/streams/_common.ts": "948735ef6d140cd6916dca861197b88fc57db52c2f923c392b7a14033d8fed4b", + "https://deno.land/std@0.224.0/streams/delimiter_stream.ts": "4e4050740ff27a8824defa6c96126229ef9d794c4ace4ef9cabb10b5ad4a5d14", + "https://deno.land/std@0.224.0/streams/text_delimiter_stream.ts": "94dfc900204e306496c1b58c80473db57b6097afdcb8ea9eaff453a193a659f1", + "https://deno.land/std@0.224.0/streams/to_transform_stream.ts": "4c4836455ef89bab9ece55975ee3a819f07d3d8b0e43101ec7f4ed033c8a2b61", "https://deno.land/std@0.224.0/testing/_test_suite.ts": "f10a8a6338b60c403f07a76f3f46bdc9f1e1a820c0a1decddeb2949f7a8a0546", "https://deno.land/std@0.224.0/testing/bdd.ts": "3e4de4ff6d8f348b5574661cef9501b442046a59079e201b849d0e74120d476b", "https://deno.land/std@0.224.0/testing/mock.ts": "a963181c2860b6ba3eb60e08b62c164d33cf5da7cd445893499b2efda20074db", @@ -4388,6 +4447,9 @@ "https://deno.land/x/s3_lite_client@0.7.0/object-uploader.ts": "b4bad0d771d79b2bb23b8cab0e6f7be85a2390e18957c612fd5cda11c39f55b0", "https://deno.land/x/s3_lite_client@0.7.0/signing.ts": "2ba77aac07a7c94267e83d285bbd33fdb3253dfa32b035df62479d6b224bb748", "https://deno.land/x/s3_lite_client@0.7.0/transform-chunk-sizes.ts": "cecc1167ba366d086a13c754be6ed86717d6b0b27c779c4c766621435a697045", - "https://deno.land/x/s3_lite_client@0.7.0/xml-parser.ts": "de925493369718cab6f26413fbbada18eec74aa6eaf0598d77c7296f5fdfd8a9" + "https://deno.land/x/s3_lite_client@0.7.0/xml-parser.ts": "de925493369718cab6f26413fbbada18eec74aa6eaf0598d77c7296f5fdfd8a9", + "https://esm.sh/stream-json@1.8.0/filters/Pick": "b1803979c58530e935507651dc9ffb1378345c038a9dc42821879e241a071b34", + "https://esm.sh/v135/stream-chain@2.2.5/denonext/stream-chain.mjs": "7964e6c089c29cbc2f480159d06cfb475758a1bf5f7ab4801d596934af94c4f8", + "https://esm.sh/v135/stream-json@1.8.0/denonext/filters/Pick.js": "f61bc931915aaa8a607d30eae09c93dde5a1f3fb28220021f16c53899ac4d805" } } diff --git a/api/src/db/migrations/20241122000000-alter-fraudcheck_label.sql b/api/src/db/migrations/20241122000000-alter-fraudcheck_label.sql index 35173efb97..3e6390f3ac 100644 --- a/api/src/db/migrations/20241122000000-alter-fraudcheck_label.sql +++ b/api/src/db/migrations/20241122000000-alter-fraudcheck_label.sql @@ -1 +1 @@ -ALTER TABLE FRAUDCHECK.LABELS DROP COLUMN geo_code; +ALTER TABLE FRAUDCHECK.LABELS DROP COLUMN IF EXISTS geo_code CASCADE; diff --git a/api/src/deps.ts b/api/src/deps.ts index 2732edae80..37d7413651 100644 --- a/api/src/deps.ts +++ b/api/src/deps.ts @@ -50,7 +50,6 @@ export type { Response, Router, } from "npm:express@^4"; -//export type { FilterOptions as StreamJsonOptions } from "npm:stream-json@^1.8/filters/FilterBase.js"; // @deno-types="npm:@types/express@^4" import express from "npm:express@^4"; import extractZip from "npm:extract-zip@^2.0"; @@ -60,8 +59,6 @@ export { extractFull } from "npm:node-7z@^3.0"; // @deno-types="npm:@types/node-7z@^2" export type { SevenZipOptions } from "npm:node-7z@^3"; // @deno-types="https://cdn.sheetjs.com/xlsx-0.20.2/package/types/index.d.ts" -//export const jsonStreamArrayUrl = "npm:stream-json@^1.8/streamers/StreamArray.js"; -//export const jsonPickUrl = "npm:stream-json@^1.8/filters/Pick.js"; import * as xlsx from "https://cdn.sheetjs.com/xlsx-0.20.3/package/xlsx.mjs"; import excel from "npm:exceljs@^4.4"; diff --git a/api/src/etl/datasets/eurostat/countries/2020/EurostatCountries2020.ts b/api/src/etl/datasets/eurostat/countries/2020/EurostatCountries2020.ts index ed5a7e6a7a..3464df08e3 100644 --- a/api/src/etl/datasets/eurostat/countries/2020/EurostatCountries2020.ts +++ b/api/src/etl/datasets/eurostat/countries/2020/EurostatCountries2020.ts @@ -1,17 +1,13 @@ import { AbstractDataset } from "../../../../common/AbstractDataset.ts"; import { streamData } from "../../../../helpers/index.ts"; -import { - ArchiveFileTypeEnum, - FileTypeEnum, -} from "../../../../interfaces/index.ts"; +import { ArchiveFileTypeEnum, FileTypeEnum } from "../../../../interfaces/index.ts"; export class EurostatCountries2020 extends AbstractDataset { static producer = "eurostat"; static dataset = "countries"; static year = 2020; static table = "eurostat_countries_2020"; - static url = - "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2020_4326.geojson"; + static url = "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2020_4326.geojson"; readonly fileArchiveType: ArchiveFileTypeEnum = ArchiveFileTypeEnum.None; readonly rows: Map = new Map([ ["codeiso3", ["properties->>ISO3_CODE", "varchar"]], @@ -41,7 +37,7 @@ export class EurostatCountries2020 extends AbstractDataset { ) WITH tmp as( SELECT * FROM - json_to_recordset($1) + json_to_recordset($1::json) as tmp(type varchar, properties json,geometry json) ) SELECT @@ -50,7 +46,7 @@ export class EurostatCountries2020 extends AbstractDataset { FROM tmp ON CONFLICT DO NOTHING `, - values: [JSON.stringify(results.value.map((r: any) => r.value))], + values: [JSON.stringify(results.value)], }; await connection.query(query); } diff --git a/api/src/etl/datasets/eurostat/countries/2020/EurostatSimplifiedCountries2020.ts b/api/src/etl/datasets/eurostat/countries/2020/EurostatSimplifiedCountries2020.ts index 03bf4ffd10..a61df0b1c0 100644 --- a/api/src/etl/datasets/eurostat/countries/2020/EurostatSimplifiedCountries2020.ts +++ b/api/src/etl/datasets/eurostat/countries/2020/EurostatSimplifiedCountries2020.ts @@ -1,17 +1,13 @@ import { AbstractDataset } from "../../../../common/AbstractDataset.ts"; import { streamData } from "../../../../helpers/index.ts"; -import { - ArchiveFileTypeEnum, - FileTypeEnum, -} from "../../../../interfaces/index.ts"; +import { ArchiveFileTypeEnum, FileTypeEnum } from "../../../../interfaces/index.ts"; export class EurostatSimplifiedCountries2020 extends AbstractDataset { static producer = "eurostat"; static dataset = "simplified_countries"; static year = 2020; static table = "eurostat_simplified_countries_2020"; - static url = - "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_60M_2020_4326.geojson"; + static url = "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_60M_2020_4326.geojson"; readonly fileArchiveType: ArchiveFileTypeEnum = ArchiveFileTypeEnum.None; readonly rows: Map = new Map([ ["codeiso3", ["properties->>ISO3_CODE", "varchar"]], @@ -41,7 +37,7 @@ export class EurostatSimplifiedCountries2020 extends AbstractDataset { ) WITH tmp as( SELECT * FROM - json_to_recordset($1) + json_to_recordset($1::json) as tmp(type varchar, properties json,geometry json) ) SELECT @@ -50,7 +46,7 @@ export class EurostatSimplifiedCountries2020 extends AbstractDataset { FROM tmp ON CONFLICT DO NOTHING `, - values: [JSON.stringify(results.value.map((r: any) => r.value))], + values: [JSON.stringify(results.value)], }; await connection.query(query); } diff --git a/api/src/etl/datasets/eurostat/countries/2024/EurostatCountries2024.ts b/api/src/etl/datasets/eurostat/countries/2024/EurostatCountries2024.ts index 54a2e0c099..8625d34595 100644 --- a/api/src/etl/datasets/eurostat/countries/2024/EurostatCountries2024.ts +++ b/api/src/etl/datasets/eurostat/countries/2024/EurostatCountries2024.ts @@ -1,17 +1,13 @@ import { AbstractDataset } from "../../../../common/AbstractDataset.ts"; import { streamData } from "../../../../helpers/index.ts"; -import { - ArchiveFileTypeEnum, - FileTypeEnum, -} from "../../../../interfaces/index.ts"; +import { ArchiveFileTypeEnum, FileTypeEnum } from "../../../../interfaces/index.ts"; export class EurostatCountries2024 extends AbstractDataset { static producer = "eurostat"; static dataset = "countries"; static year = 2024; static table = "eurostat_countries_2024"; - static url = - "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2024_4326.geojson"; + static url = "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2024_4326.geojson"; readonly fileArchiveType: ArchiveFileTypeEnum = ArchiveFileTypeEnum.None; readonly rows: Map = new Map([ ["codeiso3", ["properties->>ISO3_CODE", "varchar"]], @@ -41,7 +37,7 @@ export class EurostatCountries2024 extends AbstractDataset { ) WITH tmp as( SELECT * FROM - json_to_recordset($1) + json_to_recordset($1::json) as tmp(type varchar, properties json,geometry json) ) SELECT @@ -50,7 +46,7 @@ export class EurostatCountries2024 extends AbstractDataset { FROM tmp ON CONFLICT DO NOTHING `, - values: [JSON.stringify(results.value.map((r: any) => r.value))], + values: [JSON.stringify(results.value)], }; await connection.query(query); } diff --git a/api/src/etl/datasets/eurostat/countries/2024/EurostatSimplifiedCountries2024.ts b/api/src/etl/datasets/eurostat/countries/2024/EurostatSimplifiedCountries2024.ts index 3d860e2323..aa387034dd 100644 --- a/api/src/etl/datasets/eurostat/countries/2024/EurostatSimplifiedCountries2024.ts +++ b/api/src/etl/datasets/eurostat/countries/2024/EurostatSimplifiedCountries2024.ts @@ -1,17 +1,13 @@ import { AbstractDataset } from "../../../../common/AbstractDataset.ts"; import { streamData } from "../../../../helpers/index.ts"; -import { - ArchiveFileTypeEnum, - FileTypeEnum, -} from "../../../../interfaces/index.ts"; +import { ArchiveFileTypeEnum, FileTypeEnum } from "../../../../interfaces/index.ts"; export class EurostatSimplifiedCountries2024 extends AbstractDataset { static producer = "eurostat"; static dataset = "simplified_countries"; static year = 2024; static table = "eurostat_simplified_countries_2024"; - static url = - "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_60M_2024_4326.geojson"; + static url = "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_60M_2024_4326.geojson"; readonly fileArchiveType: ArchiveFileTypeEnum = ArchiveFileTypeEnum.None; readonly rows: Map = new Map([ ["codeiso3", ["properties->>ISO3_CODE", "varchar"]], @@ -41,7 +37,7 @@ export class EurostatSimplifiedCountries2024 extends AbstractDataset { ) WITH tmp as( SELECT * FROM - json_to_recordset($1) + json_to_recordset($1::json) as tmp(type varchar, properties json,geometry json) ) SELECT @@ -50,7 +46,7 @@ export class EurostatSimplifiedCountries2024 extends AbstractDataset { FROM tmp ON CONFLICT DO NOTHING `, - values: [JSON.stringify(results.value.map((r: any) => r.value))], + values: [JSON.stringify(results.value)], }; await connection.query(query); } diff --git a/api/src/etl/datasets/ign/common/IgnDataset.ts b/api/src/etl/datasets/ign/common/IgnDataset.ts index 5ee4e8ce0b..1f11d5b0b8 100644 --- a/api/src/etl/datasets/ign/common/IgnDataset.ts +++ b/api/src/etl/datasets/ign/common/IgnDataset.ts @@ -101,7 +101,7 @@ export abstract class IgnDataset extends AbstractDataset { done = !!results.done; if (results.value) { const values = [ - JSON.stringify(results.value.map((r: any) => r.value)), + JSON.stringify(results.value), ]; logger.debug(`Batch ${i}`); switch (key) { @@ -113,7 +113,7 @@ export abstract class IgnDataset extends AbstractDataset { ) WITH tmp as( SELECT * FROM - json_to_recordset($1) + json_to_recordset($1::json) as t(type varchar, properties json,geometry json) ) SELECT ${ @@ -135,7 +135,7 @@ export abstract class IgnDataset extends AbstractDataset { SET ${key} = st_multi(ST_SetSRID(st_geomfromgeojson(tt.geometry),4326)) FROM ( SELECT * FROM - json_to_recordset($1) + json_to_recordset($1::json) as t(type varchar, properties json,geometry json) ) AS tt WHERE (arr is not null AND arr = (tt.properties->>'${arrField[0]}')::${arrField[1]}) @@ -151,7 +151,7 @@ export abstract class IgnDataset extends AbstractDataset { SET ${key} = ST_SetSRID(st_geomfromgeojson(tt.geometry),4326) FROM ( SELECT * FROM - json_to_recordset($1) + json_to_recordset($1::json) as t(type varchar, properties json,geometry json) ) AS tt WHERE (arr is not null AND arr = (tt.properties->>'${arrField[0]}')::${arrField[1]}) diff --git a/api/src/etl/helpers/data/streamCsv.ts b/api/src/etl/helpers/data/streamCsv.ts index 48ee072144..5434b93176 100644 --- a/api/src/etl/helpers/data/streamCsv.ts +++ b/api/src/etl/helpers/data/streamCsv.ts @@ -8,7 +8,7 @@ export async function* streamCsv( ): AsyncIterable { const fsStream = createReadStream(filepath, { encoding: "utf-8" }); const parser = parse({ - columns: (header: any) => Object.keys(header).map((k) => k.toLowerCase()), + columns: (header: Record) => Object.keys(header).map((k) => k.toLowerCase()), ...sheetOptions, }); fsStream.pipe(parser); diff --git a/api/src/etl/helpers/data/streamJson.ts b/api/src/etl/helpers/data/streamJson.ts index d1ca13736b..ed7ef0de29 100644 --- a/api/src/etl/helpers/data/streamJson.ts +++ b/api/src/etl/helpers/data/streamJson.ts @@ -1,29 +1,39 @@ -// import { createReadStream, jsonPickUrl, jsonStreamArrayUrl } from "@/deps.ts"; -// import type { JsonOptions } from "../../interfaces/index.ts"; -// const { default: Pick } = await import(jsonPickUrl); -// const { default: StreamArray } = await import(jsonStreamArrayUrl); +import { JsonOptions } from "../../interfaces/index.ts"; -// export async function* streamJson( -// filepath: string, -// sheetOptions: JsonOptions, -// chunkSize = 100, -// ): AsyncIterable { -// const pipe = createReadStream(filepath, { encoding: "utf-8" }).pipe( -// Pick.withParser(sheetOptions), -// ).pipe(new StreamArray()); +export async function* streamJson( + filepath: string, + sheetOptions: JsonOptions, + chunkSize = 100, +): AsyncIterable { + // Lire tout le fichier comme une seule chaîne + const file = await Deno.readTextFile(filepath); + try { + // Parse tout le fichier + const parsed = JSON.parse(file); + // Si un filtre est défini, extraire le sous-ensemble correspondant + const data = sheetOptions.filter ? parsed[sheetOptions.filter] : parsed; + // Vérifiez si c'est un tableau JSON + if (Array.isArray(data)) { + let chunk: T[] = []; + for (const line of data) { + chunk.push(line); -// let chunk: T[] = []; -// for await (const line of pipe) { -// if (chunk.length === chunkSize) { -// yield chunk; -// chunk = []; -// } -// chunk.push(line); -// } -// yield chunk; - -// return; -// } - -// TODO FIXME -export async function* streamJson(f: string, s: any, c = 100) {} + if (chunk.length === chunkSize) { + yield chunk; + chunk = []; + } + } + // Renvoyer le dernier chunk s'il reste des éléments + if (chunk.length > 0) { + yield chunk; + } + } else { + console.error("Le fichier JSON n'est pas un tableau !"); + throw new Error("Le fichier JSON doit être un tableau."); + } + return; + } catch (error) { + console.error("Erreur lors du parsing du fichier JSON :", error); + throw error; + } +} diff --git a/api/src/etl/interfaces/StreamDataOptions.ts b/api/src/etl/interfaces/StreamDataOptions.ts index 8464ad6e8c..82691d53c2 100644 --- a/api/src/etl/interfaces/StreamDataOptions.ts +++ b/api/src/etl/interfaces/StreamDataOptions.ts @@ -1,11 +1,13 @@ -import type { CsvOptions, StreamJsonOptions as JsonOptions } from "@/deps.ts"; +import type { CsvOptions } from "@/deps.ts"; export type { CsvOptions }; -export type { JsonOptions }; export interface XlsxOptions { name?: string; startRow?: number; } +export interface JsonOptions { + filter: string; +} export type StreamDataOptions = | CsvOptions