diff --git a/.babelrc b/.babelrc index 1e558e64..0c3f00f0 100644 --- a/.babelrc +++ b/.babelrc @@ -9,7 +9,8 @@ ], "plugins": [ ["module-resolver", { - "root": ["./src", "./test"] + "root": ["./src", "./test"], + "extensions": [".js", ".ts"] }] ] } diff --git a/.env.sample b/.env.sample index a951854f..a033133c 100644 --- a/.env.sample +++ b/.env.sample @@ -70,7 +70,7 @@ INSTAGRAM_CALLBACK_URL=PUBLIC_URL/callback/instagram GA_PAGE_SIZE=10000 # The target dataset ID (`projectId.dataset`) of LINE bot's analytics. -# The schema should match `/test/rumors-db/bq` +# The schema should match `/src/rumors-db/bq` LINE_BOT_EVENT_DATASET_ID= # The target dataset ID (`projectId.dataset`) of Google Analytics 4. GA4_DATASET_ID= diff --git a/.eslintignore b/.eslintignore index 51dd0639..ea22e0ca 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1 @@ -test/rumors-db \ No newline at end of file +src/rumors-db \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 2b8fb8ce..4dc6bd44 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "test/rumors-db"] - path = test/rumors-db + path = src/rumors-db url = https://github.com/MrOrz/rumors-db.git diff --git a/README.md b/README.md index 8a191d83..68da8667 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ $ docker-compose logs api # `api' can also be `db', `kibana'. $ docker-compose logs -f api # Tail mode ``` -### About `test/rumors-db` +### About `src/rumors-db` This directory is managed by git submodule. Use the following command to update: diff --git a/package-lock.json b/package-lock.json index a6dc5c03..5095a09d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,12 +68,15 @@ "eslint": "^7.5.0", "eslint-config-prettier": "^7.0.0", "eslint-import-resolver-babel-module": "^5.0.0", + "eslint-import-resolver-typescript": "^3.6.1", "eslint-plugin-import": "^2.26.0", "eslint-plugin-prettier": "^3.0.0", "jest": "^29.0.0", "mockdate": "^2.0.1", "prettier": "^2.8.6", - "serve-handler": "^6.1.3" + "serve-handler": "^6.1.3", + "zod": "^3.23.8", + "zoq": "^0.0.7" }, "engines": { "node": ">=18" @@ -7572,6 +7575,19 @@ "once": "^1.4.0" } }, + "node_modules/enhanced-resolve": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.0.tgz", + "integrity": "sha512-dwDPwZL0dmye8Txp2gzFmA6sxALaSvdRDjPH0viLcKrtlOL3tw62nWWweVD1SdILDTJrbrL6tdWVN58Wo6U3eA==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, "node_modules/enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -7841,17 +7857,63 @@ "ms": "^2.1.1" } }, + "node_modules/eslint-import-resolver-typescript": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.6.1.tgz", + "integrity": "sha512-xgdptdoi5W3niYeuQxKmzVDTATvLYqhpwmykwsh7f6HIOStGWEIL9iqZgQDF9u9OEzrRwR8no5q2VT+bjAujTg==", + "dev": true, + "dependencies": { + "debug": "^4.3.4", + "enhanced-resolve": "^5.12.0", + "eslint-module-utils": "^2.7.4", + "fast-glob": "^3.3.1", + "get-tsconfig": "^4.5.0", + "is-core-module": "^2.11.0", + "is-glob": "^4.0.3" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts/projects/eslint-import-resolver-ts" + }, + "peerDependencies": { + "eslint": "*", + "eslint-plugin-import": "*" + } + }, + "node_modules/eslint-import-resolver-typescript/node_modules/debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/eslint-module-utils": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.3.tgz", - "integrity": "sha512-088JEC7O3lDZM9xGe0RerkOMd0EjFl+Yvd1jPWIkMT5u3H9+HC34mWWPnqPrN13gieT9pBOO+Qt07Nb/6TresQ==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.1.tgz", + "integrity": "sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==", "dev": true, "dependencies": { - "debug": "^3.2.7", - "find-up": "^2.1.0" + "debug": "^3.2.7" }, "engines": { "node": ">=4" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } } }, "node_modules/eslint-module-utils/node_modules/debug": { @@ -8719,8 +8781,12 @@ "license": "MIT" }, "node_modules/function-bind": { - "version": "1.1.1", - "license": "MIT" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/functional-red-black-tree": { "version": "1.0.1", @@ -8895,6 +8961,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.5.tgz", + "integrity": "sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==", + "dev": true, + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/get-uri": { "version": "2.0.4", "license": "MIT", @@ -9853,6 +9931,17 @@ "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.4.tgz", "integrity": "sha512-Gjzu0Xn7IagXVkSu9cSFuK1fqzwtLwFhNhVL8IFJijRNMgUttFbBSIAzKuSIrsFMO1+g1RlsoN49zPIbwPDMGQ==" }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/homedir-polyfill": { "version": "1.0.3", "dev": true, @@ -10235,11 +10324,14 @@ } }, "node_modules/is-core-module": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", - "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.14.0.tgz", + "integrity": "sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==", "dependencies": { - "has": "^1.0.3" + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -15131,6 +15223,15 @@ "node": ">= 0.8" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/resolve.exports": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", @@ -16063,6 +16164,15 @@ "node": ">=8" } }, + "node_modules/tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/tar-fs": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", @@ -17103,6 +17213,21 @@ "tslib": "^1.9.3", "zen-observable": "^0.8.0" } + }, + "node_modules/zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zoq": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/zoq/-/zoq-0.0.7.tgz", + "integrity": "sha512-M6wfZWLFcIfLxdosyNJHlJdzb3UH49Brx+/scv3dIjpbDKfJqOs5MtdIy7i2yByWYHL+Y7RWMvIurrtwFbPkWQ==", + "dev": true } }, "dependencies": { @@ -22493,6 +22618,16 @@ "once": "^1.4.0" } }, + "enhanced-resolve": { + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.0.tgz", + "integrity": "sha512-dwDPwZL0dmye8Txp2gzFmA6sxALaSvdRDjPH0viLcKrtlOL3tw62nWWweVD1SdILDTJrbrL6tdWVN58Wo6U3eA==", + "dev": true, + "requires": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + } + }, "enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -22879,14 +23014,39 @@ } } }, + "eslint-import-resolver-typescript": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.6.1.tgz", + "integrity": "sha512-xgdptdoi5W3niYeuQxKmzVDTATvLYqhpwmykwsh7f6HIOStGWEIL9iqZgQDF9u9OEzrRwR8no5q2VT+bjAujTg==", + "dev": true, + "requires": { + "debug": "^4.3.4", + "enhanced-resolve": "^5.12.0", + "eslint-module-utils": "^2.7.4", + "fast-glob": "^3.3.1", + "get-tsconfig": "^4.5.0", + "is-core-module": "^2.11.0", + "is-glob": "^4.0.3" + }, + "dependencies": { + "debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + } + } + }, "eslint-module-utils": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.3.tgz", - "integrity": "sha512-088JEC7O3lDZM9xGe0RerkOMd0EjFl+Yvd1jPWIkMT5u3H9+HC34mWWPnqPrN13gieT9pBOO+Qt07Nb/6TresQ==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.1.tgz", + "integrity": "sha512-rXDXR3h7cs7dy9RNpUlQf80nX31XWJEyGq1tRMo+6GsO5VmTe4UTwtmonAD4ZkAsrfMVDA2wlGJ3790Ys+D49Q==", "dev": true, "requires": { - "debug": "^3.2.7", - "find-up": "^2.1.0" + "debug": "^3.2.7" }, "dependencies": { "debug": { @@ -23345,7 +23505,9 @@ } }, "function-bind": { - "version": "1.1.1" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, "functional-red-black-tree": { "version": "1.0.1", @@ -23460,6 +23622,15 @@ "get-intrinsic": "^1.1.1" } }, + "get-tsconfig": { + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.5.tgz", + "integrity": "sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==", + "dev": true, + "requires": { + "resolve-pkg-maps": "^1.0.0" + } + }, "get-uri": { "version": "2.0.4", "requires": { @@ -24119,6 +24290,14 @@ "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.4.tgz", "integrity": "sha512-Gjzu0Xn7IagXVkSu9cSFuK1fqzwtLwFhNhVL8IFJijRNMgUttFbBSIAzKuSIrsFMO1+g1RlsoN49zPIbwPDMGQ==" }, + "hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "requires": { + "function-bind": "^1.1.2" + } + }, "homedir-polyfill": { "version": "1.0.3", "dev": true, @@ -24366,11 +24545,11 @@ "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==" }, "is-core-module": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", - "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.14.0.tgz", + "integrity": "sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==", "requires": { - "has": "^1.0.3" + "hasown": "^2.0.2" } }, "is-date-object": { @@ -27922,6 +28101,12 @@ "path-is-absolute": "1.0.1" } }, + "resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true + }, "resolve.exports": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", @@ -28553,6 +28738,12 @@ } } }, + "tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "dev": true + }, "tar-fs": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", @@ -29257,6 +29448,18 @@ "tslib": "^1.9.3", "zen-observable": "^0.8.0" } + }, + "zod": { + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", + "dev": true + }, + "zoq": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/zoq/-/zoq-0.0.7.tgz", + "integrity": "sha512-M6wfZWLFcIfLxdosyNJHlJdzb3UH49Brx+/scv3dIjpbDKfJqOs5MtdIy7i2yByWYHL+Y7RWMvIurrtwFbPkWQ==", + "dev": true } } } diff --git a/package.json b/package.json index 4747cab9..a99bbf21 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "license": "MIT", "scripts": { "dev": "pm2-dev start --timestamp process-dev.json", - "seed": "cd test/rumors-db && npm run seed", + "seed": "cd src/rumors-db && npm run seed", "pretest": "npm run rumors-db:install && npm run rumors-db:test && mkdir -p build", "test": "NODE_ENV=test ELASTICSEARCH_URL=http://localhost:62223 jest --runInBand", "posttest": "NODE_ENV=test ELASTICSEARCH_URL=http://localhost:62223 babel-node test/postTest.js", @@ -20,9 +20,9 @@ "lint": "eslint src/.", "lint:fix": "eslint --fix src/.", "typecheck": "tsc", - "rumors-db:pull": "cd test/rumors-db && git pull", - "rumors-db:test": "cd test/rumors-db && ELASTICSEARCH_URL=http://localhost:62223 npm t", - "rumors-db:install": "cd test/rumors-db && npm i" + "rumors-db:pull": "cd src/rumors-db && git pull", + "rumors-db:test": "cd src/rumors-db && ELASTICSEARCH_URL=http://localhost:62223 npm t", + "rumors-db:install": "cd src/rumors-db && npm i" }, "dependencies": { "@cofacts/media-manager": "^0.3.0", @@ -75,6 +75,8 @@ "@babel/preset-env": "^7.16.11", "@babel/preset-typescript": "^7.24.1", "@google-cloud/storage": "^6.11.0", + "@typescript-eslint/eslint-plugin": "^5.56.0", + "@typescript-eslint/parser": "^5.56.0", "apollo-server-testing": "^2.18.2", "babel-jest": "^29.0.0", "babel-plugin-module-resolver": "^4.0.0", @@ -82,14 +84,15 @@ "eslint": "^7.5.0", "eslint-config-prettier": "^7.0.0", "eslint-import-resolver-babel-module": "^5.0.0", + "eslint-import-resolver-typescript": "^3.6.1", "eslint-plugin-import": "^2.26.0", "eslint-plugin-prettier": "^3.0.0", - "@typescript-eslint/eslint-plugin": "^5.56.0", - "@typescript-eslint/parser": "^5.56.0", "jest": "^29.0.0", "mockdate": "^2.0.1", "prettier": "^2.8.6", - "serve-handler": "^6.1.3" + "serve-handler": "^6.1.3", + "zod": "^3.23.8", + "zoq": "^0.0.7" }, "jest": { "testEnvironment": "node", diff --git a/test/rumors-db b/src/rumors-db similarity index 100% rename from test/rumors-db rename to src/rumors-db diff --git a/src/util/getAllDocs.js b/src/util/getAllDocs.js deleted file mode 100644 index dcf731c6..00000000 --- a/src/util/getAllDocs.js +++ /dev/null @@ -1,45 +0,0 @@ -import client from 'util/client'; - -/** - * A generator that fetches all docs in the specified index and yield 1 doc at a time. - * - * @param {String} index The name of the index to fetch - * @param {Object} query The elasticsearch query. Fetches all doc in the specified index if not given. - * @param {Object} settings Query param settings in ES. - * @param {number} settings.size Number of docs per batch. Default to 1000. - * @param {string} settings.scroll The scroll timeout setting. Default to 30s. - * @yields {Object} the document - */ -async function* getAllDocs( - index, - query = { match_all: {} }, - { scroll = '30s', size = 1000 } = {} -) { - let resp = await client.search({ - index, - scroll, - size, - body: { - query, - }, - }); - - while (true) { - const docs = resp.body.hits.hits; - if (docs.length === 0) break; - for (const doc of docs) { - yield doc; - } - - if (!resp.body._scroll_id) { - break; - } - - resp = await client.scroll({ - scroll, - scrollId: resp.body._scroll_id, - }); - } -} - -export default getAllDocs; diff --git a/src/util/getAllDocs.ts b/src/util/getAllDocs.ts new file mode 100644 index 00000000..b6453c69 --- /dev/null +++ b/src/util/getAllDocs.ts @@ -0,0 +1,45 @@ +import client from 'util/client'; + +/** + * A generator that fetches all docs in the specified index and yield 1 doc at a time. + * + * @param index The name of the index to fetch + * @param query The elasticsearch query. Fetches all doc in the specified index if not given. + * @param settings Query param settings in ES. + * @param settings.size Number of docs per batch. Default to 1000. + * @param settings.scroll The scroll timeout setting. Default to 30s. + * @yields {Object} the document + */ +async function* getAllDocs( + index: string, + query: object = { match_all: {} }, + { scroll = '30s', size = 1000 }: { size?: number; scroll?: string } = {} +): AsyncGenerator { + let resp = await client.search({ + index, + scroll, + size, + body: { + query, + }, + }); + + while (true) { + const docs = resp.body.hits.hits; + if (docs.length === 0) break; + for (const doc of docs) { + yield doc; + } + + if (!resp.body._scroll_id) { + break; + } + + resp = await client.scroll({ + scroll, + scroll_id: resp.body._scroll_id, + }); + } +} + +export default getAllDocs; diff --git a/tsconfig.json b/tsconfig.json index 3cb046ba..74e69e64 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,9 @@ { "compilerOptions": { - "baseUrl": ".", + "baseUrl": "./", + "paths": { + "*": ["src/*", "test/*"] + }, "allowJs": true, "skipLibCheck": true, "strict": true, @@ -9,7 +12,8 @@ "module": "esnext", "moduleResolution": "node", "resolveJsonModule": true, - "isolatedModules": true + "isolatedModules": true, + "target": "es2018" }, "include": [ "src/**/*"