From e8fbce9624b5ca5b031e859384aca7c44fe00edb Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 11 Oct 2023 15:41:51 +0200 Subject: [PATCH 1/5] S3UTILS-148: bump arsenal version --- package.json | 2 +- yarn.lock | 204 ++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 177 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 814134e6..028174dc 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@senx/warp10": "^1.1.2", "JSONStream": "^1.3.5", - "arsenal": "git+https://github.com/scality/arsenal#8.1.78", + "arsenal": "git+https://github.com/scality/arsenal#8.1.113", "async": "^2.6.4", "aws-sdk": "^2.1005.0", "bucketclient": "git+https://github.com/scality/bucketclient#8.1.5", diff --git a/yarn.lock b/yarn.lock index eb5ef5d8..fe0be4f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -978,6 +978,13 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@mongodb-js/saslprep@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@mongodb-js/saslprep/-/saslprep-1.1.0.tgz#022fa36620a7287d17acd05c4aae1e5f390d250d" + integrity sha512-Xfijy7HvfzzqiOAhAepF4SGN5e9leLkMvg/OPOF97XemjfVCYN/oWa75wnkc6mltMSTwY+XlbhWgUOJmkFspSw== + dependencies: + sparse-bitfield "^3.0.3" + "@npmcli/fs@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@npmcli/fs/-/fs-1.1.0.tgz#bec1d1b89c170d40e1b73ad6c943b0b75e7d2951" @@ -1037,6 +1044,11 @@ dependencies: "@sinonjs/commons" "^1.7.0" +"@socket.io/component-emitter@~3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" + integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== + "@tootallnate/once@1": version "1.1.2" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" @@ -1052,6 +1064,18 @@ resolved "https://registry.yarnpkg.com/@types/async/-/async-3.2.15.tgz#26d4768fdda0e466f18d6c9918ca28cc89a4e1fe" integrity sha512-PAmPfzvFA31mRoqZyTVsgJMsvbynR429UTTxhmfsUCrWGh3/fxOrzqBtaTPJsn4UtzTv4Vb0+/O7CARWb69N4g== +"@types/cookie@^0.4.1": + version "0.4.1" + resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d" + integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q== + +"@types/cors@^2.8.12": + version "2.8.14" + resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.14.tgz#94eeb1c95eda6a8ab54870a3bf88854512f43a92" + integrity sha512-RXHUvNWYICtbP6s18PnOCaqToK8y14DnLd75c6HfyKf228dxy7pHNOQkxPtvXKp/hINFMDjbYzsj63nnpPMSRQ== + dependencies: + "@types/node" "*" + "@types/json-schema@^7.0.3": version "7.0.4" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.4.tgz#38fd73ddfd9b55abb1e1b2ed578cb55bd7b7d339" @@ -1075,6 +1099,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.17.tgz#5c009e1d9c38f4a2a9d45c0b0c493fe6cdb4bcb5" integrity sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng== +"@types/node@>=10.0.0": + version "20.8.4" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.4.tgz#0e9ebb2ff29d5c3302fc84477d066fa7c6b441aa" + integrity sha512-ZVPnqU58giiCjSxjVUESDtdPk4QR5WQhhINbc9UBrKLU68MX5BF6kbQzTrkwbolyr0X8ChBpXfavr5mZFKZQ5A== + dependencies: + undici-types "~5.25.1" + "@types/tmp@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.2.3.tgz#908bfb113419fd6a42273674c00994d40902c165" @@ -1522,9 +1553,9 @@ arrify@^1.0.1: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#8.1.78": - version "8.1.78" - resolved "git+https://github.com/scality/arsenal#c460338163bcc9decde7c18d2835beeef53273e6" +"arsenal@git+https://github.com/scality/arsenal#8.1.113": + version "8.1.113" + resolved "git+https://github.com/scality/arsenal#17b5bbc233b0bfbca8b68a45dcd730903479a39e" dependencies: "@azure/identity" "^3.1.1" "@azure/storage-blob" "^12.12.0" @@ -1541,7 +1572,7 @@ arrify@^1.0.1: bson "4.0.0" debug "~4.1.0" diskusage "^1.1.1" - fcntl "github:scality/node-fcntl#0.2.0" + fcntl "github:scality/node-fcntl#0.2.2" hdclient scality/hdclient#1.1.5 httpagent scality/httpagent#1.0.6 https-proxy-agent "^2.2.0" @@ -1550,13 +1581,13 @@ arrify@^1.0.1: joi "^17.6.0" level "~5.0.1" level-sublevel "~6.6.5" - mongodb "^3.0.1" + mongodb "^5.2.0" node-forge "^1.3.0" - prom-client "10.2.3" + prom-client "14.2.0" simple-glob "^0.2.0" - socket.io "2.4.1" - socket.io-client "2.4.0" - sproxydclient scality/sproxydclient#8.0.7 + socket.io "~4.6.1" + socket.io-client "~4.6.1" + sproxydclient "git+https://github.com/scality/sproxydclient#8.0.10" utf8 "3.0.0" uuid "^3.0.1" werelogs scality/werelogs#8.1.2 @@ -1873,7 +1904,7 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -base64id@2.0.0: +base64id@2.0.0, base64id@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6" integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog== @@ -2046,6 +2077,11 @@ bson@^4.7.0: dependencies: buffer "^5.6.0" +bson@^5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-5.5.0.tgz#a419cc69f368d2def3b8b22ea03ed1c9be40e53f" + integrity sha512-B+QB4YmDx9RStKv8LLSl/aVIEV3nYJc3cJNNTK2Cd1TL+7P+cNpw9mAPeCgc5K+j01Dv6sxUzcITXDx7ZU3F0w== + "bucketclient@git+https://github.com/scality/bucketclient#8.1.5": version "8.1.5" resolved "git+https://github.com/scality/bucketclient#3941311c147a36178caf02dfeea8c4041f017ef6" @@ -2447,6 +2483,14 @@ core-util-is@1.0.2, core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +cors@~2.8.5: + version "2.8.5" + resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" + integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== + dependencies: + object-assign "^4" + vary "^1" + cron-parser@^2.18.0: version "2.18.0" resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.18.0.tgz#de1bb0ad528c815548371993f81a54e5a089edcf" @@ -2541,7 +2585,7 @@ debug@^4.1.1, debug@~4.1.0: dependencies: ms "^2.1.1" -debug@^4.3.4: +debug@^4.3.4, debug@~4.3.1, debug@~4.3.2: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -2811,6 +2855,17 @@ engine.io-client@~3.5.0: xmlhttprequest-ssl "~1.6.2" yeast "0.1.2" +engine.io-client@~6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.4.0.tgz#88cd3082609ca86d7d3c12f0e746d12db4f47c91" + integrity sha512-GyKPDyoEha+XZ7iEqam49vz6auPnNJ9ZBfy89f+rMMas8AuiMWOZ9PVzu8xb9ZC6rafUqiGHSCfu22ih66E+1g== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + engine.io-parser "~5.0.3" + ws "~8.11.0" + xmlhttprequest-ssl "~2.0.0" + engine.io-parser@~2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-2.2.1.tgz#57ce5611d9370ee94f99641b589f94c97e4f5da7" @@ -2822,6 +2877,11 @@ engine.io-parser@~2.2.0: blob "0.0.5" has-binary2 "~1.0.2" +engine.io-parser@~5.0.3: + version "5.0.7" + resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.7.tgz#ed5eae76c71f398284c578ab6deafd3ba7e4e4f6" + integrity sha512-P+jDFbvK6lE3n1OL+q9KuzdOFWkkZ/cMV9gol/SbVfpyqfvrfrFTOFJ6fQm2VC3PZHlU3QPhVwmbsCnauHF2MQ== + engine.io@~3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-3.5.0.tgz#9d6b985c8a39b1fe87cd91eb014de0552259821b" @@ -2834,6 +2894,22 @@ engine.io@~3.5.0: engine.io-parser "~2.2.0" ws "~7.4.2" +engine.io@~6.4.2: + version "6.4.2" + resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.2.tgz#ffeaf68f69b1364b0286badddf15ff633476473f" + integrity sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg== + dependencies: + "@types/cookie" "^0.4.1" + "@types/cors" "^2.8.12" + "@types/node" ">=10.0.0" + accepts "~1.3.4" + base64id "2.0.0" + cookie "~0.4.1" + cors "~2.8.5" + debug "~4.3.1" + engine.io-parser "~5.0.3" + ws "~8.11.0" + entities@~2.0.0: version "2.0.3" resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.3.tgz#5c487e5742ab93c15abb5da22759b8590ec03b7f" @@ -3374,6 +3450,14 @@ fb-watchman@^2.0.0: nan "^2.3.2" node-gyp "^8.0.0" +"fcntl@github:scality/node-fcntl#0.2.2": + version "0.2.1" + resolved "https://codeload.github.com/scality/node-fcntl/tar.gz/b1335ca204c6265cedc50c26020c4d63aabe920e" + dependencies: + bindings "^1.1.1" + nan "^2.3.2" + node-gyp "^8.0.0" + fd-slicer@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" @@ -3982,12 +4066,6 @@ http-signature@~1.2.0: dependencies: agentkeepalive "^4.2.1" -"httpagent@github:scality/httpagent#1.0.5": - version "1.0.5" - resolved "https://codeload.github.com/scality/httpagent/tar.gz/ec4e28f05001eaa6528daa174dbeab9e1df40ff6" - dependencies: - agentkeepalive "^4.2.1" - "httpagent@github:scality/httpagent#1.0.6", httpagent@scality/httpagent#1.0.6: version "1.0.6" uid "66e97ab93532d79dbddcf6c4ea4df849b84c33c9" @@ -5940,7 +6018,7 @@ moment-timezone@^0.5.31: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== -mongodb-connection-string-url@^2.5.4: +mongodb-connection-string-url@^2.5.4, mongodb-connection-string-url@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz#57901bf352372abdde812c81be47b75c6b2ec5cf" integrity sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ== @@ -5991,6 +6069,17 @@ mongodb@^3.0.1: optionalDependencies: saslprep "^1.0.0" +mongodb@^5.2.0: + version "5.9.0" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-5.9.0.tgz#5a22065fa8cfaf1d58bf2e3c451cd2c4bfa983a2" + integrity sha512-g+GCMHN1CoRUA+wb1Agv0TI4YTSiWr42B5ulkiAfLLHitGK1R+PkSAf3Lr5rPZwi/3F04LiaZEW0Kxro9Fi2TA== + dependencies: + bson "^5.5.0" + mongodb-connection-string-url "^2.6.0" + socks "^2.7.1" + optionalDependencies: + "@mongodb-js/saslprep" "^1.1.0" + mongodb@~4.11.0: version "4.11.0" resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.11.0.tgz#d28fdc7509f24d0d274f456529441fa3e570415c" @@ -6284,7 +6373,7 @@ oauth-sign@~0.9.0: resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== -object-assign@^4.1.0: +object-assign@^4, object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= @@ -6726,6 +6815,13 @@ prom-client@10.2.3: dependencies: tdigest "^0.1.1" +prom-client@14.2.0: + version "14.2.0" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-14.2.0.tgz#ca94504e64156f6506574c25fb1c34df7812cf11" + integrity sha512-sF308EhTenb/pDRPakm+WgiN+VdM/T1RaHj1x+MvAuT8UiQP8JmOEbxVqtkbfR4LrvOg5n7ic01kRBDGXjYikA== + dependencies: + tdigest "^0.1.1" + promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" @@ -7468,6 +7564,13 @@ socket.io-adapter@~1.1.0: resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-1.1.2.tgz#ab3f0d6f66b8fc7fca3959ab5991f82221789be9" integrity sha512-WzZRUj1kUjrTIrUKpZLEzFZ1OLj5FwLlAFQs9kuZJzJi5DKdU7FsWc36SNmA8iDOtwBQyT8FkrriRM8vXLYz8g== +socket.io-adapter@~2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz#5de9477c9182fdc171cd8c8364b9a8894ec75d12" + integrity sha512-87C3LO/NOMc+eMcpcxUBebGjkpMDkNBS9tf7KJqcDsmL936EChtVva71Dw2q4tQcuVC+hAUy4an2NO/sYXmwRA== + dependencies: + ws "~8.11.0" + socket.io-client@2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-2.4.0.tgz#aafb5d594a3c55a34355562fc8aea22ed9119a35" @@ -7485,6 +7588,16 @@ socket.io-client@2.4.0: socket.io-parser "~3.3.0" to-array "0.1.4" +socket.io-client@~4.6.1: + version "4.6.2" + resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.6.2.tgz#2bfde952e74625d54e622718a7cb1d591ee62fd6" + integrity sha512-OwWrMbbA8wSqhBAR0yoPK6EdQLERQAYjXb3A0zLpgxfM1ZGLKoxHx8gVmCHA6pcclRX5oA/zvQf7bghAS11jRA== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.2" + engine.io-client "~6.4.0" + socket.io-parser "~4.2.4" + socket.io-parser@~3.3.0: version "3.3.2" resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-3.3.2.tgz#ef872009d0adcf704f2fbe830191a14752ad50b6" @@ -7503,6 +7616,14 @@ socket.io-parser@~3.4.0: debug "~4.1.0" isarray "2.0.1" +socket.io-parser@~4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" + integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + socket.io@2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-2.4.1.tgz#95ad861c9a52369d7f1a68acf0d4a1b16da451d2" @@ -7515,6 +7636,18 @@ socket.io@2.4.1: socket.io-client "2.4.0" socket.io-parser "~3.4.0" +socket.io@~4.6.1: + version "4.6.2" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.2.tgz#d597db077d4df9cbbdfaa7a9ed8ccc3d49439786" + integrity sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ== + dependencies: + accepts "~1.3.4" + base64id "~2.0.0" + debug "~4.3.2" + engine.io "~6.4.2" + socket.io-adapter "~2.5.2" + socket.io-parser "~4.2.4" + socks-proxy-agent@^6.0.0: version "6.1.1" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-6.1.1.tgz#e664e8f1aaf4e1fb3df945f09e3d94f911137f87" @@ -7631,6 +7764,14 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= +"sproxydclient@git+https://github.com/scality/sproxydclient#8.0.10": + version "8.0.10" + resolved "git+https://github.com/scality/sproxydclient#dca287a1d50c2a38919710e44a58af2e8b4127e0" + dependencies: + async "^3.2.0" + httpagent "github:scality/httpagent#1.0.6" + werelogs scality/werelogs#8.1.2 + "sproxydclient@github:scality/sproxydclient#8.0.4": version "8.0.4" resolved "https://codeload.github.com/scality/sproxydclient/tar.gz/d413a3ee26dc090a6db1af6cf7b0094f131a4913" @@ -7638,14 +7779,6 @@ sprintf-js@~1.0.2: async "^3.2.0" werelogs scality/werelogs#8.1.0 -sproxydclient@scality/sproxydclient#8.0.7: - version "8.0.7" - resolved "https://codeload.github.com/scality/sproxydclient/tar.gz/46049c28d2487c8e4ea4c7dde2be2fc90dbcc9ec" - dependencies: - async "^3.2.0" - httpagent "github:scality/httpagent#1.0.5" - werelogs scality/werelogs#8.1.2 - sshpk@^1.7.0: version "1.15.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.15.1.tgz#b79a089a732e346c6e0714830f36285cd38191a2" @@ -8183,6 +8316,11 @@ underscore@^1.12.1: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== +undici-types@~5.25.1: + version "5.25.3" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3" + integrity sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA== + union-value@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.0.tgz#5c71c34cb5bad5dcebe3ea0cd08207ba5aa1aea4" @@ -8319,7 +8457,7 @@ validator@^13.7.0: resolved "https://registry.yarnpkg.com/validator/-/validator-13.7.0.tgz#4f9658ba13ba8f3d82ee881d3516489ea85c0857" integrity sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw== -vary@~1.1.2: +vary@^1, vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== @@ -8544,6 +8682,11 @@ ws@~7.4.2: resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== +ws@~8.11.0: + version "8.11.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" + integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== + xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" @@ -8587,6 +8730,11 @@ xmlhttprequest-ssl@~1.6.2: resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.6.3.tgz#03b713873b01659dfa2c1c5d056065b27ddc2de6" integrity sha512-3XfeQE/wNkvrIktn2Kf0869fC0BN6UpydVasGIeSm2B1Llihf7/0UfZM+eCkOw3P7bP4+qPgqhm7ZoxuJtFU0Q== +xmlhttprequest-ssl@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67" + integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== + xtend@^4.0.2, xtend@~4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" From b382a4f440192de24fc2ef5b4fda011022e2a34d Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 26 Oct 2023 10:45:36 +0200 Subject: [PATCH 2/5] S3UTILS-148: update mongodb callbacks to promises --- StalledRetry/MongoClientInterfaceStalled.js | 17 +- ...ucketMetricsAndUpdateBucketCapacityInfo.js | 2 +- tests/functional/countItems.js | 2 +- tests/functional/stalled.js | 2 +- tests/functional/utils/S3UtilsMongoClient.js | 29 +- utils/S3UtilsMongoClient.js | 399 ++++++++++-------- 6 files changed, 246 insertions(+), 205 deletions(-) diff --git a/StalledRetry/MongoClientInterfaceStalled.js b/StalledRetry/MongoClientInterfaceStalled.js index a10e7fda..e4c1ea48 100644 --- a/StalledRetry/MongoClientInterfaceStalled.js +++ b/StalledRetry/MongoClientInterfaceStalled.js @@ -31,20 +31,20 @@ class MongoClientInterfaceStalled extends MongoClientInterface { }, }, { $project: reducedFields }, - ]); + ]).stream(); } - queueStalledObjects(expiredBy, cb) { + async queueStalledObjects(expiredBy, cb) { const cmpDate = new Date(); cmpDate.setHours(cmpDate.getHours() - expiredBy); const reqHandler = this.requestHandlerFactory(this.logger); let stalledCount = 0; - this.db.listCollections().toArray((err, collections) => { - if (err) { - return cb(err); - } + try { + const collections = await this.db.listCollections().toArray(); + let i = 0; return async.eachSeries(collections, (value, next) => { + i++; if (this._isSpecialCollection(value.name)) { // skip return next(); @@ -56,7 +56,6 @@ class MongoClientInterfaceStalled extends MongoClientInterface { this._getStalledObjectsByBucket(bucketName), this.logger, ); - return reqHandler.handleRequests(wrapper, (err, results) => { if (err) { this.logger.error( @@ -76,7 +75,9 @@ class MongoClientInterfaceStalled extends MongoClientInterface { return next(); }); }, err => cb(err, stalledCount)); - }); + } catch (err) { + return cb(err); + } } } diff --git a/tests/functional/collectBucketMetricsAndUpdateBucketCapacityInfo.js b/tests/functional/collectBucketMetricsAndUpdateBucketCapacityInfo.js index 0e6ea15a..b1def587 100644 --- a/tests/functional/collectBucketMetricsAndUpdateBucketCapacityInfo.js +++ b/tests/functional/collectBucketMetricsAndUpdateBucketCapacityInfo.js @@ -40,7 +40,7 @@ describe('collectBucketMetricsAndUpdateBucketCapacityInfo', () => { afterAll(done => { async.series([ - next => client.db.dropDatabase(next), + next => client.db.dropDatabase().then(() => next()).catch(() => next()), next => client.close(next), ], done); }); diff --git a/tests/functional/countItems.js b/tests/functional/countItems.js index 5e85a607..cbb01726 100644 --- a/tests/functional/countItems.js +++ b/tests/functional/countItems.js @@ -213,7 +213,7 @@ describe('CountItems', () => { afterAll(done => { async.series([ - next => client.db.dropDatabase(next), + next => client.db.dropDatabase().then(() => next()).catch(() => next()), next => client.close(next), ], done); }); diff --git a/tests/functional/stalled.js b/tests/functional/stalled.js index e6ddef67..a9d1b845 100644 --- a/tests/functional/stalled.js +++ b/tests/functional/stalled.js @@ -232,7 +232,7 @@ describe('StalledRetry', () => { afterAll(done => { async.series([ - next => mgoClient.db.dropDatabase(next), + next => mgoClient.db.dropDatabase().then(() => next()).catch(() => next()), next => mgoClient.close(next), ], done); }); diff --git a/tests/functional/utils/S3UtilsMongoClient.js b/tests/functional/utils/S3UtilsMongoClient.js index d060e55d..2cd89f41 100644 --- a/tests/functional/utils/S3UtilsMongoClient.js +++ b/tests/functional/utils/S3UtilsMongoClient.js @@ -834,19 +834,30 @@ describe('S3UtilsMongoClient::getBucketInfos', () => { done(); })); - it('Should ignore buckets in collection list but not in metastore', done => { + it('Should ignore buckets in collection list but not in metastore', async () => { const metastoreCollection = client.getCollection('__metastore'); - return metastoreCollection.deleteOne({ _id: buckets[0]._name }, err => { - assert.equal(err, null); + + // Delete a bucket from the metastore + const deleteResult = await metastoreCollection.deleteOne({ _id: buckets[0]._name }); + assert.equal(deleteResult.deletedCount, 1); + + // Fetch bucket information + const getBucketInfosPromise = new Promise((resolve, reject) => { client.getBucketInfos(logger, (err, data) => { - assert.equal(err, null); - assert.strictEqual(data.bucketCount, 2); - assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[0]._name)), undefined); - assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[1]._name)), JSON.stringify(buckets[1])); - assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[2]._name)), JSON.stringify(buckets[2])); - done(); + if (err) { + return reject(err); + } + return resolve(data); }); }); + + const data = await getBucketInfosPromise; + + // Perform the assertions + assert.strictEqual(data.bucketCount, 2); + assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[0]._name)), undefined); + assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[1]._name)), JSON.stringify(buckets[1])); + assert.strictEqual(JSON.stringify(data.bucketInfos.find(bucket => bucket._name === buckets[2]._name)), JSON.stringify(buckets[2])); }); }); }); diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index 3cef703b..6722bcfe 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -14,51 +14,59 @@ const __COUNT_ITEMS = 'countitems'; class S3UtilsMongoClient extends MongoClientInterface { - getObjectMDStats(bucketName, bucketInfo, isTransient, log, callback) { - const c = this.getCollection(bucketName); - const cursor = c.find({}, { - projection: { - '_id': 1, - 'value.last-modified': 1, - 'value.replicationInfo': 1, - 'value.dataStoreName': 1, - 'value.content-length': 1, - 'value.versionId': 1, - 'value.owner-id': 1, - 'value.isDeleteMarker': 1, - 'value.isNull': 1, - 'value.archive': 1, - 'value.x-amz-storage-class': 1, - 'value.isPHD': 1, - }, - }); - const collRes = { - bucket: {}, // bucket level metrics - location: {}, // location level metrics - account: {}, // account level metrics - }; - let stalledCount = 0; - const cmpDate = new Date(); - cmpDate.setHours(cmpDate.getHours() - 1); - - const locationConfig = getLocationConfig(log); + async getObjectMDStats(bucketName, bucketInfo, isTransient, log, callback) { + try { + const c = this.getCollection(bucketName); + const cursor = c.find({}, { + projection: { + '_id': 1, + 'value.last-modified': 1, + 'value.replicationInfo': 1, + 'value.dataStoreName': 1, + 'value.content-length': 1, + 'value.versionId': 1, + 'value.owner-id': 1, + 'value.isDeleteMarker': 1, + 'value.isNull': 1, + 'value.archive': 1, + 'value.x-amz-storage-class': 1, + 'value.isPHD': 1, + }, + }); + const collRes = { + bucket: {}, // bucket level metrics + location: {}, // location level metrics + account: {}, // account level metrics + }; + let stalledCount = 0; + const cmpDate = new Date(); + cmpDate.setHours(cmpDate.getHours() - 1); + + const locationConfig = getLocationConfig(log); + + const usersBucketCreationDatesArray = await this.getCollection(USERSBUCKET).find({}, { + projection: { + 'value.creationDate': 1, + }, + }).toArray(); - return this.getCollection(USERSBUCKET).find({}, { - projection: { - 'value.creationDate': 1, - }, - }).toArray((err, usersBucketCreationDatesArray) => { - if (err) { - log.error('Failed to get users bucket creation dates', { - method: 'getObjectMDStats', - error: err, - }); - return callback(err); - } const usersBucketCreationDatesMap = usersBucketCreationDatesArray .reduce((map, obj) => ({ ...map, [obj._id]: obj.value.creationDate }), {}); - return cursor.forEach( + let startCursorDate = new Date(); + let processed = 0; + await cursor.forEach( res => { + // Periodically display information about the cursor + // if more than 30s elapsed + const currentDate = Date.now(); + if (currentDate - startCursorDate > 30000) { + startCursorDate = currentDate; + log.info('Processing cursor', { + method: 'getObjectMDStats', + bucketName, + processed, + }); + } const { data, error } = this._processEntryData( bucketName, bucketInfo, @@ -154,12 +162,14 @@ class S3UtilsMongoClient extends MongoClientInterface { collRes.account[account].locations[location].deleteMarkerCount += res.value.isDeleteMarker ? 1 : 0; }); }); + processed++; }, err => { if (err) { log.error('Error when processing mongo entries', { method: 'getObjectMDStats', - error: err, + errDetails: { ...err }, + errorString: err.toString(), }); return callback(err); } @@ -171,7 +181,22 @@ class S3UtilsMongoClient extends MongoClientInterface { return callback(null, retResult); }, ); - }); + + const bucketStatus = bucketInfo.getVersioningConfiguration(); + const isVer = (bucketStatus && (bucketStatus.Status === 'Enabled' + || bucketStatus.Status === 'Suspended')); + const retResult = this._handleResults(collRes, isVer); + retResult.stalled = stalledCount; + + return callback(null, retResult); + } catch (err) { + log.error('An error occurred', { + method: 'getObjectMDStats', + errDetails: { ...err }, + errorString: err.toString(), + }); + return callback(err); + } } /** @@ -377,32 +402,42 @@ class S3UtilsMongoClient extends MongoClientInterface { }; } - updateBucketCapacityInfo(bucketName, capacityInfo, log, cb) { - const m = this.getCollection(METASTORE); - m.findOneAndUpdate({ - _id: bucketName, - }, { - $set: { - '_id': bucketName, - 'value.capabilities.VeeamSOSApi.CapacityInfo': { - Capacity: capacityInfo.Capacity, - Available: capacityInfo.Available, - Used: capacityInfo.Used, - LastModified: (new Date()).toISOString(), + async updateBucketCapacityInfo(bucketName, capacityInfo, log, cb) { + try { + const m = this.getCollection(METASTORE); + const updateResult = await m.findOneAndUpdate({ + _id: bucketName, + }, { + $set: { + '_id': bucketName, + 'value.capabilities.VeeamSOSApi.CapacityInfo': { + Capacity: capacityInfo.Capacity, + Available: capacityInfo.Available, + Used: capacityInfo.Used, + LastModified: (new Date()).toISOString(), + }, }, - }, - }, { - upsert: false, - }, err => { - if (err) { - log.error( - 'updateBucketCapacityInfo: error putting bucket CapacityInfo', - { error: err.message, bucketName, capacityInfo }, - ); - return cb(errors.InternalError); + }, { + upsert: false, + }); + if (!updateResult.ok) { + log.error('updateBucketCapacityInfo: failed to update bucket CapacityInfo', { + bucketName, + capacityInfo, + }); + return cb(new Error('Failed to update bucket CapacityInfo')); } return cb(); - }); + } catch (err) { + log.error('updateBucketCapacityInfo: error putting bucket CapacityInfo', { + error: err.message, + errDetails: { ...err }, + errorString: err.toString(), + bucketName, + capacityInfo, + }); + return cb(errors.InternalError); + } } static convertNumberToLong(obj) { @@ -422,79 +457,63 @@ class S3UtilsMongoClient extends MongoClientInterface { return newObj; } - updateStorageConsumptionMetrics(countItems, dataMetrics, log, cb) { - const updatedStorageMetricsList = [ - { _id: __COUNT_ITEMS, value: countItems }, - // iterate every resource through dataMetrics and add to updatedStorageMetricsList - ...Object.entries(dataMetrics) - .filter(([metricLevel]) => validStorageMetricLevels.has(metricLevel)) - .flatMap(([metricLevel, result]) => Object.entries(result) - .map(([resource, metrics]) => ({ - _id: `${metricLevel}_${resource}`, - measuredOn: new Date().toJSON(), - ...S3UtilsMongoClient.convertNumberToLong(metrics), - }))), - ]; - let tempCollection; - async.series([ - next => this.getCollection(INFOSTORE_TMP).drop(err => { - if (err && err.codeName !== 'NamespaceNotFound') { - return next(err); + async updateStorageConsumptionMetrics(countItems, dataMetrics, log, cb) { + try { + const updatedStorageMetricsList = [ + { _id: __COUNT_ITEMS, value: countItems }, + // iterate every resource through dataMetrics and add to updatedStorageMetricsList + ...Object.entries(dataMetrics) + .filter(([metricLevel]) => validStorageMetricLevels.has(metricLevel)) + .flatMap(([metricLevel, result]) => Object.entries(result) + .map(([resource, metrics]) => ({ + _id: `${metricLevel}_${resource}`, + measuredOn: new Date().toJSON(), + ...S3UtilsMongoClient.convertNumberToLong(metrics), + }))), + ]; + log.info('updateStorageConsumptionMetrics: updating storage metrics'); + + // Drop the temporary collection if it exists + try { + await this.getCollection(INFOSTORE_TMP).drop(); + } catch (err) { + if (err.codeName !== 'NamespaceNotFound') { + throw err; } - return next(); - }), - next => this.db.createCollection(INFOSTORE_TMP, (err, collection) => { - if (err) { - return next(err); - } - tempCollection = collection; - return next(); - }), - next => tempCollection.insertMany(updatedStorageMetricsList, { ordered: false }, next), - next => async.retry( - 3, - done => tempCollection.rename(INFOSTORE, { dropTarget: true }, done), - err => { - if (err) { - log.error('updateStorageConsumptionMetrics: error renaming temp collection, try again', { - error: err.message, - }); - return next(err); - } - return next(); - }, - ), - ], err => { - if (err) { - log.error('updateStorageConsumptionMetrics: error updating count items', { - error: err.message, - }); - return cb(errors.InternalError); } + const tempCollection = await this.db.createCollection(INFOSTORE_TMP); + await tempCollection.insertMany(updatedStorageMetricsList, { ordered: false }); + await async.retry( + 3, + async () => tempCollection.rename(INFOSTORE, { dropTarget: true }), + ); return cb(); - }); + } catch (err) { + log.error('updateStorageConsumptionMetrics: error updating storage metrics', { + error: err, + errDetails: { ...err }, + errorString: err.toString(), + }); + return cb(errors.InternalError); + } } - readStorageConsumptionMetrics(entityName, log, cb) { - const i = this.getCollection(INFOSTORE); - return async.retry( - 3, - done => i.findOne({ - _id: entityName, - }, done), - (err, doc) => { - if (err) { - log.error('readStorageConsumptionMetrics: error reading count items', { - error: err.message, - }); - return cb(errors.InternalError); - } - if (!doc) { - return cb(errors.NoSuchEntity); - } - return cb(null, doc); - }, - ); + async readStorageConsumptionMetrics(entityName, log, cb) { + try { + const i = this.getCollection(INFOSTORE); + const doc = await i.findOne({ _id: entityName }); + if (!doc) { + return cb(errors.NoSuchEntity); + } + return cb(null, doc); + } catch (err) { + log.error('readStorageConsumptionMetrics: error reading metrics', { + error: err, + errDetails: { ...err }, + errorString: err.toString(), + }); + return cb(errors.InternalError); + } } /* @@ -502,74 +521,84 @@ class S3UtilsMongoClient extends MongoClientInterface { * bucket collection exists but bucket is not in metastore collection. * For now, to make the count-items cronjob more robust, we ignore those "bad buckets" */ - getBucketInfos(log, cb) { - const bucketInfos = []; - this.db.listCollections().toArray((err, collInfos) => { - if (err) { - log.error('could not get list of collections', { - method: '_getBucketInfos', - error: err, - }); - return cb(err); - } - return async.eachLimit(collInfos, 10, (value, next) => { + async getBucketInfos(log, cb) { + try { + const bucketInfos = []; + const collInfos = await this.db.listCollections().toArray(); + for (const value of collInfos) { if (this._isSpecialCollection(value.name)) { // skip - return next(); + continue; } const bucketName = value.name; - // FIXME: there is currently no way of distinguishing - // master from versions and searching for VID_SEP - // does not work because there cannot be null bytes - // in $regex - return this.getBucketAttributes(bucketName, log, (err, bucketInfo) => { - if (err) { - if (err.message === 'NoSuchBucket') { - log.debug('bucket does not exist in metastore, ignore it', { - bucketName, - }); - return next(); - } + try { + // eslint-disable-next-line no-await-in-loop + const bucketInfo = await new Promise((resolve, reject) => { + this.getBucketAttributes(bucketName, log, (err, info) => { + if (err) { + reject(err); + } else { + resolve(info); + } + }); + }); + bucketInfos.push(bucketInfo); + } catch (err) { + if (err.message === 'NoSuchBucket') { + log.debug('bucket does not exist in metastore, ignore it', { + bucketName, + }); + } else { log.error('failed to get bucket attributes', { bucketName, - error: err, + errDetails: { ...err }, + errorString: err.toString(), }); - return next(errors.InternalError); + throw errors.InternalError; } - bucketInfos.push(bucketInfo); - return next(); - }); - }, err => { - if (err) { - return cb(err); } - return cb(null, { - bucketCount: bucketInfos.length, - bucketInfos, - }); + } + return cb(null, { + bucketCount: bucketInfos.length, + bucketInfos, }); - }); + } catch (err) { + log.error('could not get list of collections', { + method: '_getBucketInfos', + errDetails: { ...err }, + errorString: err.toString(), + }); + return cb(err); + } } - getUsersBucketCreationDate(ownerId, bucketName, log, cb) { - const usersBucketCol = this.getCollection(USERSBUCKET); - return usersBucketCol.findOne({ - _id: `${ownerId}${constants.splitter}${bucketName}`, - }, { - projection: { - 'value.creationDate': 1, - }, - }, (err, res) => { - if (err) { - log.error('failed to read bucket entry from __usersbucket', { + async getUsersBucketCreationDate(ownerId, bucketName, log, cb) { + try { + const usersBucketCol = this.getCollection(USERSBUCKET); + const res = await usersBucketCol.findOne({ + _id: `${ownerId}${constants.splitter}${bucketName}`, + }, { + projection: { + 'value.creationDate': 1, + }, + }); + if (!res || !res.value || !res.value.creationDate) { + log.error('bucket entry not found in __usersbucket', { bucketName, ownerId, - error: err, }); - return cb(err); + return cb(new Error('Bucket entry not found')); } return cb(null, res.value.creationDate); - }); + } catch (err) { + log.error('failed to read bucket entry from __usersbucket', { + bucketName, + ownerId, + errDetails: { ...err }, + errorString: err.toString(), + }); + return cb(err); + } } } From 9ac7df63c534ffa3eca64aedae666f16439c3b3c Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 26 Oct 2023 10:45:48 +0200 Subject: [PATCH 3/5] S3UTILS-148: improve the count items logging --- CountItems/CountManager.js | 5 +++++ CountItems/CountMaster.js | 5 ++++- CountItems/CountWorker.js | 2 +- CountItems/masterProcess.js | 5 ++++- CountItems/workerProcess.js | 5 ++++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CountItems/CountManager.js b/CountItems/CountManager.js index 356ccfdb..f1f17130 100644 --- a/CountItems/CountManager.js +++ b/CountItems/CountManager.js @@ -36,6 +36,11 @@ class CountManager { } const id = this.workerList.shift(); return this.workers[id].count(bucketInfo, (err, res) => { + this.log.info('processing a bucket', { + method: 'CountManager::_setupQueue', + workInQueue: this.q.length(), + bucketInfo, + }); if (err) { return done(err); } diff --git a/CountItems/CountMaster.js b/CountItems/CountMaster.js index 522c7728..2ccefbe0 100644 --- a/CountItems/CountMaster.js +++ b/CountItems/CountMaster.js @@ -42,11 +42,14 @@ class CountMaster { }); return next(err); } + this.log.info('got buckets infos', { + bucketCount: bucketList.bucketCount, + }); this.manager.addWork(bucketList); return next(); }), next => this.manager.start(next), - next => this.client.updateStorageConsumptionMetrics(this.manager.store, this.manager.dataMetrics, this._log, next), + next => this.client.updateStorageConsumptionMetrics(this.manager.store, this.manager.dataMetrics, this.log, next), ], err => { if (err) { this.log.error('error occurred in count items job', { diff --git a/CountItems/CountWorker.js b/CountItems/CountWorker.js index 6b76ff90..41fc9ec0 100644 --- a/CountItems/CountWorker.js +++ b/CountItems/CountWorker.js @@ -31,7 +31,7 @@ class CountWorker { } const bucketInfo = BucketInfo.fromObj(bucketInfoObj); const bucketName = bucketInfo.getName(); - this.log.debug(`${process.pid} handling ${bucketName}`); + this.log.info(`${process.pid} handling ${bucketName}`); return async.waterfall([ next => this.client._getIsTransient(bucketInfo, this.log, next), (isTransient, next) => this.client.getObjectMDStats(bucketName, bucketInfo, isTransient, this.log, next), diff --git a/CountItems/masterProcess.js b/CountItems/masterProcess.js index 6fb5704d..35888508 100644 --- a/CountItems/masterProcess.js +++ b/CountItems/masterProcess.js @@ -7,8 +7,11 @@ const CountManager = require('./CountManager'); const createMongoParams = require('../utils/createMongoParams'); const createWorkers = require('./utils/createWorkers'); +const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1 + ? 'debug' : 'info'; + const loggerConfig = { - level: 'debug', + level: logLevel, dump: 'error', }; diff --git a/CountItems/workerProcess.js b/CountItems/workerProcess.js index e48a9f7c..090c48b6 100644 --- a/CountItems/workerProcess.js +++ b/CountItems/workerProcess.js @@ -5,8 +5,11 @@ const S3UtilsMongoClient = require('../utils/S3UtilsMongoClient'); const CountWorker = require('./CountWorker'); const createMongoParams = require('../utils/createMongoParams'); +const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1 + ? 'debug' : 'info'; + const loggerConfig = { - level: 'info', + level: logLevel, dump: 'error', }; From 32400d04b0a2fb490019c612ef7501a21fd9b2c7 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 26 Oct 2023 10:46:12 +0200 Subject: [PATCH 4/5] S3UTILS-148: fix test framework issue --- tests/conf/envSetup.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/conf/envSetup.js b/tests/conf/envSetup.js index d0d326e7..17e8e4c7 100644 --- a/tests/conf/envSetup.js +++ b/tests/conf/envSetup.js @@ -1,2 +1,7 @@ +const { TextEncoder, TextDecoder } = require('util'); + +global.TextEncoder = TextEncoder; +global.TextDecoder = TextDecoder; + process.env.BUCKETD_HOSTPORT = 'localhost:9000'; process.env.SPROXYD_HOSTPORT = 'localhost'; From abe7c14243c649a3c6948316fd08db9878c90e47 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 26 Oct 2023 10:47:18 +0200 Subject: [PATCH 5/5] S3UTILS-148: bump project version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 028174dc..15f4b3e9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3utils", - "version": "1.14.1", + "version": "1.14.2", "engines": { "node": ">= 16" },