Merge pull request #2120 from openzim/no_s3_concurrency

Remove S3 upload concurrency
openzim · Jan 3, 2025 · a4db759 · a4db759
2 parents b66f1a9 + ea3db9b
commit a4db759
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 10 deletions.
diff --git a/src/Downloader.ts b/src/Downloader.ts
@@ -588,7 +588,7 @@ class Downloader {
           // Check for the ETag and upload to cache
           const etag = this.removeEtagWeakPrefix(mwResp.headers.etag)
           if (etag) {
-            this.s3.uploadBlob(stripHttpFromUrl(url), mwResp.data, etag, mwResp.headers['content-type'], this.webp ? 'webp' : '1')
+            await this.s3.uploadBlob(stripHttpFromUrl(url), mwResp.data, etag, mwResp.headers['content-type'], this.webp ? 'webp' : '1')
           }
 
           // Proceed with image

diff --git a/src/S3.ts b/src/S3.ts
@@ -3,9 +3,6 @@ import * as logger from './Logger.js'
 import { Readable } from 'stream'
 import { publicIpv4 } from 'public-ip'
 
-import * as https from 'https'
-import { NodeHttpHandler } from '@smithy/node-http-handler'
-
 interface BucketParams {
   Bucket: string
   Key: string
@@ -37,11 +34,6 @@ class S3 {
   }
 
   public async initialise() {
-    const requestHandler = new NodeHttpHandler({
-      httpsAgent: new https.Agent({
-        maxSockets: 250,
-      }),
-    })
     const s3UrlBase: any = new URL(this.url)
     this.s3Handler = new S3Client({
       credentials: {
@@ -51,7 +43,6 @@ class S3 {
       endpoint: s3UrlBase.href,
       forcePathStyle: s3UrlBase.protocol === 'http:',
       region: this.region,
-      requestHandler,
     })
 
     return this.bucketExists(this.bucketName)