connections and loading issue on specific sites (#335)

* Test sites watsons.com.sg and guardian.com.sg * Update playwright to 1.44.0 and bump package --------- Co-authored-by: younglim <[email protected]>
GovTechSG · May 22, 2024 · 2c917ca · 2c917ca
1 parent 2464d09
commit 2c917ca
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 13 deletions.
diff --git a/constants/common.js b/constants/common.js
@@ -141,6 +141,7 @@ const document = new JSDOM('').window;
 const httpsAgent = new https.Agent({
   // Run in environments with custom certificates
   rejectUnauthorized: false,
+  keepAlive: true,
 });
 
 export const messageOptions = {
@@ -684,6 +685,7 @@ const getRobotsTxtViaAxios = async (robotsUrl) => {
   const instance = axios.create({
     httpsAgent: new https.Agent({
       rejectUnauthorized: false,
+      keepAlive: true,
     }),
   });
 
@@ -855,6 +857,7 @@ export const getLinksFromSitemap = async (
           const instance = axios.create({
             httpsAgent: new https.Agent({
               rejectUnauthorized: false,
+              keepAlive: true,
             }),
           });
           data = await (await instance.get(url, { timeout: 80000 })).data;

diff --git a/crawlers/crawlDomain.js b/crawlers/crawlDomain.js
@@ -315,7 +315,7 @@ const crawlDomain = async (
         }
 
         // Ensure page navigation completes to capture final URL in a redirect chain
-        await page.goto(request.url, { waitUntil: 'networkidle' });
+        await page.goto(request.url, { waitUntil: 'load' });
 
         let finalUrl = page.url(); // Initialize with the request URL
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/purple-hats",
   "main": "npmIndex.js",
-  "version": "0.9.52",
+  "version": "0.9.53",
   "type": "module",
   "imports": {
     "#root/*.js": "./*.js"
@@ -20,7 +20,7 @@
     "lodash": "^4.17.21",
     "minimatch": "^9.0.3",
     "pdfjs-dist": "github:veraPDF/pdfjs-dist#v2.14.305-taggedPdf-0.1.11",
-    "playwright": "1.42.1",
+    "playwright": "1.44.0",
     "prettier": "^3.1.0",
     "print-message": "^3.0.1",
     "safe-regex": "^2.1.1",