Fix basic auth sitemap scan

GovTechSG · Jun 6, 2024 · 3c539c9 · 3c539c9
1 parent d85496d
commit 3c539c9
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 5 deletions.
diff --git a/src/constants/common.ts b/src/constants/common.ts
@@ -750,7 +750,7 @@ export const getLinksFromSitemap = async (
       ? (url = addBasicAuthCredentials(url, username, password))
       : url;
 
-    const request = new Request({ url: encodeURI(url) });
+    const request = new Request({ url: url });
     if (isUrlPdf(url)) {
       request.skipNavigation = true;
     }

diff --git a/src/crawlers/crawlSitemap.ts b/src/crawlers/crawlSitemap.ts
@@ -114,7 +114,7 @@ const crawlSitemap = async (
 
 
   printMessage(['Fetching URLs. This might take some time...'], { border: false });
-
+  
 
   finalLinks = [...finalLinks, ...linksFromSitemap];
 
@@ -149,8 +149,6 @@ const crawlSitemap = async (
     preNavigationHooks: isBasicAuth
       ? [
         async ({ page, request }) => {
-
-          request.url = encodeURI(request.url);
           await page.setExtraHTTPHeaders({
             Authorization: authHeader,
             ...extraHTTPHeaders,
@@ -159,7 +157,6 @@ const crawlSitemap = async (
       ]
       : [
         async ({ page, request }) => {
-        request.url = encodeURI(request.url);
         preNavigationHooks(extraHTTPHeaders)
         //insert other code here
         },