From 3c539c96d7afce8d1d2486ceada5b3b564c4c1c1 Mon Sep 17 00:00:00 2001
From: younglim <younglim@users.noreply.github.com>
Date: Thu, 6 Jun 2024 17:35:15 +0800
Subject: [PATCH] Fix basic auth sitemap scan

---
 src/constants/common.ts      | 2 +-
 src/crawlers/crawlSitemap.ts | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/constants/common.ts b/src/constants/common.ts
index 0f63401e..58f9e54b 100644
--- a/src/constants/common.ts
+++ b/src/constants/common.ts
@@ -750,7 +750,7 @@ export const getLinksFromSitemap = async (
       ? (url = addBasicAuthCredentials(url, username, password))
       : url;
 
-    const request = new Request({ url: encodeURI(url) });
+    const request = new Request({ url: url });
     if (isUrlPdf(url)) {
       request.skipNavigation = true;
     }
diff --git a/src/crawlers/crawlSitemap.ts b/src/crawlers/crawlSitemap.ts
index dadaf87d..297e51bc 100644
--- a/src/crawlers/crawlSitemap.ts
+++ b/src/crawlers/crawlSitemap.ts
@@ -114,7 +114,7 @@ const crawlSitemap = async (
 
 
   printMessage(['Fetching URLs. This might take some time...'], { border: false });
-
+  
 
   finalLinks = [...finalLinks, ...linksFromSitemap];
 
@@ -149,8 +149,6 @@ const crawlSitemap = async (
     preNavigationHooks: isBasicAuth
       ? [
         async ({ page, request }) => {
-
-          request.url = encodeURI(request.url);
           await page.setExtraHTTPHeaders({
             Authorization: authHeader,
             ...extraHTTPHeaders,
@@ -159,7 +157,6 @@ const crawlSitemap = async (
       ]
       : [
         async ({ page, request }) => {
-        request.url = encodeURI(request.url);
         preNavigationHooks(extraHTTPHeaders)
         //insert other code here
         },