From e12e5fd6280b2971860a6b53059c321756001a12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
 <31509435+aleksasiriski@users.noreply.github.com>
Date: Tue, 28 May 2024 21:44:15 +0200
Subject: [PATCH] fix: headers in colly and image proxy

---
 src/router/proxy.go                    |  2 +-
 src/search/engines/_sedefaults/init.go | 20 ++++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/router/proxy.go b/src/router/proxy.go
index 428b46f9..e0eb2eec 100644
--- a/src/router/proxy.go
+++ b/src/router/proxy.go
@@ -69,7 +69,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, salt string, timeouts config.
 		ProtoMinor: 1,
 		Header: map[string][]string{
 			"Accept":          {"image/avif", "image/webp", "image/apng", "image/svg+xml", "image/*", "*/*;q=0.8"},
-			"Accept-Encoding": {"gzip", "deflate", "br"}, // Google Chrome also has "zstd" but that isn't supported by Firefox and Safari
+			"Accept-Encoding": {"gzip", "deflate", "br", "zstd"}, // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
 			"Accept-Language": {"en-US,en;q=0.9"},
 			// "Connection":   {"keep-alive"}, // commented since it's not present by default in Google Chrome
 			// "DNT":          {"1"}, // do not track, commented since it's not present by default in Google Chrome
diff --git a/src/search/engines/_sedefaults/init.go b/src/search/engines/_sedefaults/init.go
index cb6bb0d4..ca1d12b0 100644
--- a/src/search/engines/_sedefaults/init.go
+++ b/src/search/engines/_sedefaults/init.go
@@ -24,7 +24,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
 		colly.UserAgent(userAgent),
 		colly.IgnoreRobotsTxt(),
 		colly.Headers(map[string]string{
-			"Sec-Ch-Ua": secChUa,
+			"Accept":             "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+			"Accept-Encoding":    "gzip, deflate, br, zstd", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
+			"Accept-Language":    "en-US,en;q=0.9",
+			"Sec-Ch-Ua":          secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
+			"Sec-Ch-Ua-Mobile":   "?0",
+			"Sec-Ch-Ua-Platform": "\"Windows\"",
+			"Sec-Fetch-Dest":     "document",
+			"Sec-Fetch-Mode":     "navigate",
+			"Sec-Fetch-Site":     "none",
 		}),
 	)
 	pagesCol := colly.NewCollector(
@@ -33,7 +41,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
 		colly.UserAgent(userAgent),
 		colly.IgnoreRobotsTxt(),
 		colly.Headers(map[string]string{
-			"Sec-Ch-Ua": secChUa,
+			"Accept":             "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+			"Accept-Encoding":    "gzip, deflate, br, zstd", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
+			"Accept-Language":    "en-US,en;q=0.9",
+			"Sec-Ch-Ua":          secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
+			"Sec-Ch-Ua-Mobile":   "?0",
+			"Sec-Ch-Ua-Platform": "\"Windows\"",
+			"Sec-Fetch-Dest":     "document",
+			"Sec-Fetch-Mode":     "navigate",
+			"Sec-Fetch-Site":     "none",
 		}),
 	)