Merge pull request #318 from hearchco/as/fix/headers

fix: headers in colly and image proxy
hearchco · May 28, 2024 · 796f6f1 · 796f6f1
2 parents 1754815 + e12e5fd
commit 796f6f1
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/src/router/proxy.go b/src/router/proxy.go
@@ -69,7 +69,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, salt string, timeouts config.
 		ProtoMinor: 1,
 		Header: map[string][]string{
 			"Accept":          {"image/avif", "image/webp", "image/apng", "image/svg+xml", "image/*", "*/*;q=0.8"},
-			"Accept-Encoding": {"gzip", "deflate", "br"}, // Google Chrome also has "zstd" but that isn't supported by Firefox and Safari
+			"Accept-Encoding": {"gzip", "deflate", "br", "zstd"}, // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
 			"Accept-Language": {"en-US,en;q=0.9"},
 			// "Connection":   {"keep-alive"}, // commented since it's not present by default in Google Chrome
 			// "DNT":          {"1"}, // do not track, commented since it's not present by default in Google Chrome

diff --git a/src/search/engines/_sedefaults/init.go b/src/search/engines/_sedefaults/init.go
@@ -24,7 +24,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
 		colly.UserAgent(userAgent),
 		colly.IgnoreRobotsTxt(),
 		colly.Headers(map[string]string{
-			"Sec-Ch-Ua": secChUa,
+			"Accept":             "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+			"Accept-Encoding":    "gzip, deflate, br, zstd", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
+			"Accept-Language":    "en-US,en;q=0.9",
+			"Sec-Ch-Ua":          secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
+			"Sec-Ch-Ua-Mobile":   "?0",
+			"Sec-Ch-Ua-Platform": "\"Windows\"",
+			"Sec-Fetch-Dest":     "document",
+			"Sec-Fetch-Mode":     "navigate",
+			"Sec-Fetch-Site":     "none",
 		}),
 	)
 	pagesCol := colly.NewCollector(
@@ -33,7 +41,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
 		colly.UserAgent(userAgent),
 		colly.IgnoreRobotsTxt(),
 		colly.Headers(map[string]string{
-			"Sec-Ch-Ua": secChUa,
+			"Accept":             "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+			"Accept-Encoding":    "gzip, deflate, br, zstd", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
+			"Accept-Language":    "en-US,en;q=0.9",
+			"Sec-Ch-Ua":          secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
+			"Sec-Ch-Ua-Mobile":   "?0",
+			"Sec-Ch-Ua-Platform": "\"Windows\"",
+			"Sec-Fetch-Dest":     "document",
+			"Sec-Fetch-Mode":     "navigate",
+			"Sec-Fetch-Site":     "none",
 		}),
 	)