handle UnicodeDecodeError on non-utf-8 check_website

monosans · Mar 6, 2025 · 6a6e05e · 6a6e05e
1 parent c9bd280
commit 6a6e05e
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py
@@ -69,7 +69,7 @@ async def check(self, *, settings: Settings) -> None:
             self.exit_ip = parse_ipv4(orjson.loads(content)["origin"])
         elif settings.check_website_type == CheckWebsiteType.PLAIN_IP:
             self.exit_ip = parse_ipv4(
-                content.decode(response.get_encoding(), errors="replace")
+                content.decode(response.get_encoding())
             )
         else:
             self.exit_ip = None

diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py
@@ -158,8 +158,10 @@ async def _get_check_website_type_and_real_ip(
     except orjson.JSONDecodeError:
         try:
             return CheckWebsiteType.PLAIN_IP, parse_ipv4(
-                content.decode(response.get_encoding(), errors="replace")
+                content.decode(response.get_encoding())
             )
+        except UnicodeDecodeError:
+            _logger.error("Error when decoding check_website response as utf-8")
         except ValueError:
             pass
     else: