Add support for SVG favicon + upgrade to scraperlib 4.0.0 (support auto PDF indexing)

CHANGELOG.md

@@ -12,9 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - New fuzzy-rule for cheatography.com (#342), der-postillon.com (#330)
 - Properly rewrite redirect target url when present in <meta> HTML tag (#237)
 - New `--encoding-aliases` argument to pass encoding/charset aliases (#331)
+- Add support for SVG favicon (#148)
+- Automatically index PDF content and use PDF title (#289 and #290)
 
 ### Changed
 
+- Upgrade to python-scraperlib 4.0.0
 - Generate fuzzy rules tests in Python and Javascript (#284)
 - Refactor HTML rewriter class to make it more open to change and expressive (#305)
 - Detect charset in document header only for HTML documents (#331)

Dockerfile

@@ -3,7 +3,7 @@ LABEL org.opencontainers.image.source https://github.com/openzim/warc2zim
 
 RUN apt-get update -y \
  && apt-get install -y --no-install-recommends \
-    locales-all libmagic1 \
+    locales-all libmagic1 libcairo2 \
  && rm -rf /var/lib/apt/lists/* \
  && python -m pip install --no-cache-dir -U \
       pip \

pyproject.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 dependencies = [
   "warcio==1.7.4",
   "requests==2.32.3",
-  "zimscraperlib==3.3.2",
+  "zimscraperlib==4.0.0",
   "jinja2==3.1.4", # also update version in build-system above and in build_js.sh
   # to support possible brotli content in warcs, must be added separately
   "brotlipy==0.7.0",

src/warc2zim/converter.py

@@ -41,7 +41,8 @@
     RECOMMENDED_MAX_TITLE_LENGTH,
 )
 from zimscraperlib.download import stream_file
-from zimscraperlib.image.convertion import convert_image
+from zimscraperlib.image.conversion import convert_image, convert_svg2png
+from zimscraperlib.image.probing import format_for
 from zimscraperlib.image.transformation import resize_image
 from zimscraperlib.types import FALLBACK_MIME
 from zimscraperlib.zim.creator import Creator
@@ -776,12 +777,11 @@ def convert_illustration(self):
         src = io.BytesIO(self.illustration)
         dst = io.BytesIO()
         try:
-            convert_image(
-                src,  # pyright: ignore[reportGeneralTypeIssues, reportArgumentType]
-                dst,  # pyright: ignore[reportGeneralTypeIssues, reportArgumentType]
-                fmt="PNG",  # pyright: ignore[reportGeneralTypeIssues, reportArgumentType]
-            )
-            resize_image(dst, width=48, height=48, method="cover")
+            if format_for(src, from_suffix=False) == "SVG":
+                convert_svg2png(src, dst, width=48, height=48)
+            else:
+                convert_image(src, dst, fmt="PNG")
+                resize_image(dst, width=48, height=48, method="cover")
         except Exception as exc:  # pragma: no cover
             logger.warning(f"Failed to convert or resize favicon: {exc}")
             self.illustration = DEFAULT_DEV_ZIM_METADATA["Illustration_48x48_at_1"]

src/warc2zim/items.py

@@ -56,7 +56,9 @@ def __init__(
         ).rewrite(pre_head_template, post_head_template)
 
     def get_hints(self):
-        is_front = self.mimetype.startswith("text/html")
+        is_front = self.mimetype.startswith("text/html") or self.mimetype.startswith(
+            "application/pdf"
+        )
         return {Hint.FRONT_ARTICLE: is_front}
 
 
@@ -67,7 +69,7 @@ class StaticArticle(StaticItem):
     """
 
     def __init__(self, filename: Path, main_path: str, **kwargs):
-        super().__init__(**kwargs)
+        super().__init__(auto_index=False, **kwargs)
         self.filename = filename
         self.main_path = main_path
 
@@ -90,6 +92,7 @@ class StaticFile(StaticItem):
     """A file to store in _zim_static folder, based on known content and mimetype"""
 
     def __init__(self, content: str | bytes, filename: str, mimetype: str):
+        super().__init__(auto_index=False)
         self.filename = filename
         self.mime = mimetype
         self.content = content

src/warc2zim/language.py

@@ -18,9 +18,9 @@ def parse_language(input_lang: str) -> str:
     for lang in [lang.strip() for lang in input_lang.split(",")]:
         try:
             lang_data = get_language_details(lang)
-            parsed_lang = lang_data["iso-639-3"]
-            if parsed_lang not in langs:
-                langs.append(parsed_lang)
+            if parsed_lang := (lang_data.iso_639_3 if lang_data else None):
+                if parsed_lang not in langs:
+                    langs.append(parsed_lang)
         except Exception:
             logger.warning(f"Skipping invalid language setting `{lang}`.")
             continue  # skip unrecognized

tests/test_warc_to_zim.py

@@ -10,7 +10,7 @@
 
 import pytest
 import requests
-from zimscraperlib.image.convertion import convert_image, resize_image
+from zimscraperlib.image.conversion import convert_image, resize_image
 from zimscraperlib.zim import Archive
 
 from warc2zim.__about__ import __version__
@@ -218,13 +218,7 @@ def rebuild_favicon_bytes(self, zim, favicon_path) -> bytes:
         )
         assert favicon_bytes
         dst = io.BytesIO()
-        convert_image(
-            io.BytesIO(
-                favicon_bytes
-            ),  # pyright: ignore[reportGeneralTypeIssues, reportArgumentType]
-            dst,  # pyright: ignore[reportGeneralTypeIssues, reportArgumentType]
-            fmt="PNG",  # pyright: ignore[reportGeneralTypeIssues, reportArgumentType]
-        )
+        convert_image(io.BytesIO(favicon_bytes), dst, fmt="PNG")
         resize_image(dst, width=48, height=48, method="cover")
         return dst.getvalue()