Detect rewrite mode based on WARC-Resource-Type when available

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Use the new `WARC-Resource-Type` header to decide rewrite mode (when present in WARC) (#296)
+
 ### Fixed
 
 - Drop `integrity` attribute in HTML `<script>` and `<link>` tags (#298)

src/warc2zim/content_rewriting/generic.py

@@ -122,6 +122,64 @@ def rewrite(
         return ("", self.content)
 
     def get_rewrite_mode(self, record, mimetype):
+        """Get current record rewrite mode
+
+        The rewrite mode is used to decide which kind of resource we have (html, css,
+        js, ...) and this is used to decide how it should be parsed and rewritten.
+        """
+        mimetype_rewrite_mode = self.get_mimetype_rewrite_mode(record, mimetype)
+
+        resourcetype = record.rec_headers["WARC-Resource-Type"]
+        if not resourcetype:
+            return mimetype_rewrite_mode  # fallback for WARCs without resource type
+        if not isinstance(resourcetype, str):
+            raise Exception(f"Unsupported resourcetype class: {resourcetype.__class__}")
+        resourcetype = resourcetype.lower().strip()
+
+        resourcetype_rewrite_mode = self.get_resourcetype_rewrite_mode(
+            record, resourcetype, mimetype
+        )
+
+        if mimetype_rewrite_mode != resourcetype_rewrite_mode:
+            logger.warning(
+                f"Rewrite mode has changed in 2.0.1 for {self.path.value} record: was "
+                f"{mimetype_rewrite_mode}, now is {resourcetype_rewrite_mode} ("
+                f"mimetype: {mimetype}, resourcetype: {resourcetype})"
+            )
+
+    def get_resourcetype_rewrite_mode(self, record, resourcetype, mimetype):
+        """Get current record rewrite mode based on WARC-Resource-Type and mimetype"""
+
+        if resourcetype == "document":
+            # TODO : Handle header "Accept" == "application/json"
+            if getattr(record, "method", "GET") == "GET":
+                return "html"
+
+            return None
+
+        if resourcetype == "stylesheet":
+            return "css"
+
+        if resourcetype in ["script", "fetch"] and (
+            mimetype == "application/json" or self.path.value.endswith(".json")
+        ):
+            return "json"
+
+        if resourcetype == "script" and mimetype in [
+            "text/javascript",
+            "application/javascript",
+            "application/x-javascript",
+        ]:
+            if extract_jsonp_callback(self.orig_url_str):
+                return "jsonp"
+
+            return "javascript"
+
+        return None
+
+    def get_mimetype_rewrite_mode(self, record, mimetype):
+        """Get current record rewrite mode based on mimetype"""
+
         if mimetype == "text/html":
             if getattr(record, "method", "GET") == "POST":
                 return None