Use a separate field Resource.article to hold the parsed/cleaned-up…

… DOM tree, so it’s clear whether article extraction has been performed or not. This also leaves the original `Document` untouched, so chained plugins can continue to operate on the original DOM tree.
chimbori · May 24, 2022 · 30c2267 · 30c2267
1 parent cd85faa
commit 30c2267
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/src/main/kotlin/com/chimbori/crux/Crux.kt b/src/main/kotlin/com/chimbori/crux/Crux.kt
@@ -96,6 +96,9 @@ public data class Resource(
   /** Parsed DOM tree for this resource, if available. */
   val document: Document? = null,
 
+  /** Extracted and cleaned-up DOM tree for this resource, if available. */
+  val article: Document? = null,
+
   /**
    * Text fields extracted from this resource, stored as key-value pairs. It is recommended to use well-defined keys
    * from [com.chimbori.crux.Fields] for all standard fields. Custom fields are also supported, in case none of the
@@ -126,8 +129,10 @@ public data class Resource(
   public operator fun plus(anotherResource: Resource): Resource = Resource(
     url = anotherResource.url ?: url,
     document = anotherResource.document ?: document,
+    article = anotherResource.article ?: article,
     fields = fields + anotherResource.fields,
     urls = urls + anotherResource.urls,
+    objects = objects + anotherResource.objects,
   )
 
   /**
@@ -137,6 +142,7 @@ public data class Resource(
   public fun removeNullValues(): Resource = copy(
     fields = fields.filterValues { !it.isNullOrBlank() },
     urls = urls.filterValues { it != null },
+    objects = objects.filterValues { it != null },
   )
 
   /** For any potential extension functions to be defined on the companion object. */

diff --git a/src/main/kotlin/com/chimbori/crux/plugins/ArticleExtractor.kt b/src/main/kotlin/com/chimbori/crux/plugins/ArticleExtractor.kt
@@ -37,7 +37,7 @@ public class ArticleExtractor : Plugin {
     val extractedDoc = PostprocessHelpers.postprocess(bestMatchElement)
     return Resource(
       objects = mapOf(DURATION_MS to extractedDoc.estimatedReadingTimeMs()),
-      document = extractedDoc
+      article = extractedDoc
     )
   }
 }
diff --git a/src/test/kotlin/com/chimbori/crux/plugins/ArticleExtractorTest.kt b/src/test/kotlin/com/chimbori/crux/plugins/ArticleExtractorTest.kt
@@ -24,7 +24,7 @@ class ArticleExtractorTest {
       val readingTimeMinutes = (parsed.objects.get(DURATION_MS) as? Int)?.div(60_000)
       assertEquals(51, readingTimeMinutes)
 
-      val extractedArticle = parsed.document
+      val extractedArticle = parsed.article
       assertNotNull(extractedArticle)
       assertStartsWith(
         """"Galileo" redirects here. For other uses, see Galileo (disambiguation) and Galileo Galilei (disambiguation).""",