Backport recent v3 changes

This is a backport of recent changes from the 3 branch. Its provided as a minimal effort change so doesn't include tests, comments docs etc as CMSv4 is out of full support

_config/extensions.yml

@@ -26,4 +26,5 @@ Only:
 ---
 SilverStripe\CMS\Model\SiteTree:
   extensions:
-    - SilverStripe\SearchService\Extensions\SearchServiceExtension
+    SearchServiceExtension: SilverStripe\SearchService\Extensions\SearchServiceExtension
+    SiteTreeHierarchyExtension: SilverStripe\SearchService\Extensions\SiteTreeHierarchyExtension

composer.json

@@ -18,7 +18,8 @@
         "guzzlehttp/guzzle": "^7",
         "symbiote/silverstripe-queuedjobs": "^4",
         "elastic/enterprise-search": "^8.3",
-        "silverstripe/versioned": "^1"
+        "silverstripe/versioned": "^1",
+        "silverstripe/cms": "^4"
     },
     "require-dev": {
         "phpunit/phpunit": "^9.5",

src/DataObject/DataObjectDocument.php

@@ -19,6 +19,7 @@
 use SilverStripe\ORM\RelationList;
 use SilverStripe\ORM\UnsavedRelationList;
 use SilverStripe\SearchService\Exception\IndexConfigurationException;
+use SilverStripe\SearchService\Exception\IndexingServiceException;
 use SilverStripe\SearchService\Extensions\DBFieldExtension;
 use SilverStripe\SearchService\Extensions\SearchServiceExtension;
 use SilverStripe\SearchService\Interfaces\DataObjectDocumentInterface;
@@ -147,8 +148,10 @@ public function shouldIndex(): bool
             return false;
         }
 
-        // Dataobject is only in draft
-        if ($dataObject->hasExtension(Versioned::class) && !$dataObject->isLiveVersion()) {
+        // DataObject has no published version (or draft changes could cause a doc to be removed)
+        if ($dataObject->hasExtension(Versioned::class) && !$dataObject->isPublished()) {
+            // note even if we pass a draft object to the indexer onAddToSearchIndexes will
+            // set the version to live before adding
             return false;
         }
 
@@ -204,24 +207,20 @@ public function getIndexes(): array
     /**
      * Generates a map of all the fields and values which will be sent
      *
+     * This will always use the current DataObject so you must ensure
+     * it is in the correct state (eg Live) prior to calling toArray.
+     * For example the onAddToSearchIndexes method will set the data
+     * object to LIVE when adding to the index
+     *
+     * @see DataObjectDocument::onAddToSearchIndexes()
      * @throws IndexConfigurationException
      */
     public function toArray(): array
     {
         $pageContentField = $this->config()->get('page_content_field');
 
-        if ($this->getDataObject()->hasExtension(Versioned::class)) {
-            $dataObject = Versioned::withVersionedMode(function () {
-                Versioned::set_stage(Versioned::LIVE);
-
-                return DataObject::get_by_id($this->getSourceClass(), $this->getDataObject()->ID);
-            });
-        } else {
-            $dataObject = DataObject::get_by_id(
-                $this->getSourceClass(),
-                $this->getDataObject()->ID
-            );
-        }
+        // assume shouldIndex is called before this
+        $dataObject = $this->getDataObject();
 
         if (!$dataObject || !$dataObject->exists()) {
             throw new IndexConfigurationException(
@@ -371,6 +370,12 @@ public function getFieldValue(Field $field): mixed
     }
 
     /**
+     * Collects documents that depend on the current DataObject for indexing.
+     * It will inspect the search index configuration for anything using this object in a field or, if the
+     * current object is an instance of SiteTree, it will respect `enforce_strict_hierarchy`
+     * and add any child objects.
+     *
+     * @see [the dependency tracking docs](docs/en/usage.md#dependency-tracking)
      * @return DocumentInterface[]
      */
     public function getDependentDocuments(): array
@@ -440,15 +445,15 @@ public function getDependentDocuments(): array
 
                     /** @var DataObjectDocument $candidateDocument */
                     foreach ($chunker->chunk() as $candidateDocument) {
-                        $relatedObj = $candidateDocument->getFieldValue($field);
+                        $relatedObj = $candidateDocument->getFieldDependency($field);
 
                         // Singleton returned a dataobject, but this record did not. Rare, but possible.
                         if (!$relatedObj instanceof $objectClass) {
                             continue;
                         }
 
                         if ($relatedObj->ID === $ownedDataObject->ID) {
-                            $docs[$document->getIdentifier()] = $document;
+                            $docs[$candidateDocument->getIdentifier()] = $candidateDocument;
                         }
                     }
                 }
@@ -622,8 +627,32 @@ public function __unserialize(array $data): void
         }
     }
 
+    /**
+     * Add to index event handler
+     *
+     * @throws IndexingServiceException
+     * @return void
+     */
     public function onAddToSearchIndexes(string $event): void
     {
+        if ($event === DocumentAddHandler::BEFORE_ADD) {
+            // make sure DataObject is always live on adding to the index
+            Versioned::withVersionedMode(function (): void {
+                Versioned::set_stage(Versioned::LIVE);
+
+                $currentDataObject = $this->getDataObject();
+
+                $liveDataObject = DataObject::get($currentDataObject->ClassName)->byID($currentDataObject->ID);
+
+                if (!$liveDataObject) {
+                    // unlikely case as indexer calls 'shouldIndex' immediately prior to this
+                    throw new IndexingServiceException('Only published DataObjects may be added to the index');
+                }
+
+                $this->setDataObject($liveDataObject);
+            });
+        }
+
         if ($event === DocumentAddHandler::AFTER_ADD) {
             $this->markIndexed();
         }

src/Extensions/SiteTreeHierarchyExtension.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace SilverStripe\SearchService\Extensions;
+
+use SilverStripe\CMS\Model\SiteTree;
+use SilverStripe\Core\Extension;
+use SilverStripe\SearchService\DataObject\DataObjectDocument;
+
+class SiteTreeHierarchyExtension extends Extension
+{
+
+    public function updateSearchDependentDocuments(array &$dependentDocs): void
+    {
+        if (!SiteTree::config()->get('enforce_strict_hierarchy')) {
+            return;
+        }
+
+        $page = $this->getOwner();
+
+        foreach ($page->AllChildren() as $record) {
+            $document = DataObjectDocument::create($record);
+            $dependentDocs[$document->getIdentifier()] = $document;
+            $dependentDocs = array_merge($dependentDocs, $document->getDependentDocuments());
+        }
+    }
+
+}

src/Jobs/RemoveDataObjectJob.php

@@ -11,6 +11,10 @@
 use SilverStripe\Versioned\Versioned;
 
 /**
+ *  By virtue of the default parameter, Index::METHOD_ADD, this does not remove the documents straight away.
+ *  It checks first the status of the underlying DataObjects and decides whether to remove or add them to the index.
+ *  Then pass on to the parent's process() method to handle the job.
+ *
  * @property DataObjectDocument|null $document
  * @property int|null $timestamp
  */
@@ -19,6 +23,8 @@ class RemoveDataObjectJob extends IndexJob
 
     public function __construct(?DataObjectDocument $document = null, ?int $timestamp = null, ?int $batchSize = null)
     {
+        // Indexer::METHOD_ADD as default parameter make sure we check first its related documents
+        // whether we should delete or update them automatically.
         parent::__construct([], Indexer::METHOD_ADD, $batchSize);
 
         if ($document !== null) {
@@ -46,26 +52,43 @@ public function getTitle(): string
      */
     public function setup(): void
     {
-        // Set the documents in setup to ensure async
+        /** @var DBDatetime $datetime - set the documents in setup to ensure async */
         $datetime = DBField::create_field('Datetime', $this->getTimestamp());
         $archiveDate = $datetime->format($datetime->getISOFormat());
         $documents = Versioned::withVersionedMode(function () use ($archiveDate) {
             Versioned::reading_archived_date($archiveDate);
 
+            $currentDocument = $this->getDocument();
             // Go back in time to find out what the owners were before unpublish
-            $dependentDocs = $this->document->getDependentDocuments();
+            $dependentDocs = $currentDocument->getDependentDocuments();
 
             // refetch everything on the live stage
             Versioned::set_stage(Versioned::LIVE);
 
-            return array_map(function (DataObjectDocument $doc) {
-                return DataObjectDocument::create(
-                    DataObject::get_by_id(
-                        $doc->getSourceClass(),
-                        $doc->getDataObject()->ID
-                    )
-                );
-            }, $dependentDocs);
+            return array_reduce(
+                $dependentDocs,
+                function (array $carry, DataObjectDocument $doc) {
+                    $record = DataObject::get_by_id($doc->getSourceClass(), $doc->getDataObject()->ID);
+
+                    // Since SiteTree::onBeforeDelete recursively deletes the child pages,
+                    // they end up not found on a live environment which breaks DataObjectDocument::_constructor
+                    if ($record) {
+                        $document = DataObjectDocument::create($record);
+                        $carry[$document->getIdentifier()] = $document;
+
+                        return $carry;
+                    }
+
+                    // Taking into account that this queued job has a reference of existing child pages
+                    // We need to make sure that we are able to send these pages to ElasticSearch etc. for removal
+                    $oldRecord = $doc->getDataObject();
+                    $document = DataObjectDocument::create($oldRecord);
+                    $carry[$document->getIdentifier()] = $document;
+
+                    return $carry;
+                },
+                []
+            );
         });
 
         $this->setDocuments($documents);