Merge pull request #48 from werstreamtes/main

Enable batch processing in ReindexTask
WPP-Public · Nov 18, 2024 · a420be9 · a420be9
2 parents 98f1945 + 8b4e83c
commit a420be9
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 17 deletions.
diff --git a/src/ElasticaService.php b/src/ElasticaService.php
@@ -84,7 +84,7 @@ class ElasticaService
      */
     public function __construct(
         Client $client,
-        $indexName,
+               $indexName,
         LoggerInterface $logger = null,
         $indexingMemory = null,
         $searchableExtensionClassName = Searchable::class
@@ -412,35 +412,50 @@ public function define($recreate = false)
     /**
      * Re-indexes each record in the index.
      *
+     * @param  bool $withBatch
      * @throws Exception
      */
-    public function refresh()
+    public function refresh($withBatch = false)
     {
         Versioned::withVersionedMode(
-            function () {
+            function () use ($withBatch) {
                 Versioned::set_stage(Versioned::LIVE);
 
                 foreach ($this->getIndexedClasses() as $class) {
                     $list = DataObject::get($class);
 
                     $this->printMessage($list->count(), sprintf('FOUND %s records of type %s', $list->count(), $class));
 
-                    $records = $list->chunkedFetch();
-                    foreach ($records as $record) {
-                        // Only index records with Show In Search enabled, or those that don't expose that field
-                        if (!$record->hasField('ShowInSearch') || $record->ShowInSearch) {
-                            if ($this->index($record)) {
-                                $this->printActionMessage($record, 'INDEXED');
-                            } else {
-                                $this->printActionMessage($record, 'ERROR INDEXING');
+                    $total = $list->count();
+                    $offset = 0;
+                    $limit = 1000;
+
+                    while ($offset < $total) {
+                        $indexFunction = function() use ($list, $limit, $offset) {
+                            $records = $list->limit($limit, $offset);
+                            foreach ($records as $record) {
+                                // Only index records with Show In Search enabled, or those that don't expose that field
+                                if (!$record->hasField('ShowInSearch') || $record->ShowInSearch) {
+                                    if ($this->index($record)) {
+                                        $this->printActionMessage($record, 'INDEXED');
+                                    } else {
+                                        $this->printActionMessage($record, 'ERROR INDEXING');
+                                    }
+                                } else {
+                                    if ($this->remove($record)) {
+                                        $this->printActionMessage($record, 'REMOVED');
+                                    } else {
+                                        $this->printActionMessage($record, 'ERROR REMOVING');
+                                    }
+                                }
                             }
+                        };
+                        if ($withBatch) {
+                            $this->batch($indexFunction);
                         } else {
-                            if ($this->remove($record)) {
-                                $this->printActionMessage($record, 'REMOVED');
-                            } else {
-                                $this->printActionMessage($record, 'ERROR REMOVING');
-                            }
+                            $indexFunction();
                         }
+                        $offset += $limit;
                     }
                 }
             }

diff --git a/src/ReindexTask.php b/src/ReindexTask.php
@@ -50,11 +50,17 @@ public function run($request)
             print(Director::is_cli() ? "$content\n" : "<p>$content</p>");
         };
 
+        $withBatch = (bool) $request->getVar('batch');
+        if ($withBatch) {
+            $message('batch processing is enabled to speed up the reindexing process');
+            //$this->service->enableBatch();
+        }
+
         $message('Defining the mappings');
         $recreate = (bool) $request->getVar('recreate');
         $this->service->define($recreate);
 
         $message('Refreshing the index');
-        $this->service->refresh();
+        $this->service->refresh($withBatch);
     }
 }