From bc1ff7c159bd5f1c342a337814df3aff1fb8df05 Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Sat, 16 Sep 2023 17:35:24 +0200 Subject: [PATCH 1/9] fix: fixed TotalItems in case of null Results --- src/ResultList.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ResultList.php b/src/ResultList.php index 16bb0f0..aaf1e38 100644 --- a/src/ResultList.php +++ b/src/ResultList.php @@ -387,7 +387,7 @@ public function count(): int */ public function getTotalItems() { - return $this->getResults()->getTotalHits(); + return ($results = $this->getResults()) ? $results->getTotalHits() : 0; } /** From fa473e4e0464bd0d2d185a76d8d6dbceb18ef3a5 Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Sun, 17 Sep 2023 00:11:23 +0200 Subject: [PATCH 2/9] feat: added support for track total hit count --- src/ElasticaService.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ElasticaService.php b/src/ElasticaService.php index 99068bc..1717a7f 100644 --- a/src/ElasticaService.php +++ b/src/ElasticaService.php @@ -130,13 +130,25 @@ protected function getIndexConfig() * @param Query|string|array $query * @param array $options Options defined in \Elastica\Search * @param bool $returnResultList + * @param bool|int $trackTotalHits * @return ResultList | ResultSet */ - public function search($query, $options = null, $returnResultList = true) + public function search($query, $options = null, $returnResultList = true, $trackTotalHits = false) { if ($returnResultList) { - return new ResultList($this->getIndex(), Query::create($query), $this->logger); + $query = Query::create($query); + + if ($trackTotalHits) { + $query = $query->setTrackTotalHits(true); + } + + return new ResultList($this->getIndex(), $query, $this->logger); } + + if ($trackTotalHits) { + $query = Query::create($query)->setTrackTotalHits(true); + } + return $this->getIndex()->search($query, $options); } From 1a34f6aaf07b10a5af320e71e4fbbcc34933af3f Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Sun, 17 Sep 2023 00:13:01 +0200 Subject: [PATCH 3/9] perf: optimized reindexing of records (via chunks) --- src/ElasticaService.php | 8 ++++---- src/ReindexTask.php | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/ElasticaService.php b/src/ElasticaService.php index 1717a7f..2dbd734 100644 --- a/src/ElasticaService.php +++ b/src/ElasticaService.php @@ -423,17 +423,17 @@ public function define($recreate = false) /** * Re-indexes each record in the index. - * + * @param int $chunkSize * @throws Exception */ - public function refresh() + public function refresh($chunkSize = 1000) { Versioned::withVersionedMode( - function () { + function () use ($chunkSize) { Versioned::set_stage(Versioned::LIVE); foreach ($this->getIndexedClasses() as $class) { - foreach (DataObject::get($class) as $record) { + foreach (DataObject::get($class)->chunkedFetch($chunkSize) as $record) { // Only index records with Show In Search enabled, or those that don't expose that fielid if (!$record->hasField('ShowInSearch') || $record->ShowInSearch) { if ($this->index($record)) { diff --git a/src/ReindexTask.php b/src/ReindexTask.php index 01aed2c..869cba8 100644 --- a/src/ReindexTask.php +++ b/src/ReindexTask.php @@ -53,5 +53,11 @@ public function run($request) $message('Refreshing the index'); $this->service->refresh(); + + if (($chunkSize = (int) $request->getVar('chunkSize')) <= 0) { + $chunkSize = 1000; + } + + $this->service->refresh($chunkSize); } } From 3c2f1865e3cfa1485884314d3a4a0e0da65ff4ef Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Sun, 17 Sep 2023 00:28:49 +0200 Subject: [PATCH 4/9] feat: added support for reindexing only one class --- src/ElasticaService.php | 21 ++++++++++++--------- src/ReindexTask.php | 26 +++++++++++++++++++++----- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/ElasticaService.php b/src/ElasticaService.php index 2dbd734..dcc8d10 100644 --- a/src/ElasticaService.php +++ b/src/ElasticaService.php @@ -395,10 +395,11 @@ public function getVersion(): string /** * Creates the index and the type mappings. * - * @param bool $recreate + * @param bool $recreate + * @param string $class * @throws Exception */ - public function define($recreate = false) + public function define($recreate = false, $class = null) { $index = $this->getIndex(); $exists = $index->exists(); @@ -413,7 +414,7 @@ public function define($recreate = false) $this->createIndex(); } - foreach ($this->getIndexedClasses() as $class) { + foreach ($this->getIndexedClasses($class) as $class) { /** @var Searchable */ $sng = singleton($class); $props = $sng->getElasticaMapping(); @@ -423,16 +424,17 @@ public function define($recreate = false) /** * Re-indexes each record in the index. - * @param int $chunkSize + * @param int $chunkSize + * @param string $class * @throws Exception */ - public function refresh($chunkSize = 1000) + public function refresh($chunkSize = 1000, $class = null) { Versioned::withVersionedMode( - function () use ($chunkSize) { + function () use ($chunkSize, $class) { Versioned::set_stage(Versioned::LIVE); - foreach ($this->getIndexedClasses() as $class) { + foreach ($this->getIndexedClasses($class) as $class) { foreach (DataObject::get($class)->chunkedFetch($chunkSize) as $record) { // Only index records with Show In Search enabled, or those that don't expose that fielid if (!$record->hasField('ShowInSearch') || $record->ShowInSearch) { @@ -453,13 +455,14 @@ function () use ($chunkSize) { /** * Gets the classes which are indexed (i.e. have the extension applied). * + * @param string $class * @return array * @throws ReflectionException */ - public function getIndexedClasses() + public function getIndexedClasses($class = null) { $classes = array(); - foreach (ClassInfo::subclassesFor(DataObject::class) as $candidate) { + foreach ($class ? [$class] : ClassInfo::subclassesFor(DataObject::class) as $candidate) { $candidateInstance = DataObject::singleton($candidate); if ($candidateInstance->hasExtension($this->searchableExtensionClassName)) { $classes[] = $candidate; diff --git a/src/ReindexTask.php b/src/ReindexTask.php index 869cba8..66fac1f 100644 --- a/src/ReindexTask.php +++ b/src/ReindexTask.php @@ -47,17 +47,33 @@ public function run($request) print(Director::is_cli() ? "$content\n" : "

$content

"); }; - $message('Defining the mappings'); + $class = $request->getVar('class'); + + if ($class && !class_exists($class)) { + $message("Class {$class} does not exist"); + + return; + } + + if ($class) { + $message('Defining the mappings for class ' . $class); + } else { + $message('Defining the mappings'); + } + $recreate = (bool) $request->getVar('recreate'); - $this->service->define($recreate); + $this->service->define($recreate, $class); - $message('Refreshing the index'); - $this->service->refresh(); + if ($class) { + $message('Refreshing the index for class ' . $class); + } else { + $message('Refreshing the index'); + } if (($chunkSize = (int) $request->getVar('chunkSize')) <= 0) { $chunkSize = 1000; } - $this->service->refresh($chunkSize); + $this->service->refresh($chunkSize, $class); } } From 13c846ae3f8009c79de063eb7ba953ee39401b32 Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Sun, 17 Sep 2023 01:21:41 +0200 Subject: [PATCH 5/9] feat: added support for custom datalist to reindexing --- src/ElasticaService.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ElasticaService.php b/src/ElasticaService.php index dcc8d10..81d04a2 100644 --- a/src/ElasticaService.php +++ b/src/ElasticaService.php @@ -414,9 +414,8 @@ public function define($recreate = false, $class = null) $this->createIndex(); } - foreach ($this->getIndexedClasses($class) as $class) { + foreach ($this->getIndexedClasses($class) as $sng) { /** @var Searchable */ - $sng = singleton($class); $props = $sng->getElasticaMapping(); $props->send($index); } @@ -434,8 +433,14 @@ public function refresh($chunkSize = 1000, $class = null) function () use ($chunkSize, $class) { Versioned::set_stage(Versioned::LIVE); - foreach ($this->getIndexedClasses($class) as $class) { - foreach (DataObject::get($class)->chunkedFetch($chunkSize) as $record) { + foreach ($this->getIndexedClasses($class) as $sng) { + if ($sng->hasMethod('getDataListToIndex')) { + $list = $sng->getDataListToIndex(); + } else { + $list = $sng::get(); + } + + foreach ($list->chunkedFetch($chunkSize) as $record) { // Only index records with Show In Search enabled, or those that don't expose that fielid if (!$record->hasField('ShowInSearch') || $record->ShowInSearch) { if ($this->index($record)) { @@ -465,7 +470,7 @@ public function getIndexedClasses($class = null) foreach ($class ? [$class] : ClassInfo::subclassesFor(DataObject::class) as $candidate) { $candidateInstance = DataObject::singleton($candidate); if ($candidateInstance->hasExtension($this->searchableExtensionClassName)) { - $classes[] = $candidate; + $classes[] = $candidateInstance; } } return $classes; From ef1eb8072255de9266a3524e45170d9412213e1f Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Sun, 17 Sep 2023 11:54:22 +0200 Subject: [PATCH 6/9] perf!: removed check for potential nested relation class (without "relationClass" key) --- src/Searchable.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Searchable.php b/src/Searchable.php index b8bf262..2c2619c 100644 --- a/src/Searchable.php +++ b/src/Searchable.php @@ -230,15 +230,17 @@ public function getElasticaFields() $field = isset($params['field']) ? $params['field'] : $fieldName; - $relationClass = isset($params['relationClass']) - ? $params['relationClass'] - : $this->owner->getRelationClass($fieldName); // Don't send these to elasticsearch - unset($params['relationClass'], $params['field']); + unset($params['field']); // Build nested field from relation - if ($relationClass) { + if (isset($params['relationClass'])) { + $relationClass = $params['relationClass']; + + // Don't send these to elasticsearch + unset($params['relationClass']); + // Relations can add multiple fields, so merge them all here $nestedFields = $this->getSearchableFieldsForRelation($fieldName, $params, $relationClass); $result = array_merge($result, $nestedFields); @@ -358,16 +360,14 @@ public function getSearchableFieldValues() { $fieldValues = []; foreach ($this->owner->indexedFields() as $fieldName => $params) { - // Check nested relation class - $relationClass = isset($params['relationClass']) - ? $params['relationClass'] - : $this->owner->getRelationClass($fieldName); $field = isset($params['field']) ? $params['field'] : $fieldName; // Build nested field from relation - if ($relationClass) { + if (isset($params['relationClass'])) { + $relationClass = $params['relationClass']; + // Relations can add multiple fields, so merge them all here $nestedFieldValues = $this->getSearchableFieldValuesForRelation($fieldName, $params, $relationClass); $fieldValues = array_merge($fieldValues, $nestedFieldValues); From 055b0599921c6e920382eef8fe90873e3cfab093 Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Mon, 18 Sep 2023 00:54:42 +0200 Subject: [PATCH 7/9] feat: added support for selecting only certain fields from relation class --- src/Searchable.php | 72 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/src/Searchable.php b/src/Searchable.php index 2c2619c..542a434 100644 --- a/src/Searchable.php +++ b/src/Searchable.php @@ -221,12 +221,20 @@ public function inheritedDatabaseFields() * if needed. First we go through all the regular fields belonging to pages, then to the dataobjects related to * those pages * + * @param array $onlyFields * @return array */ - public function getElasticaFields() + public function getElasticaFields($onlyFields = []) { + $indexedFields = $this->owner->indexedFields(); + + if ($onlyFields) { + $indexedFields = array_intersect_key($indexedFields, $onlyFields); + $indexedFields = array_replace_recursive($indexedFields, $onlyFields); + } + $result = []; - foreach ($this->owner->indexedFields() as $fieldName => $params) { + foreach ($indexedFields as $fieldName => $params) { $field = isset($params['field']) ? $params['field'] : $fieldName; @@ -247,6 +255,9 @@ public function getElasticaFields() continue; } + // Don't send these to elasticsearch + unset($params['only_fields']); + // Get extra params $params = $this->getExtraFieldParams($field, $params); @@ -354,12 +365,20 @@ public function getElasticaDocument() * Get values for all searchable fields as an array. * Similr to getSearchableFields() but returns field values instead of spec * + * @param array $onlyFields * @return array */ - public function getSearchableFieldValues() + public function getSearchableFieldValues($onlyFields = []) { + $indexedFields = $this->owner->indexedFields(); + + if ($onlyFields) { + $indexedFields = array_intersect_key($indexedFields, $onlyFields); + $indexedFields = array_replace_recursive($indexedFields, $onlyFields); + } + $fieldValues = []; - foreach ($this->owner->indexedFields() as $fieldName => $params) { + foreach ($indexedFields as $fieldName => $params) { $field = isset($params['field']) ? $params['field'] : $fieldName; @@ -376,6 +395,8 @@ public function getSearchableFieldValues() // Get value from object if ($this->owner->hasField($field)) { + unset($params['only_fields']); + // Check field exists on parent $params = $this->getExtraFieldParams($field, $params); $fieldValue = $this->formatValue($params, $this->owner->relField($field)); @@ -559,7 +580,14 @@ protected function getSearchableFieldsForRelation($fieldName, $params, $classNam } // Get nested fields - $nestedFields = $related->getElasticaFields(); + + if (isset($params['only_fields'])) { + $nestedFields = $related->getElasticaFields($params['only_fields']); + + unset($params['only_fields']); + } else { + $nestedFields = $related->getElasticaFields(); + } // Determine if merging into parent as either a multilevel object (default) // or nested objects (requires 'nested' param to be set) @@ -643,8 +671,15 @@ protected function getSearchableFieldValuesForRelation($fieldName, $params, $cla * @var DataObject|Searchable $relationListItem */ foreach ($relatedList as $relationListItem) { - $relationValues[] = $relationListItem->getSearchableFieldValues(); + if (isset($params['only_fields'])) { + $searchableFieldValues = $relationListItem->getSearchableFieldValues($params['only_fields']); + } else { + $searchableFieldValues = $relationListItem->getSearchableFieldValues(); + } + + $relationValues[] = $searchableFieldValues; } + return [$fieldName => $relationValues]; } @@ -653,9 +688,15 @@ protected function getSearchableFieldValuesForRelation($fieldName, $params, $cla // Handle unary-multilevel // I.e. Relation_Field = 'value' if ($isUnary) { + if (isset($params['only_fields'])) { + $searchableFieldValues = $relatedItem->getSearchableFieldValues($params['only_fields']); + } else { + $searchableFieldValues = $relatedItem->getSearchableFieldValues(); + } + // We will return multiple values, one for each sub-column $fieldValues = []; - foreach ($relatedItem->getSearchableFieldValues() as $relatedFieldName => $relatedFieldValue) { + foreach ($searchableFieldValues as $relatedFieldName => $relatedFieldValue) { $nestedName = "{$fieldName}_{$relatedFieldName}"; $fieldValues[$nestedName] = $relatedItem->IsInDB() ? $relatedFieldValue : null; } @@ -666,16 +707,29 @@ protected function getSearchableFieldValuesForRelation($fieldName, $params, $cla // I.e. Relation_Field = ['value1', 'value2'] $fieldValues = []; + + if (isset($params['only_fields'])) { + $elasticaFields = $relatedSingleton->getElasticaFields($params['only_fields']); + } else { + $elasticaFields = $relatedSingleton->getElasticaFields(); + } + // Bootstrap set with empty arrays for each top level key // This also ensures we set empty data if $relatedList is empty - foreach ($relatedSingleton->getElasticaFields() as $relatedFieldName => $spec) { + foreach ($elasticaFields as $relatedFieldName => $spec) { $nestedName = "{$fieldName}_{$relatedFieldName}"; $fieldValues[$nestedName] = []; } // Add all documents to the list foreach ($relatedList as $relatedListItem) { - foreach ($relatedListItem->getSearchableFieldValues() as $relatedFieldName => $relatedFieldValue) { + if (isset($params['only_fields'])) { + $searchableFieldValues = $relatedListItem->getSearchableFieldValues($params['only_fields']); + } else { + $searchableFieldValues = $relatedListItem->getSearchableFieldValues(); + } + + foreach ($searchableFieldValues as $relatedFieldName => $relatedFieldValue) { $nestedName = "{$fieldName}_{$relatedFieldName}"; $fieldValues[$nestedName][] = $relatedFieldValue; } From 79ff3c091a071efbd279751b7d0d7bdfa9aa5713 Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Mon, 18 Sep 2023 09:39:00 +0200 Subject: [PATCH 8/9] fix: fixed reading mode check --- src/ResultList.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ResultList.php b/src/ResultList.php index aaf1e38..9338c2e 100644 --- a/src/ResultList.php +++ b/src/ResultList.php @@ -60,7 +60,7 @@ public function __construct(Index $index, Query $query, LoggerInterface $logger ] ); - if (Versioned::get_reading_mode() == Versioned::LIVE) { + if (Versioned::get_reading_mode() === 'Stage.' . Versioned::LIVE) { $publishedFilter = $query->hasParam('post_filter') ? $query->getParam('post_filter') : null; if (!$publishedFilter) { From 5781d68f0e5def651937181dddbaeeecf2298409 Mon Sep 17 00:00:00 2001 From: Rastislav Brandobur Date: Mon, 18 Sep 2023 09:41:42 +0200 Subject: [PATCH 9/9] feat: added support for custom datalist to results --- src/ResultList.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ResultList.php b/src/ResultList.php index 9338c2e..102fd8a 100644 --- a/src/ResultList.php +++ b/src/ResultList.php @@ -221,7 +221,15 @@ public function toArray() return end($parts); }, $documentIds); - foreach (DataObject::get($class)->byIDs($ids) as $record) { + $sng = singleton($class); + + if ($sng->hasMethod('getDataListToIndex')) { + $list = $sng->getDataListToIndex(); + } else { + $list = $sng::get(); + } + + foreach ($list->byIDs($ids) as $record) { $retrieved[$class][$record->ID] = $record; } }