From cc809c738010f8e3aabc05396e6b3c46d3a8a391 Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Fri, 16 Aug 2024 15:42:38 +0200 Subject: [PATCH 1/8] Support serialization for DynamicTemplates. Signed-off-by: Youssef Aouichaoui --- .../data/elasticsearch/annotations/Field.java | 7 +++++ .../MappingElasticsearchConverter.java | 15 +++++++-- .../ElasticsearchPersistentProperty.java | 7 +++++ ...SimpleElasticsearchPersistentProperty.java | 7 +++++ .../index/MappingBuilderIntegrationTests.java | 31 +++++++++++++++++++ ...test-dynamic_templates_mappings_three.json | 12 +++++++ 6 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/mappings/test-dynamic_templates_mappings_three.json diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java b/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java index 0a9d4f4635..f1ba459a68 100644 --- a/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java @@ -252,4 +252,11 @@ * @since 5.4 */ String mappedTypeName() default ""; + + /** + * Maps your data beyond the dynamic field mapping rules. + * + * @since 5.4 + */ + boolean dynamicTemplate() default false; } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java index ccac971fd1..ff231bd08f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java @@ -1035,7 +1035,14 @@ protected void writeProperty(ElasticsearchPersistentProperty property, Object va if (valueType.isMap()) { Map mapDbObj = createMap((Map) value, property); - sink.set(property, mapDbObj); + if (property.isDynamicFieldMapping()) { + for (Entry entry : mapDbObj.entrySet()) { + sink.set(entry.getKey(), entry.getValue()); + } + } else { + sink.set(property, mapDbObj); + } + return; } @@ -1499,7 +1506,11 @@ public void set(ElasticsearchPersistentProperty property, @Nullable Object value } } - target.put(property.getFieldName(), value); + set(property.getFieldName(), value); + } + + public void set(String key, @Nullable Object value) { + target.put(key, value); } private Map getAsMap(Object result) { diff --git a/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentProperty.java b/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentProperty.java index 2b9d6232e0..7575b88648 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentProperty.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentProperty.java @@ -110,6 +110,13 @@ public interface ElasticsearchPersistentProperty extends PersistentProperty owner, SimpleTypeHolder simpleTypeHolder) { @@ -114,6 +115,7 @@ public SimpleElasticsearchPersistentProperty(Property property, : isMultiField && getRequiredAnnotation(MultiField.class).mainField().storeNullValue(); storeEmptyValue = isField ? getRequiredAnnotation(Field.class).storeEmptyValue() : !isMultiField || getRequiredAnnotation(MultiField.class).mainField().storeEmptyValue(); + isDynamicFieldMapping = isField && getRequiredAnnotation(Field.class).dynamicTemplate(); } @Override @@ -393,4 +395,9 @@ public boolean isCompletionProperty() { public boolean isIndexedIndexNameProperty() { return isAnnotationPresent(IndexedIndexName.class); } + + @Override + public boolean isDynamicFieldMapping() { + return isDynamicFieldMapping; + } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java index b4aaab0e95..eba9a8ede0 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java @@ -16,8 +16,10 @@ package org.springframework.data.elasticsearch.core.index; +import static java.util.UUID.randomUUID; import static org.assertj.core.api.Assertions.*; import static org.springframework.data.elasticsearch.annotations.FieldType.*; +import static org.springframework.data.elasticsearch.core.query.StringQuery.MATCH_ALL; import java.time.Instant; import java.time.LocalDate; @@ -40,7 +42,9 @@ import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.IndexOperations; import org.springframework.data.elasticsearch.core.MappingContextBaseTests; +import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.StringQuery; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.lang.Nullable; @@ -280,6 +284,24 @@ void shouldWriteMappingWithFieldAliases() { operations.indexOps(FieldAliasEntity.class).createWithMapping(); } + @Test + void shouldMapDynamicFields() { + // Given + IndexOperations documentOperations = operations.indexOps(DynamicFieldDocument.class); + documentOperations.createWithMapping(); + + DynamicFieldDocument document = new DynamicFieldDocument(); + document.dynamicFields = Map.of("a_str", randomUUID().toString(), "b_str", randomUUID().toString()); + operations.save(document); + + // When + SearchHits results = operations.search(new StringQuery(MATCH_ALL), DynamicFieldDocument.class); + + // Then + assertThat(results.getTotalHits()).isEqualTo(1); + documentOperations.delete(); + } + // region Entities @Document(indexName = "#{@indexNameProvider.indexName()}") static class Book { @@ -933,5 +955,14 @@ private static class FieldAliasEntity { @Field(type = Text) private String otherText; } + @SuppressWarnings("unused") + @Document(indexName = "foo") + @DynamicTemplates(mappingPath = "/mappings/test-dynamic_templates_mappings_three.json") + private static class DynamicFieldDocument { + @Nullable + @Id String id; + + @Field(name = "*_str", dynamicTemplate = true) private Map dynamicFields = new HashMap<>(); + } // endregion } diff --git a/src/test/resources/mappings/test-dynamic_templates_mappings_three.json b/src/test/resources/mappings/test-dynamic_templates_mappings_three.json new file mode 100644 index 0000000000..4ead5aa64b --- /dev/null +++ b/src/test/resources/mappings/test-dynamic_templates_mappings_three.json @@ -0,0 +1,12 @@ +{ + "dynamic_templates": [ + { + "_str": { + "match": "*_str", + "mapping": { + "type": "keyword" + } + } + } + ] +} From 4805a4637172e716852f1be8184e591681ebd52e Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Tue, 20 Aug 2024 16:54:28 +0200 Subject: [PATCH 2/8] Support deserialization for DynamicTemplates. Signed-off-by: Youssef Aouichaoui --- .../MappingElasticsearchConverter.java | 40 +++++++++++++++++++ .../index/MappingBuilderIntegrationTests.java | 5 ++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java index ff231bd08f..841b364076 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java @@ -15,6 +15,9 @@ */ package org.springframework.data.elasticsearch.core.convert; +import static org.springframework.util.PatternMatchUtils.simpleMatch; +import static org.springframework.util.StringUtils.hasText; + import java.time.temporal.TemporalAccessor; import java.util.*; import java.util.Map.Entry; @@ -38,8 +41,10 @@ import org.springframework.core.env.EnvironmentCapable; import org.springframework.core.env.StandardEnvironment; import org.springframework.data.convert.CustomConversions; +import org.springframework.data.elasticsearch.annotations.DynamicTemplates; import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.ScriptedField; +import org.springframework.data.elasticsearch.core.ResourceUtil; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.document.SearchDocument; import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; @@ -53,6 +58,7 @@ import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm; import org.springframework.data.elasticsearch.core.query.SourceFilter; +import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; import org.springframework.data.mapping.InstanceCreatorMetadata; import org.springframework.data.mapping.MappingException; import org.springframework.data.mapping.Parameter; @@ -388,6 +394,40 @@ private R readEntity(ElasticsearchPersistentEntity entity, Map().fromJson(jsonString).get("dynamic_templates"); + if (templates instanceof List array) { + for (Object node : array) { + if (node instanceof Map entry) { + Entry templateEntry = entry.entrySet().stream().findFirst().orElse(null); + if (templateEntry != null) { + ElasticsearchPersistentProperty property = targetEntity + .getPersistentPropertyWithFieldName((String) templateEntry.getKey()); + if (property != null && property.isDynamicFieldMapping()) { + targetEntity.getPropertyAccessor(result).getProperty(property); + targetEntity.getPropertyAccessor(result).setProperty(property, + document.entrySet().stream().filter(fieldKey -> { + if (templateEntry.getValue() instanceof Map templateValue) { + if (templateValue.containsKey("match")) { + return simpleMatch((String) templateValue.get("match"), fieldKey.getKey()); + } + } + + return false; + }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + } + } + } + } + } + } + } } if (source instanceof SearchDocument searchDocument) { diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java index eba9a8ede0..9d68b7905f 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java @@ -42,6 +42,7 @@ import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.IndexOperations; import org.springframework.data.elasticsearch.core.MappingContextBaseTests; +import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.StringQuery; @@ -299,6 +300,8 @@ void shouldMapDynamicFields() { // Then assertThat(results.getTotalHits()).isEqualTo(1); + assertThat(results.getSearchHits()).first().extracting(SearchHit::getContent).extracting(doc -> doc.dynamicFields) + .isEqualTo(document.dynamicFields); documentOperations.delete(); } @@ -962,7 +965,7 @@ private static class DynamicFieldDocument { @Nullable @Id String id; - @Field(name = "*_str", dynamicTemplate = true) private Map dynamicFields = new HashMap<>(); + @Field(name = "_str", dynamicTemplate = true) private Map dynamicFields = new HashMap<>(); } // endregion } From cdf3fd23c9efc562f793387eba87642f2d271a9f Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Mon, 26 Aug 2024 16:54:57 +0200 Subject: [PATCH 3/8] Extract method to populate fields [dynamic]. Signed-off-by: Youssef Aouichaoui --- .../MappingElasticsearchConverter.java | 66 ++++++++++--------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java index 841b364076..7fb91d7153 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java @@ -396,37 +396,7 @@ private R readEntity(ElasticsearchPersistentEntity entity, Map().fromJson(jsonString).get("dynamic_templates"); - if (templates instanceof List array) { - for (Object node : array) { - if (node instanceof Map entry) { - Entry templateEntry = entry.entrySet().stream().findFirst().orElse(null); - if (templateEntry != null) { - ElasticsearchPersistentProperty property = targetEntity - .getPersistentPropertyWithFieldName((String) templateEntry.getKey()); - if (property != null && property.isDynamicFieldMapping()) { - targetEntity.getPropertyAccessor(result).getProperty(property); - targetEntity.getPropertyAccessor(result).setProperty(property, - document.entrySet().stream().filter(fieldKey -> { - if (templateEntry.getValue() instanceof Map templateValue) { - if (templateValue.containsKey("match")) { - return simpleMatch((String) templateValue.get("match"), fieldKey.getKey()); - } - } - - return false; - }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - } - } - } - } - } - } - } + populateFieldsUsingDynamicTemplates(targetEntity, result, document); } } @@ -704,6 +674,40 @@ private void populateScriptFields(ElasticsearchPersistentEntity entity, T }); } + private void populateFieldsUsingDynamicTemplates(ElasticsearchPersistentEntity targetEntity, R result, Document document) { + String mappingPath = targetEntity.getRequiredAnnotation(DynamicTemplates.class).mappingPath(); + if (hasText(mappingPath)) { + String jsonString = ResourceUtil.readFileFromClasspath(mappingPath); + if (hasText(jsonString)) { + Object templates = new DefaultStringObjectMap<>().fromJson(jsonString).get("dynamic_templates"); + if (templates instanceof List array) { + for (Object node : array) { + if (node instanceof Map entry) { + Entry templateEntry = entry.entrySet().stream().findFirst().orElse(null); + if (templateEntry != null) { + ElasticsearchPersistentProperty property = targetEntity + .getPersistentPropertyWithFieldName((String) templateEntry.getKey()); + if (property != null && property.isDynamicFieldMapping()) { + targetEntity.getPropertyAccessor(result).getProperty(property); + targetEntity.getPropertyAccessor(result).setProperty(property, + document.entrySet().stream().filter(fieldKey -> { + if (templateEntry.getValue() instanceof Map templateValue) { + if (templateValue.containsKey("match")) { + return simpleMatch((String) templateValue.get("match"), fieldKey.getKey()); + } + } + + return false; + }).collect(Collectors.toMap(Entry::getKey, Entry::getValue))); + } + } + } + } + } + } + } + } + /** * Compute the type to use by checking the given entity against the store type; */ From 9226ab7684d67a7053d59041cec47432f6dafb48 Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Tue, 27 Aug 2024 16:00:53 +0200 Subject: [PATCH 4/8] Get templates from cluster. Signed-off-by: Youssef Aouichaoui --- .../core/AbstractElasticsearchTemplate.java | 10 + .../MappingElasticsearchConverter.java | 50 ++--- .../core/mapping/DynamicTemplate.java | 190 ++++++++++++++++++ .../ElasticsearchPersistentEntity.java | 23 +++ .../SimpleElasticsearchPersistentEntity.java | 48 +++++ 5 files changed, 286 insertions(+), 35 deletions(-) create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/mapping/DynamicTemplate.java diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java index 4275e17f8b..7c789a5afe 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -602,6 +603,15 @@ protected SearchDocumentResponse.EntityCreator getEntityCreator(ReadDocum // region Entity callbacks protected T maybeCallbackBeforeConvert(T entity, IndexCoordinates index) { + // get entity metadata + Map mapping = indexOps(index).getMapping(); + if (mapping.containsKey("dynamic_templates")) { + Object dynamicTemplates = mapping.get("dynamic_templates"); + if (dynamicTemplates instanceof List value) { + getRequiredPersistentEntity(entity.getClass()).buildDynamicTemplates(value); + } + } + if (entityCallbacks != null) { return entityCallbacks.callback(BeforeConvertCallback.class, entity, index); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java index 7fb91d7153..64ea4afcb5 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java @@ -16,7 +16,6 @@ package org.springframework.data.elasticsearch.core.convert; import static org.springframework.util.PatternMatchUtils.simpleMatch; -import static org.springframework.util.StringUtils.hasText; import java.time.temporal.TemporalAccessor; import java.util.*; @@ -41,12 +40,11 @@ import org.springframework.core.env.EnvironmentCapable; import org.springframework.core.env.StandardEnvironment; import org.springframework.data.convert.CustomConversions; -import org.springframework.data.elasticsearch.annotations.DynamicTemplates; import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.ScriptedField; -import org.springframework.data.elasticsearch.core.ResourceUtil; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.document.SearchDocument; +import org.springframework.data.elasticsearch.core.mapping.DynamicTemplate; import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty; import org.springframework.data.elasticsearch.core.mapping.PropertyValueConverter; @@ -58,7 +56,6 @@ import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm; import org.springframework.data.elasticsearch.core.query.SourceFilter; -import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; import org.springframework.data.mapping.InstanceCreatorMetadata; import org.springframework.data.mapping.MappingException; import org.springframework.data.mapping.Parameter; @@ -395,7 +392,7 @@ private R readEntity(ElasticsearchPersistentEntity entity, Map void populateScriptFields(ElasticsearchPersistentEntity entity, T }); } - private void populateFieldsUsingDynamicTemplates(ElasticsearchPersistentEntity targetEntity, R result, Document document) { - String mappingPath = targetEntity.getRequiredAnnotation(DynamicTemplates.class).mappingPath(); - if (hasText(mappingPath)) { - String jsonString = ResourceUtil.readFileFromClasspath(mappingPath); - if (hasText(jsonString)) { - Object templates = new DefaultStringObjectMap<>().fromJson(jsonString).get("dynamic_templates"); - if (templates instanceof List array) { - for (Object node : array) { - if (node instanceof Map entry) { - Entry templateEntry = entry.entrySet().stream().findFirst().orElse(null); - if (templateEntry != null) { - ElasticsearchPersistentProperty property = targetEntity - .getPersistentPropertyWithFieldName((String) templateEntry.getKey()); - if (property != null && property.isDynamicFieldMapping()) { - targetEntity.getPropertyAccessor(result).getProperty(property); - targetEntity.getPropertyAccessor(result).setProperty(property, - document.entrySet().stream().filter(fieldKey -> { - if (templateEntry.getValue() instanceof Map templateValue) { - if (templateValue.containsKey("match")) { - return simpleMatch((String) templateValue.get("match"), fieldKey.getKey()); - } - } - - return false; - }).collect(Collectors.toMap(Entry::getKey, Entry::getValue))); - } - } - } - } - } + private void populateFieldsUsingDynamicTemplates(ElasticsearchPersistentEntity targetEntity, R result, + Document document) { + for (Entry templateEntry : targetEntity.getDynamicTemplates().entrySet()) { + ElasticsearchPersistentProperty property = targetEntity + .getPersistentPropertyWithFieldName(templateEntry.getKey()); + if (property != null && property.isDynamicFieldMapping()) { + targetEntity.getPropertyAccessor(result).setProperty(property, + document.entrySet().stream() + .filter(fieldKey -> templateEntry.getValue().getMatch().stream() + .anyMatch(regex -> simpleMatch(regex, fieldKey.getKey())) + && templateEntry.getValue().getUnmatch().stream() + .noneMatch(regex -> simpleMatch(regex, fieldKey.getKey()))) + .collect(Collectors.toMap(Entry::getKey, Entry::getValue))); } } } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/mapping/DynamicTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/mapping/DynamicTemplate.java new file mode 100644 index 0000000000..0ee59fc72c --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/core/mapping/DynamicTemplate.java @@ -0,0 +1,190 @@ +package org.springframework.data.elasticsearch.core.mapping; + +import java.util.ArrayList; +import java.util.List; + +/** + * Immutable Value object encapsulating dynamic template(s). + * {@see Elastic + * docs} + * + * @author Youssef Aouichaoui + * @since 5.4 + */ +public class DynamicTemplate { + /** + * Patterns to match on the field name. + */ + private final List match; + + /** + * Path patterns for a nested type to match the field name. + */ + private final List pathMatch; + + /** + * Patterns that do not match the field name. + */ + private final List unmatch; + + /** + * Path patterns for a nested type that do not match the field name. + */ + private final List pathUnmatch; + + /** + * Data types that correspond to the field. + */ + private final List matchMappingType; + + /** + * Data types that do not match to the field. + */ + private final List unmatchMappingType; + + private DynamicTemplate(Builder builder) { + this.match = builder.match; + this.pathMatch = builder.pathMatch; + + this.unmatch = builder.unmatch; + this.pathUnmatch = builder.pathUnmatch; + + this.matchMappingType = builder.matchMappingType; + this.unmatchMappingType = builder.unmatchMappingType; + } + + public List getMatch() { + return match; + } + + public List getPathMatch() { + return pathMatch; + } + + public List getUnmatch() { + return unmatch; + } + + public List getPathUnmatch() { + return pathUnmatch; + } + + public List getMatchMappingType() { + return matchMappingType; + } + + public List getUnmatchMappingType() { + return unmatchMappingType; + } + + public boolean isRegexMatching() { + return false; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final List match = new ArrayList<>(); + private final List pathMatch = new ArrayList<>(); + + private final List unmatch = new ArrayList<>(); + private final List pathUnmatch = new ArrayList<>(); + + private final List matchMappingType = new ArrayList<>(); + private final List unmatchMappingType = new ArrayList<>(); + + private Builder() {} + + /** + * Patterns to match on the field name. + */ + public Builder withMatch(String... match) { + for (String value : match) { + if (value != null) { + parseValues(value, this.match); + } + } + + return this; + } + + /** + * Path patterns for a nested type to match the field name. + */ + public Builder withPathMatch(String... pathMatch) { + for (String value : pathMatch) { + if (value != null) { + parseValues(value, this.pathMatch); + } + } + + return this; + } + + /** + * Patterns that do not match the field name. + */ + public Builder withUnmatch(String... unmatch) { + for (String value : unmatch) { + if (value != null) { + parseValues(value, this.unmatch); + } + } + + return this; + } + + /** + * Path patterns for a nested type that do not match the field name. + */ + public Builder withPathUnmatch(String... pathUnmatch) { + for (String value : pathUnmatch) { + if (value != null) { + parseValues(value, this.pathUnmatch); + } + } + + return this; + } + + /** + * Data types that correspond to the field. + */ + public Builder withMatchMappingType(String... matchMappingType) { + for (String value : matchMappingType) { + if (value != null) { + parseValues(value, this.matchMappingType); + } + } + + return this; + } + + /** + * Data types that do not match to the field. + */ + public Builder withUnmatchMappingType(String... unmatchMappingType) { + for (String value : unmatchMappingType) { + if (value != null) { + parseValues(value, this.unmatchMappingType); + } + } + + return this; + } + + private void parseValues(String source, List target) { + if (source.startsWith("[")) { + target.addAll(List.of(source.replace("[", "").replace("]", "").split(",", -1))); + } else { + target.add(source); + } + } + + public DynamicTemplate build() { + return new DynamicTemplate(this); + } + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java b/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java index 252376bed1..bea0e8f39b 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java @@ -15,6 +15,8 @@ */ package org.springframework.data.elasticsearch.core.mapping; +import java.util.List; +import java.util.Map; import java.util.Set; import org.springframework.data.elasticsearch.annotations.Document; @@ -203,4 +205,25 @@ default ElasticsearchPersistentProperty getRequiredSeqNoPrimaryTermProperty() { * @since 5.2 */ boolean isAlwaysWriteMapping(); + + /** + * Retrieves the dynamic templates defined for the current document. + * + * @since 5.4 + */ + Map getDynamicTemplates(); + + /** + * if the mapping should be written to the index on repository bootstrap even if the index already exists. + * + * @since 5.4 + */ + boolean hasDynamicTemplates(); + + /** + * Building the dynamic templates for the current document. + * + * @since 5.4 + */ + void buildDynamicTemplates(List dynamicTemplates); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java b/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java index fb2a124128..380e263fba 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java @@ -15,7 +15,10 @@ */ package org.springframework.data.elasticsearch.core.mapping; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -51,6 +54,8 @@ import org.springframework.lang.Nullable; import org.springframework.util.Assert; +import static java.util.Collections.unmodifiableMap; + /** * Elasticsearch specific {@link org.springframework.data.mapping.PersistentEntity} implementation holding * @@ -85,6 +90,7 @@ public class SimpleElasticsearchPersistentEntity extends BasicPersistentEntit private @Nullable String routing; private final ContextConfiguration contextConfiguration; private final Set aliases = new HashSet<>(); + private final Map dynamicTemplates = new HashMap<>(); private final ConcurrentHashMap indexNameExpressions = new ConcurrentHashMap<>(); private final Lazy indexNameEvaluationContext = Lazy.of(this::getIndexNameEvaluationContext); @@ -627,6 +633,48 @@ public Dynamic dynamic() { return dynamic; } + @Override + public Map getDynamicTemplates() { + return unmodifiableMap(dynamicTemplates); + } + + @Override + public boolean hasDynamicTemplates() { + return !dynamicTemplates.isEmpty(); + } + + @Override + public void buildDynamicTemplates(List dynamicTemplates) { + for (Object dynamicTemplate : dynamicTemplates) { + if (dynamicTemplate instanceof Map template) { + template.forEach((name, value) -> { + if (value instanceof Map settings) { + this.dynamicTemplates.put((String) name, + DynamicTemplate.builder() + .withMatch(parseMapValue(settings.get("match"))) + .withPathMatch(parseMapValue(settings.get("path_match"))) + .withUnmatch(parseMapValue(settings.get("unmatch"))) + .withPathUnmatch(parseMapValue(settings.get("path_unmatch"))) + .withMatchMappingType(parseMapValue(settings.get("match_mapping_type"))) + .withUnmatchMappingType(parseMapValue(settings.get("unmatch_mapping_type"))) + .build()); + } + }); + } + } + } + + /** + * Parses the provided value and converts it into an array of Strings. + */ + private String[] parseMapValue(@Nullable Object value) { + if (value instanceof List values) { + return values.toArray(new String[0]); + } + + return new String[0]; + } + /** * Building once the aliases for the current document. */ From ce3a615bb2a11fcfd28978738dc310f567b8dd09 Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Thu, 29 Aug 2024 12:18:20 +0200 Subject: [PATCH 5/8] Support custom classes for dynamic templates. Signed-off-by: Youssef Aouichaoui --- .../MappingElasticsearchConverter.java | 28 +++++++--- .../index/MappingBuilderIntegrationTests.java | 55 +++++++++++++++---- ...test-dynamic_templates_mappings_three.json | 9 +++ 3 files changed, 74 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java index 64ea4afcb5..d59069e6fe 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java @@ -677,13 +677,18 @@ private void populateFieldsUsingDynamicTemplates(ElasticsearchPersistentEnti ElasticsearchPersistentProperty property = targetEntity .getPersistentPropertyWithFieldName(templateEntry.getKey()); if (property != null && property.isDynamicFieldMapping()) { - targetEntity.getPropertyAccessor(result).setProperty(property, - document.entrySet().stream() - .filter(fieldKey -> templateEntry.getValue().getMatch().stream() - .anyMatch(regex -> simpleMatch(regex, fieldKey.getKey())) - && templateEntry.getValue().getUnmatch().stream() - .noneMatch(regex -> simpleMatch(regex, fieldKey.getKey()))) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue))); + // prepare value + Map values = new HashMap<>(); + // TODO: Path match and unmatched + document.entrySet().stream() + .filter(fieldKey -> templateEntry.getValue().getMatch().stream() + .anyMatch(regex -> simpleMatch(regex, fieldKey.getKey())) + && templateEntry.getValue().getUnmatch().stream() + .noneMatch(regex -> simpleMatch(regex, fieldKey.getKey()))) + .forEach(entry -> values.put(entry.getKey(), entry.getValue())); + + // set property + targetEntity.getPropertyAccessor(result).setProperty(property, read(property.getType(), Document.from(values))); } } } @@ -1089,7 +1094,14 @@ protected void writeProperty(ElasticsearchPersistentProperty property, Object va addCustomTypeKeyIfNecessary(value, document, TypeInformation.of(property.getRawType())); writeInternal(value, document, entity); - sink.set(property, document); + if (property.isDynamicFieldMapping()) { + // flatten + for (Entry entry : document.entrySet()) { + sink.set(entry.getKey(), entry.getValue()); + } + } else { + sink.set(property, document); + } } /** diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java index 9d68b7905f..55b2aa9608 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java @@ -23,14 +23,7 @@ import java.time.Instant; import java.time.LocalDate; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; @@ -293,15 +286,24 @@ void shouldMapDynamicFields() { DynamicFieldDocument document = new DynamicFieldDocument(); document.dynamicFields = Map.of("a_str", randomUUID().toString(), "b_str", randomUUID().toString()); + document.value = new DynamicFieldDocument.Value(1L, new Date()); operations.save(document); // When - SearchHits results = operations.search(new StringQuery(MATCH_ALL), DynamicFieldDocument.class); + SearchHits results = operations.search(new StringQuery(MATCH_ALL), + DynamicFieldDocument.class); // Then assertThat(results.getTotalHits()).isEqualTo(1); - assertThat(results.getSearchHits()).first().extracting(SearchHit::getContent).extracting(doc -> doc.dynamicFields) + assertThat(results.getSearchHits()).first() + .extracting(SearchHit::getContent) + .extracting(doc -> doc.dynamicFields) .isEqualTo(document.dynamicFields); + assertThat(results.getSearchHits()).first() + .extracting(SearchHit::getContent) + .extracting(doc -> doc.value) + .isEqualTo(document.value); + documentOperations.delete(); } @@ -966,6 +968,39 @@ private static class DynamicFieldDocument { @Id String id; @Field(name = "_str", dynamicTemplate = true) private Map dynamicFields = new HashMap<>(); + + @Nullable + @Field(name = "obj", dynamicTemplate = true) private Value value; + + static class Value { + @Nullable + @Field(name = "value_sum", type = FieldType.Long) + private Long sum; + + @Nullable + @Field(name = "value_date", type = FieldType.Long) + private Date date; + + public Value() { + } + + public Value(Long sum, Date date) { + this.sum = sum; + this.date = date; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Value value)) return false; + return Objects.equals(sum, value.sum) && Objects.equals(date, value.date); + } + + @Override + public int hashCode() { + return Objects.hash(sum, date); + } + } } // endregion } diff --git a/src/test/resources/mappings/test-dynamic_templates_mappings_three.json b/src/test/resources/mappings/test-dynamic_templates_mappings_three.json index 4ead5aa64b..01bf783983 100644 --- a/src/test/resources/mappings/test-dynamic_templates_mappings_three.json +++ b/src/test/resources/mappings/test-dynamic_templates_mappings_three.json @@ -7,6 +7,15 @@ "type": "keyword" } } + }, + { + "obj": { + "match": "value_*", + "mapping": { + "type": "text", + "index": false + } + } } ] } From 9b66c6908cc0d940d58162c7b19a089c0e116676 Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Mon, 2 Sep 2024 17:01:10 +0200 Subject: [PATCH 6/8] Refresh entities mapping cache. Signed-off-by: Youssef Aouichaoui --- .../client/elc/IndicesTemplate.java | 46 ++++++++ .../client/elc/ReactiveIndicesTemplate.java | 9 ++ .../client/elc/ResponseConverter.java | 11 ++ .../core/AbstractElasticsearchTemplate.java | 10 -- .../elasticsearch/core/IndexOperations.java | 8 ++ .../core/IndexOperationsAdapter.java | 6 + .../core/ReactiveIndexOperations.java | 8 ++ .../core/cluster/ClusterMapping.java | 106 ++++++++++++++++++ 8 files changed, 194 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java index c46f1da0ec..a1913fed9c 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java @@ -34,6 +34,7 @@ import org.springframework.data.elasticsearch.core.IndexInformation; import org.springframework.data.elasticsearch.core.IndexOperations; import org.springframework.data.elasticsearch.core.ResourceUtil; +import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.*; @@ -81,6 +82,8 @@ public IndicesTemplate(ElasticsearchIndicesClient client, ClusterTemplate cluste this.boundClass = boundClass; this.boundIndex = null; + // cache entities metadata + refreshEntitiesMapping(); } public IndicesTemplate(ElasticsearchIndicesClient client, ClusterTemplate clusterTemplate, @@ -96,6 +99,8 @@ public IndicesTemplate(ElasticsearchIndicesClient client, ClusterTemplate cluste this.boundClass = null; this.boundIndex = boundIndex; + // cache entities metadata + refreshEntitiesMapping(); } protected Class checkForBoundClass() { @@ -145,6 +150,8 @@ protected boolean doCreate(IndexCoordinates indexCoordinates, Map client.create(createIndexRequest)); + // refresh cached mappings + refreshEntitiesMapping(); return Boolean.TRUE.equals(createIndexResponse.acknowledged()); } @@ -241,6 +248,45 @@ public Map getMapping() { return responseConverter.indicesGetMapping(getMappingResponse, indexCoordinates); } + @Override + public ClusterMapping getClusterMapping() { + GetMappingRequest getMappingRequest = requestConverter.indicesGetMappingRequest(IndexCoordinates.of("*")); + GetMappingResponse getMappingResponse = execute(client -> client.getMapping(getMappingRequest)); + + return responseConverter.indicesGetMapping(getMappingResponse); + } + + /** + * Refreshes the mapping of entities. + *

+ * This method is responsible for retrieving and updating the metadata related to the entities. + */ + private void refreshEntitiesMapping() { + ClusterMapping clusterMapping = getClusterMapping(); + for (ClusterMapping.ClusterMappingEntry mappingEntry : clusterMapping) { + // Get entity + Class entity = null; + for (ElasticsearchPersistentEntity persistentEntity : this.elasticsearchConverter.getMappingContext().getPersistentEntities()) { + if (mappingEntry.getName().equals(persistentEntity.getIndexCoordinates().getIndexName())) { + entity = persistentEntity.getType(); + + break; + } + } + + if (entity == null) { + continue; + } + + if (mappingEntry.getMappings().containsKey("dynamic_templates")) { + Object dynamicTemplates = mappingEntry.getMappings().get("dynamic_templates"); + if (dynamicTemplates instanceof List value) { + getRequiredPersistentEntity(entity).buildDynamicTemplates(value); + } + } + } + } + @Override public Settings createSettings() { return createSettings(checkForBoundClass()); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java index d4c7a6e1b3..34f50f9a2c 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java @@ -36,6 +36,7 @@ import org.springframework.data.elasticsearch.core.IndexInformation; import org.springframework.data.elasticsearch.core.ReactiveIndexOperations; import org.springframework.data.elasticsearch.core.ReactiveResourceUtil; +import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.*; @@ -218,6 +219,14 @@ public Mono getMapping() { return getMappingResponse.map(response -> responseConverter.indicesGetMapping(response, indexCoordinates)); } + @Override + public Mono getClusterMapping() { + GetMappingRequest getMappingRequest = requestConverter.indicesGetMappingRequest(IndexCoordinates.of("*")); + Mono getMappingResponse = Mono.from(execute(client -> client.getMapping(getMappingRequest))); + + return getMappingResponse.map(responseConverter::indicesGetMapping); + } + @Override public Mono createSettings() { return createSettings(checkForBoundClass()); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java index be5e7f636a..dfb82a1dd9 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java @@ -52,6 +52,7 @@ import org.springframework.data.elasticsearch.core.IndexInformation; import org.springframework.data.elasticsearch.core.MultiGetItem; import org.springframework.data.elasticsearch.core.cluster.ClusterHealth; +import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.AliasData; import org.springframework.data.elasticsearch.core.index.Settings; @@ -213,6 +214,16 @@ public Document indicesGetMapping(GetMappingResponse getMappingResponse, IndexCo return Document.parse(toJson(indexMappingRecord.mappings(), jsonpMapper)); } + public ClusterMapping indicesGetMapping(GetMappingResponse getMappingResponse) { + ClusterMapping.Builder builder = ClusterMapping.builder(); + for (String indexName : getMappingResponse.result().keySet()) { + Map mappings = indicesGetMapping(getMappingResponse, IndexCoordinates.of(indexName)); + builder.withMapping(ClusterMapping.ClusterMappingEntry.builder(indexName).withMappings(mappings).build()); + } + + return builder.build(); + } + public List indicesGetIndexInformations(GetIndexResponse getIndexResponse) { Assert.notNull(getIndexResponse, "getIndexResponse must not be null"); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java index 7c789a5afe..4275e17f8b 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java @@ -20,7 +20,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -603,15 +602,6 @@ protected SearchDocumentResponse.EntityCreator getEntityCreator(ReadDocum // region Entity callbacks protected T maybeCallbackBeforeConvert(T entity, IndexCoordinates index) { - // get entity metadata - Map mapping = indexOps(index).getMapping(); - if (mapping.containsKey("dynamic_templates")) { - Object dynamicTemplates = mapping.get("dynamic_templates"); - if (dynamicTemplates instanceof List value) { - getRequiredPersistentEntity(entity.getClass()).buildDynamicTemplates(value); - } - } - if (entityCallbacks != null) { return entityCallbacks.callback(BeforeConvertCallback.class, entity, index); } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java index d09fb3861f..58375cd16f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java @@ -19,6 +19,7 @@ import java.util.Map; import java.util.Set; +import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.*; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; @@ -144,6 +145,13 @@ default boolean putMapping(Class clazz) { */ Map getMapping(); + /** + * Get mappings of all indices in a cluster. + * + * @return Retrieve the mappings for all indices within a cluster. + */ + ClusterMapping getClusterMapping(); + // endregion // region settings diff --git a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java index 397f5146d5..e3c7788d8f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java @@ -15,6 +15,7 @@ */ package org.springframework.data.elasticsearch.core; +import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import reactor.core.publisher.Mono; import java.util.List; @@ -96,6 +97,11 @@ public Map getMapping() { return Objects.requireNonNull(reactiveIndexOperations.getMapping().block()); } + @Override + public ClusterMapping getClusterMapping() { + return Objects.requireNonNull(reactiveIndexOperations.getClusterMapping().block()); + } + @Override public Settings createSettings() { return Objects.requireNonNull(reactiveIndexOperations.createSettings().block()); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java index a4cc50c222..8d5df3bc0b 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java @@ -15,6 +15,7 @@ */ package org.springframework.data.elasticsearch.core; +import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -144,6 +145,13 @@ default Mono putMapping(Class clazz) { * @return the mapping */ Mono getMapping(); + + /** + * Get mappings of all indices in a cluster. + * + * @return Retrieve the mappings for all indices within a cluster. + */ + Mono getClusterMapping(); // endregion // region settings diff --git a/src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java b/src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java new file mode 100644 index 0000000000..46f21df0fb --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java @@ -0,0 +1,106 @@ +/* + * Copyright 2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core.cluster; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.jetbrains.annotations.NotNull; + +/** + * Retrieves mapping definitions of all indices in the cluster. + * + * @author Youssef Aouichaoui + * @since 5.4 + */ +public class ClusterMapping implements Iterable { + private final List mappings; + + private ClusterMapping(Builder builder) { + this.mappings = builder.mappings; + } + + @NotNull + @Override + public Iterator iterator() { + return mappings.iterator(); + } + + public static Builder builder() { + return new Builder(); + } + + public static class ClusterMappingEntry { + private final String name; + private final Map mappings; + + private ClusterMappingEntry(Builder builder) { + this.name = builder.name; + this.mappings = builder.mappings; + } + + public String getName() { + return name; + } + + public Map getMappings() { + return Collections.unmodifiableMap(mappings); + } + + public static Builder builder(String name) { + return new Builder(name); + } + + public static class Builder { + private final String name; + private final Map mappings = new HashMap<>(); + + private Builder(String name) { + this.name = name; + } + + public Builder withMappings(Map mappings) { + this.mappings.putAll(mappings); + + return this; + } + + public ClusterMappingEntry build() { + return new ClusterMappingEntry(this); + } + } + } + + public static class Builder { + private final List mappings = new ArrayList<>(); + + private Builder() {} + + public Builder withMapping(ClusterMappingEntry entry) { + mappings.add(entry); + + return this; + } + + public ClusterMapping build() { + return new ClusterMapping(this); + } + } +} From e38fbb3763a3aec300e8833bbef0735a35d20c16 Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Fri, 6 Sep 2024 13:21:38 +0200 Subject: [PATCH 7/8] Polishing. Signed-off-by: Youssef Aouichaoui --- .../data/elasticsearch/client/elc/IndicesTemplate.java | 6 +++--- .../core/index/MappingBuilderIntegrationTests.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java index a1913fed9c..a0645c87ba 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java @@ -265,10 +265,10 @@ private void refreshEntitiesMapping() { ClusterMapping clusterMapping = getClusterMapping(); for (ClusterMapping.ClusterMappingEntry mappingEntry : clusterMapping) { // Get entity - Class entity = null; + ElasticsearchPersistentEntity entity = null; for (ElasticsearchPersistentEntity persistentEntity : this.elasticsearchConverter.getMappingContext().getPersistentEntities()) { if (mappingEntry.getName().equals(persistentEntity.getIndexCoordinates().getIndexName())) { - entity = persistentEntity.getType(); + entity = persistentEntity; break; } @@ -281,7 +281,7 @@ private void refreshEntitiesMapping() { if (mappingEntry.getMappings().containsKey("dynamic_templates")) { Object dynamicTemplates = mappingEntry.getMappings().get("dynamic_templates"); if (dynamicTemplates instanceof List value) { - getRequiredPersistentEntity(entity).buildDynamicTemplates(value); + entity.buildDynamicTemplates(value); } } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java index 55b2aa9608..dffd5b5145 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java @@ -961,7 +961,7 @@ private static class FieldAliasEntity { } @SuppressWarnings("unused") - @Document(indexName = "foo") + @Document(indexName = "#{@indexNameProvider.indexName()}-foo") @DynamicTemplates(mappingPath = "/mappings/test-dynamic_templates_mappings_three.json") private static class DynamicFieldDocument { @Nullable From 9ac79d18fbaed1c6d5f82cbd283ec635426e18cb Mon Sep 17 00:00:00 2001 From: Youssef Aouichaoui Date: Tue, 10 Sep 2024 15:58:20 +0200 Subject: [PATCH 8/8] Get only the current index mapping. Signed-off-by: Youssef Aouichaoui --- .../client/elc/IndicesTemplate.java | 54 +++------ .../client/elc/ReactiveIndicesTemplate.java | 35 ++++-- .../client/elc/ResponseConverter.java | 11 -- .../elasticsearch/core/IndexOperations.java | 8 -- .../core/IndexOperationsAdapter.java | 6 - .../core/ReactiveIndexOperations.java | 8 -- .../core/cluster/ClusterMapping.java | 106 ------------------ ...ReactiveIndexTemplateIntegrationTests.java | 86 +++++++++++++- 8 files changed, 126 insertions(+), 188 deletions(-) delete mode 100644 src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java index a0645c87ba..4b20ef1f9a 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/IndicesTemplate.java @@ -34,7 +34,6 @@ import org.springframework.data.elasticsearch.core.IndexInformation; import org.springframework.data.elasticsearch.core.IndexOperations; import org.springframework.data.elasticsearch.core.ResourceUtil; -import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.*; @@ -81,9 +80,6 @@ public IndicesTemplate(ElasticsearchIndicesClient client, ClusterTemplate cluste this.elasticsearchConverter = elasticsearchConverter; this.boundClass = boundClass; this.boundIndex = null; - - // cache entities metadata - refreshEntitiesMapping(); } public IndicesTemplate(ElasticsearchIndicesClient client, ClusterTemplate clusterTemplate, @@ -98,9 +94,6 @@ public IndicesTemplate(ElasticsearchIndicesClient client, ClusterTemplate cluste this.elasticsearchConverter = elasticsearchConverter; this.boundClass = null; this.boundIndex = boundIndex; - - // cache entities metadata - refreshEntitiesMapping(); } protected Class checkForBoundClass() { @@ -151,7 +144,7 @@ protected boolean doCreate(IndexCoordinates indexCoordinates, Map client.create(createIndexRequest)); // refresh cached mappings - refreshEntitiesMapping(); + refreshMapping(); return Boolean.TRUE.equals(createIndexResponse.acknowledged()); } @@ -248,42 +241,25 @@ public Map getMapping() { return responseConverter.indicesGetMapping(getMappingResponse, indexCoordinates); } - @Override - public ClusterMapping getClusterMapping() { - GetMappingRequest getMappingRequest = requestConverter.indicesGetMappingRequest(IndexCoordinates.of("*")); - GetMappingResponse getMappingResponse = execute(client -> client.getMapping(getMappingRequest)); - - return responseConverter.indicesGetMapping(getMappingResponse); - } - /** - * Refreshes the mapping of entities. + * Refreshes the mapping for the current entity. *

- * This method is responsible for retrieving and updating the metadata related to the entities. + * This method is responsible for retrieving and updating the metadata related to the current entity. */ - private void refreshEntitiesMapping() { - ClusterMapping clusterMapping = getClusterMapping(); - for (ClusterMapping.ClusterMappingEntry mappingEntry : clusterMapping) { - // Get entity - ElasticsearchPersistentEntity entity = null; - for (ElasticsearchPersistentEntity persistentEntity : this.elasticsearchConverter.getMappingContext().getPersistentEntities()) { - if (mappingEntry.getName().equals(persistentEntity.getIndexCoordinates().getIndexName())) { - entity = persistentEntity; - - break; - } - } + private void refreshMapping() { + if (boundClass == null) { + return; + } - if (entity == null) { - continue; - } + ElasticsearchPersistentEntity entity = this.elasticsearchConverter.getMappingContext() + .getPersistentEntity(boundClass); + if (entity == null) { + return; + } - if (mappingEntry.getMappings().containsKey("dynamic_templates")) { - Object dynamicTemplates = mappingEntry.getMappings().get("dynamic_templates"); - if (dynamicTemplates instanceof List value) { - entity.buildDynamicTemplates(value); - } - } + Object dynamicTemplates = getMapping().get("dynamic_templates"); + if (dynamicTemplates instanceof List value) { + entity.buildDynamicTemplates(value); } } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java index 34f50f9a2c..2bb2e8583b 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveIndicesTemplate.java @@ -25,6 +25,7 @@ import reactor.core.publisher.Mono; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -36,7 +37,6 @@ import org.springframework.data.elasticsearch.core.IndexInformation; import org.springframework.data.elasticsearch.core.ReactiveIndexOperations; import org.springframework.data.elasticsearch.core.ReactiveResourceUtil; -import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.*; @@ -143,7 +143,12 @@ private Mono doCreate(IndexCoordinates indexCoordinates, Map createIndexResponse = Mono.from(execute(client -> client.create(createIndexRequest))); - return createIndexResponse.map(CreateIndexResponse::acknowledged); + return createIndexResponse + .doOnNext((result) -> { + // refresh cached mappings + refreshMapping(); + }) + .map(CreateIndexResponse::acknowledged); } @Override @@ -219,12 +224,28 @@ public Mono getMapping() { return getMappingResponse.map(response -> responseConverter.indicesGetMapping(response, indexCoordinates)); } - @Override - public Mono getClusterMapping() { - GetMappingRequest getMappingRequest = requestConverter.indicesGetMappingRequest(IndexCoordinates.of("*")); - Mono getMappingResponse = Mono.from(execute(client -> client.getMapping(getMappingRequest))); + /** + * Refreshes the mapping for the current entity. + *

+ * This method is responsible for retrieving and updating the metadata related to the current entity. + */ + private void refreshMapping() { + if (boundClass == null) { + return; + } + + ElasticsearchPersistentEntity entity = this.elasticsearchConverter.getMappingContext() + .getPersistentEntity(boundClass); + if (entity == null) { + return; + } - return getMappingResponse.map(responseConverter::indicesGetMapping); + getMapping().subscribe((mappings) -> { + Object dynamicTemplates = mappings.get("dynamic_templates"); + if (dynamicTemplates instanceof List value) { + entity.buildDynamicTemplates(value); + } + }); } @Override diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java index dfb82a1dd9..be5e7f636a 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ResponseConverter.java @@ -52,7 +52,6 @@ import org.springframework.data.elasticsearch.core.IndexInformation; import org.springframework.data.elasticsearch.core.MultiGetItem; import org.springframework.data.elasticsearch.core.cluster.ClusterHealth; -import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.AliasData; import org.springframework.data.elasticsearch.core.index.Settings; @@ -214,16 +213,6 @@ public Document indicesGetMapping(GetMappingResponse getMappingResponse, IndexCo return Document.parse(toJson(indexMappingRecord.mappings(), jsonpMapper)); } - public ClusterMapping indicesGetMapping(GetMappingResponse getMappingResponse) { - ClusterMapping.Builder builder = ClusterMapping.builder(); - for (String indexName : getMappingResponse.result().keySet()) { - Map mappings = indicesGetMapping(getMappingResponse, IndexCoordinates.of(indexName)); - builder.withMapping(ClusterMapping.ClusterMappingEntry.builder(indexName).withMappings(mappings).build()); - } - - return builder.build(); - } - public List indicesGetIndexInformations(GetIndexResponse getIndexResponse) { Assert.notNull(getIndexResponse, "getIndexResponse must not be null"); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java index 58375cd16f..d09fb3861f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperations.java @@ -19,7 +19,6 @@ import java.util.Map; import java.util.Set; -import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import org.springframework.data.elasticsearch.core.document.Document; import org.springframework.data.elasticsearch.core.index.*; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; @@ -145,13 +144,6 @@ default boolean putMapping(Class clazz) { */ Map getMapping(); - /** - * Get mappings of all indices in a cluster. - * - * @return Retrieve the mappings for all indices within a cluster. - */ - ClusterMapping getClusterMapping(); - // endregion // region settings diff --git a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java index e3c7788d8f..397f5146d5 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/IndexOperationsAdapter.java @@ -15,7 +15,6 @@ */ package org.springframework.data.elasticsearch.core; -import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import reactor.core.publisher.Mono; import java.util.List; @@ -97,11 +96,6 @@ public Map getMapping() { return Objects.requireNonNull(reactiveIndexOperations.getMapping().block()); } - @Override - public ClusterMapping getClusterMapping() { - return Objects.requireNonNull(reactiveIndexOperations.getClusterMapping().block()); - } - @Override public Settings createSettings() { return Objects.requireNonNull(reactiveIndexOperations.createSettings().block()); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java index 8d5df3bc0b..a4cc50c222 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ReactiveIndexOperations.java @@ -15,7 +15,6 @@ */ package org.springframework.data.elasticsearch.core; -import org.springframework.data.elasticsearch.core.cluster.ClusterMapping; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -145,13 +144,6 @@ default Mono putMapping(Class clazz) { * @return the mapping */ Mono getMapping(); - - /** - * Get mappings of all indices in a cluster. - * - * @return Retrieve the mappings for all indices within a cluster. - */ - Mono getClusterMapping(); // endregion // region settings diff --git a/src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java b/src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java deleted file mode 100644 index 46f21df0fb..0000000000 --- a/src/main/java/org/springframework/data/elasticsearch/core/cluster/ClusterMapping.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2024 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.data.elasticsearch.core.cluster; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.jetbrains.annotations.NotNull; - -/** - * Retrieves mapping definitions of all indices in the cluster. - * - * @author Youssef Aouichaoui - * @since 5.4 - */ -public class ClusterMapping implements Iterable { - private final List mappings; - - private ClusterMapping(Builder builder) { - this.mappings = builder.mappings; - } - - @NotNull - @Override - public Iterator iterator() { - return mappings.iterator(); - } - - public static Builder builder() { - return new Builder(); - } - - public static class ClusterMappingEntry { - private final String name; - private final Map mappings; - - private ClusterMappingEntry(Builder builder) { - this.name = builder.name; - this.mappings = builder.mappings; - } - - public String getName() { - return name; - } - - public Map getMappings() { - return Collections.unmodifiableMap(mappings); - } - - public static Builder builder(String name) { - return new Builder(name); - } - - public static class Builder { - private final String name; - private final Map mappings = new HashMap<>(); - - private Builder(String name) { - this.name = name; - } - - public Builder withMappings(Map mappings) { - this.mappings.putAll(mappings); - - return this; - } - - public ClusterMappingEntry build() { - return new ClusterMappingEntry(this); - } - } - } - - public static class Builder { - private final List mappings = new ArrayList<>(); - - private Builder() {} - - public Builder withMapping(ClusterMappingEntry entry) { - mappings.add(entry); - - return this; - } - - public ClusterMapping build() { - return new ClusterMapping(this); - } - } -} diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/ReactiveIndexTemplateIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/ReactiveIndexTemplateIntegrationTests.java index e64ce7e104..adbad32236 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/ReactiveIndexTemplateIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/ReactiveIndexTemplateIntegrationTests.java @@ -18,12 +18,12 @@ import static org.assertj.core.api.Assertions.*; import static org.skyscreamer.jsonassert.JSONAssert.*; import static org.springframework.data.elasticsearch.core.IndexOperationsAdapter.*; +import static org.springframework.data.elasticsearch.core.query.StringQuery.MATCH_ALL; +import reactor.core.publisher.Flux; import reactor.test.StepVerifier; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import org.assertj.core.api.SoftAssertions; import org.json.JSONException; @@ -34,16 +34,20 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.Id; import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.DynamicTemplates; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.elasticsearch.annotations.Setting; import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.IndexOperationsAdapter; import org.springframework.data.elasticsearch.core.ReactiveElasticsearchOperations; import org.springframework.data.elasticsearch.core.ReactiveIndexOperations; +import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.data.elasticsearch.core.query.Criteria; import org.springframework.data.elasticsearch.core.query.CriteriaQuery; import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.StringQuery; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; import org.springframework.data.elasticsearch.utils.IndexNameProvider; import org.springframework.lang.Nullable; @@ -415,6 +419,40 @@ void shouldDeleteTemplate() { assertThat(exists).isFalse(); } + @Test + void shouldMapDynamicFields() { + // Given + IndexOperationsAdapter documentOperations = blocking(operations.indexOps(DynamicFieldDocument.class)); + documentOperations.createWithMapping(); + + DynamicFieldDocument document = new DynamicFieldDocument(); + document.dynamicFields = Map.of("a_str", UUID.randomUUID().toString(), "b_str", UUID.randomUUID().toString()); + document.value = new DynamicFieldDocument.Value(1L, new Date()); + operations.save(document).block(); + + // When + Flux> results = operations.search(new StringQuery(MATCH_ALL), + DynamicFieldDocument.class); + + // Then + results.as(StepVerifier::create) + .expectNextMatches((hits) -> { + assertThat(hits) + .extracting(SearchHit::getContent) + .extracting(doc -> doc.dynamicFields) + .isEqualTo(document.dynamicFields); + + assertThat(hits) + .extracting(SearchHit::getContent) + .extracting(doc -> doc.value) + .isEqualTo(document.value); + + return true; + }).verifyComplete(); + + documentOperations.delete(); + } + @Document(indexName = "#{@indexNameProvider.indexName()}") @Setting(refreshInterval = "5s") static class TemplateClass { @@ -442,4 +480,46 @@ public void setMessage(@Nullable String message) { } } + @SuppressWarnings("unused") + @Document(indexName = "#{@indexNameProvider.indexName()}-foo") + @DynamicTemplates(mappingPath = "/mappings/test-dynamic_templates_mappings_three.json") + private static class DynamicFieldDocument { + @Nullable + @Id String id; + + @Field(name = "_str", dynamicTemplate = true) private Map dynamicFields = new HashMap<>(); + + @Nullable + @Field(name = "obj", dynamicTemplate = true) private Value value; + + static class Value { + @Nullable + @Field(name = "value_sum", type = FieldType.Long) + private Long sum; + + @Nullable + @Field(name = "value_date", type = FieldType.Long) + private Date date; + + public Value() { + } + + public Value(Long sum, Date date) { + this.sum = sum; + this.date = date; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Value value)) return false; + return Objects.equals(sum, value.sum) && Objects.equals(date, value.date); + } + + @Override + public int hashCode() { + return Objects.hash(sum, date); + } + } + } }