diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index f0d9ee944..60156b4a7 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index c00eda255..7293708be 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -16,6 +16,7 @@ import org.apache.lucene.store.FilterDirectory; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; diff --git a/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java index d573b9194..862d680ce 100644 --- a/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.plugin.stats.KNNCounter; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 05a0873f7..491f6e1cc 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -30,7 +30,7 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.common.KNNConstants; import java.io.Closeable; diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java similarity index 68% rename from src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java rename to src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 628995b5c..45d34dd68 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -3,16 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.mapper; import lombok.Getter; -import org.opensearch.common.Strings; import org.opensearch.common.ValidationException; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.common.KNNConstants; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; @@ -20,7 +16,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.opensearch.common.Explicit; -import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentParser; @@ -36,9 +31,11 @@ import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.KNNVectorIndexFieldData; +import org.opensearch.knn.index.VectorField; import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -47,17 +44,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; -import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; /** * Field Mapper for KNN vector type. @@ -69,8 +59,6 @@ */ public abstract class KNNVectorFieldMapper extends ParametrizedFieldMapper { - private static Logger logger = LogManager.getLogger(KNNVectorFieldMapper.class); - public static final String CONTENT_TYPE = "knn_vector"; public static final String KNN_FIELD = "knn_field"; @@ -99,11 +87,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder { } int value = XContentMapValues.nodeIntegerValue(o); if (value > MAX_DIMENSION) { - throw new IllegalArgumentException("Dimension value cannot be greater than " + MAX_DIMENSION + " for vector: " + name); + throw new IllegalArgumentException( + String.format("Dimension value cannot be greater than %s for vector: %s", MAX_DIMENSION, name) + ); } if (value <= 0) { - throw new IllegalArgumentException("Dimension value must be greater than 0 " + "for vector: " + name); + throw new IllegalArgumentException(String.format("Dimension value must be greater than 0 for vector: %s", name)); } return value; }, m -> toType(m).dimension); @@ -285,12 +275,12 @@ public Mapper.Builder parse(String name, Map node, ParserCont // is done before any mappers are built. Therefore, validation should be done during parsing // so that it can fail early. if (builder.knnMethodContext.get() != null && builder.modelId.get() != null) { - throw new IllegalArgumentException("Method and model can not be both specified in the mapping: " + name); + throw new IllegalArgumentException(String.format("Method and model can not be both specified in the mapping: %s", name)); } // Dimension should not be null unless modelId is used if (builder.dimension.getValue() == -1 && builder.modelId.get() == null) { - throw new IllegalArgumentException("Dimension value missing for vector: " + name); + throw new IllegalArgumentException(String.format("Dimension value missing for vector: %s", name)); } return builder; @@ -337,7 +327,7 @@ public Query existsQuery(QueryShardContext context) { public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException( context, - "KNN vector do not support exact searching, use KNN queries " + "instead: [" + name() + "]" + String.format("KNN vector do not support exact searching, use KNN queries instead: [%s]", name()) ); } @@ -392,16 +382,39 @@ protected void parseCreateField(ParseContext context) throws IOException { protected void parseCreateField(ParseContext context, int dimension) throws IOException { - if (!KNNSettings.isKNNPluginEnabled()) { - throw new IllegalStateException("KNN plugin is disabled. To enable " + "update knn.plugin.enabled setting to true"); + validateIfKNNPluginEnabled(); + validateIfCircuitBreakerIsNotTriggered(); + + Optional arrayOptional = getFloatsFromContext(context, dimension); + + if (!arrayOptional.isPresent()) { + return; } + final float[] array = arrayOptional.get(); + VectorField point = new VectorField(name(), array, fieldType); + context.doc().add(point); + if (fieldType.stored()) { + context.doc().add(new StoredField(name(), point.toString())); + } + context.path().remove(); + } + + void validateIfCircuitBreakerIsNotTriggered() { if (KNNSettings.isCircuitBreakerTriggered()) { throw new IllegalStateException( - "Indexing knn vector fields is rejected as circuit breaker triggered." + " Check _opendistro/_knn/stats for detailed state" + "Indexing knn vector fields is rejected as circuit breaker triggered. Check _opendistro/_knn/stats for detailed state" ); } + } + void validateIfKNNPluginEnabled() { + if (!KNNSettings.isKNNPluginEnabled()) { + throw new IllegalStateException("KNN plugin is disabled. To enable update knn.plugin.enabled setting to true"); + } + } + + Optional getFloatsFromContext(ParseContext context, int dimension) throws IOException { context.path().add(simpleName()); ArrayList vector = new ArrayList<>(); @@ -438,7 +451,7 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { context.path().remove(); - return; + return Optional.empty(); } if (dimension != vector.size()) { @@ -451,14 +464,7 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx for (Float f : vector) { array[i++] = f; } - - VectorField point = new VectorField(name(), array, fieldType); - - context.doc().add(point); - if (fieldType.stored()) { - context.doc().add(new StoredField(name(), point.toString())); - } - context.path().remove(); + return Optional.of(array); } @Override @@ -505,187 +511,4 @@ public static class Defaults { FIELD_TYPE.freeze(); } } - - /** - * Field mapper for original implementation - */ - protected static class LegacyFieldMapper extends KNNVectorFieldMapper { - - protected String spaceType; - protected String m; - protected String efConstruction; - - private LegacyFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - String spaceType, - String m, - String efConstruction - ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); - - this.spaceType = spaceType; - this.m = m; - this.efConstruction = efConstruction; - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - - this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); - this.fieldType.putAttribute(SPACE_TYPE, spaceType); - this.fieldType.putAttribute(KNN_ENGINE, KNNEngine.NMSLIB.getName()); - - // These are extra just for legacy - this.fieldType.putAttribute(HNSW_ALGO_M, m); - this.fieldType.putAttribute(HNSW_ALGO_EF_CONSTRUCTION, efConstruction); - - this.fieldType.freeze(); - } - - @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new KNNVectorFieldMapper.Builder(simpleName(), this.spaceType, this.m, this.efConstruction).init(this); - } - - static String getSpaceType(Settings indexSettings) { - String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); - if (spaceType == null) { - logger.info( - "[KNN] The setting \"" - + METHOD_PARAMETER_SPACE_TYPE - + "\" was not set for the index. " - + "Likely caused by recent version upgrade. Setting the setting to the default value=" - + KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE - ); - return KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; - } - return spaceType; - } - - static String getM(Settings indexSettings) { - String m = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING.getKey()); - if (m == null) { - logger.info( - "[KNN] The setting \"" - + HNSW_ALGO_M - + "\" was not set for the index. " - + "Likely caused by recent version upgrade. Setting the setting to the default value=" - + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M - ); - return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M); - } - return m; - } - - static String getEfConstruction(Settings indexSettings) { - String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); - if (efConstruction == null) { - logger.info( - "[KNN] The setting \"" - + HNSW_ALGO_EF_CONSTRUCTION - + "\" was not set for" - + " the index. Likely caused by recent version upgrade. Setting the setting to the default value=" - + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION - ); - return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION); - } - return efConstruction; - } - } - - /** - * Field mapper for method definition in mapping - */ - protected static class MethodFieldMapper extends KNNVectorFieldMapper { - - private MethodFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - KNNMethodContext knnMethodContext - ) { - - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); - - this.knnMethod = knnMethodContext; - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - - this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); - this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); - - KNNEngine knnEngine = knnMethodContext.getKnnEngine(); - this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); - - try { - this.fieldType.putAttribute( - PARAMETERS, - Strings.toString(XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext))) - ); - } catch (IOException ioe) { - throw new RuntimeException("Unable to create KNNVectorFieldMapper: " + ioe); - } - - this.fieldType.freeze(); - } - } - - /** - * Field mapper for model in mapping - */ - protected static class ModelFieldMapper extends KNNVectorFieldMapper { - - private ModelFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - ModelDao modelDao, - String modelId - ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); - - this.modelId = modelId; - this.modelDao = modelDao; - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - this.fieldType.putAttribute(MODEL_ID, modelId); - this.fieldType.freeze(); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { - // For the model field mapper, we cannot validate the model during index creation due to - // an issue with reading cluster state during mapper creation. So, we need to validate the - // model when ingestion starts. - ModelMetadata modelMetadata = this.modelDao.getMetadata(modelId); - - if (modelMetadata == null) { - throw new IllegalStateException( - "Model \"" - + modelId - + "\" from " - + context.mapperService().index().getName() - + "'s mapping does not exist. Because the " - + "\"" - + MODEL_ID - + "\" parameter is not updateable, this index will need to " - + "be recreated with a valid model." - ); - } - - parseCreateField(context, modelMetadata.getDimension()); - } - } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java new file mode 100644 index 000000000..868aec3e6 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java @@ -0,0 +1,120 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.document.FieldType; +import org.opensearch.common.Explicit; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.mapper.ParametrizedFieldMapper; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.util.KNNEngine; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; + +/** + * Field mapper for original implementation. It defaults to using nmslib as the engine and retrieves parameters from index settings. + * + * Example of this mapper output: + * + * { + * "type": "knn_vector", + * "dimension": 128 + * } + */ +@Log4j2 +public class LegacyFieldMapper extends KNNVectorFieldMapper { + + protected String spaceType; + protected String m; + protected String efConstruction; + + LegacyFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + String spaceType, + String m, + String efConstruction + ) { + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + + this.spaceType = spaceType; + this.m = m; + this.efConstruction = efConstruction; + + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + + this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); + this.fieldType.putAttribute(SPACE_TYPE, spaceType); + this.fieldType.putAttribute(KNN_ENGINE, KNNEngine.NMSLIB.getName()); + + // These are extra just for legacy + this.fieldType.putAttribute(HNSW_ALGO_M, m); + this.fieldType.putAttribute(HNSW_ALGO_EF_CONSTRUCTION, efConstruction); + + this.fieldType.freeze(); + } + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new KNNVectorFieldMapper.Builder(simpleName(), this.spaceType, this.m, this.efConstruction).init(this); + } + + static String getSpaceType(Settings indexSettings) { + String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); + if (spaceType == null) { + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + METHOD_PARAMETER_SPACE_TYPE, + KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE + ) + ); + return KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; + } + return spaceType; + } + + static String getM(Settings indexSettings) { + String m = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING.getKey()); + if (m == null) { + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + HNSW_ALGO_M, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M + ) + ); + return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M); + } + return m; + } + + static String getEfConstruction(Settings indexSettings) { + String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); + if (efConstruction == null) { + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + HNSW_ALGO_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION + ) + ); + return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION); + } + return efConstruction; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java new file mode 100644 index 000000000..42c12a5db --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.opensearch.common.Explicit; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; + +/** + * Field mapper for method definition in mapping + */ +public class MethodFieldMapper extends KNNVectorFieldMapper { + + MethodFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + KNNMethodContext knnMethodContext + ) { + + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + + this.knnMethod = knnMethodContext; + + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + + this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); + this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); + + KNNEngine knnEngine = knnMethodContext.getKnnEngine(); + this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); + + try { + this.fieldType.putAttribute( + PARAMETERS, + Strings.toString(XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext))) + ); + } catch (IOException ioe) { + throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe)); + } + + this.fieldType.freeze(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java new file mode 100644 index 000000000..0fa116937 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.opensearch.common.Explicit; +import org.opensearch.index.mapper.ParseContext; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; + +/** + * Field mapper for model in mapping + */ +public class ModelFieldMapper extends KNNVectorFieldMapper { + + ModelFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + ModelDao modelDao, + String modelId + ) { + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + + this.modelId = modelId; + this.modelDao = modelDao; + + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + this.fieldType.putAttribute(MODEL_ID, modelId); + this.fieldType.freeze(); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + // For the model field mapper, we cannot validate the model during index creation due to + // an issue with reading cluster state during mapper creation. So, we need to validate the + // model when ingestion starts. + ModelMetadata modelMetadata = this.modelDao.getMetadata(modelId); + + if (modelMetadata == null) { + throw new IllegalStateException( + String.format( + "Model \"%s\" from %s's mapping does not exist. Because the \"%s\" parameter is not updatable, this index will need to be recreated with a valid model.", + modelId, + context.mapperService().index().getName(), + MODEL_ID + ) + ); + } + + parseCreateField(context, modelMetadata.getDimension()); + } +} diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 9aa0d133b..1f0ea786f 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -34,7 +34,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; -import static org.opensearch.knn.index.KNNVectorFieldMapper.MAX_DIMENSION; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.MAX_DIMENSION; public class ModelMetadata implements Writeable, ToXContentObject { diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 9cf0696f2..49a170eda 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -10,7 +10,7 @@ import org.opensearch.knn.index.KNNCircuitBreaker; import org.opensearch.knn.index.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.KNNWeight; import org.opensearch.knn.index.codec.KNNCodecService; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 3e066b7c9..4a419fd23 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -5,7 +5,7 @@ package org.opensearch.knn.plugin.script; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.KNNWeight; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.index.mapper.MappedFieldType; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index 4d64b5b96..6f68d16b6 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -5,7 +5,7 @@ package org.opensearch.knn.plugin.script; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.MappedFieldType; diff --git a/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java index ccf6bcfca..8a252ca72 100644 --- a/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java @@ -12,6 +12,7 @@ import org.opensearch.index.Index; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 6a06de14a..3fd492782 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -27,6 +27,7 @@ import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; import org.opensearch.rest.RestStatus; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index b86b4051a..19c57673d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -26,7 +26,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index b67bd1115..042dad331 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -15,7 +15,7 @@ import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.KNNQuery; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.KNNWeight; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorField; diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java similarity index 97% rename from src/test/java/org/opensearch/knn/index/KNNVectorFieldMapperTests.java rename to src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index b6a65cce9..6f489bd92 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.mapper; import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; @@ -17,6 +17,10 @@ import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -79,7 +83,7 @@ public void testBuilder_build_fromKnnMethodContext() { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof KNNVectorFieldMapper.MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); assertNotNull(knnVectorFieldMapper.knnMethod); assertNull(knnVectorFieldMapper.modelId); } @@ -116,7 +120,7 @@ public void testBuilder_build_fromModel() { when(modelDao.getMetadata(modelId)).thenReturn(mockedModelMetadata); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof KNNVectorFieldMapper.ModelFieldMapper); + assertTrue(knnVectorFieldMapper instanceof ModelFieldMapper); assertNotNull(knnVectorFieldMapper.modelId); assertNull(knnVectorFieldMapper.knnMethod); } @@ -140,7 +144,7 @@ public void testBuilder_build_fromLegacy() { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof KNNVectorFieldMapper.LegacyFieldMapper); + assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); assertNull(knnVectorFieldMapper.modelId); assertNull(knnVectorFieldMapper.knnMethod); diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index 7a01fd528..fb1e807fd 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -21,7 +21,7 @@ import java.util.HashMap; import java.util.Map; -import static org.opensearch.knn.index.KNNVectorFieldMapper.MAX_DIMENSION; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.MAX_DIMENSION; public class ModelTests extends KNNTestCase { diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java index 70dbee248..24bc74ff4 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.index.mapper.NumberFieldMapper; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index b40cc9e86..c80949b43 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -7,7 +7,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java index 789432ec8..92fd56e45 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index a8617896b..0f656e2d1 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -7,7 +7,7 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.apache.http.util.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 06b6f3d01..2030bc944 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -25,7 +25,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao;