From 0e58cf283f0072b311d14d7623f8272405bf2847 Mon Sep 17 00:00:00 2001 From: panguixin Date: Wed, 27 Nov 2024 21:32:54 +0800 Subject: [PATCH 01/10] Fix multi-value sort for unsigned long Signed-off-by: panguixin --- .../UnsignedLongMultiValueMode.java | 400 ++++++++++++++++++ .../UnsignedLongValuesComparatorSource.java | 6 +- .../UnsignedLongMultiValueModeTests.java | 271 ++++++++++++ 3 files changed, 675 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java create mode 100644 server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java new file mode 100644 index 0000000000000..4e20460f5c820 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java @@ -0,0 +1,400 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata.fieldcomparator; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.opensearch.common.Numbers; +import org.opensearch.index.fielddata.AbstractNumericDocValues; +import org.opensearch.index.fielddata.FieldData; +import org.opensearch.search.MultiValueMode; + +import java.io.IOException; +import java.util.Locale; + +/** + * Defines what values to pick in the case a document contains multiple values for an unsigned long field. + * + * @opensearch.internal + */ +enum UnsignedLongMultiValueMode { + /** + * Pick the sum of all the values. + */ + SUM { + @Override + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); + } + return total; + } + + @Override + protected long pick( + SortedNumericDocValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + int totalCount = 0; + long totalValue = 0; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + + final int docCount = values.docValueCount(); + for (int index = 0; index < docCount; ++index) { + totalValue += values.nextValue(); + } + totalCount += docCount; + } + } + return totalCount > 0 ? totalValue : missingValue; + } + }, + + /** + * Pick the average of all the values. + */ + AVG { + @Override + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); + } + return count > 1 ? divideUnsignedAndRoundUp(total, count) : total; + } + + @Override + protected long pick( + SortedNumericDocValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + int totalCount = 0; + long totalValue = 0; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + final int docCount = values.docValueCount(); + for (int index = 0; index < docCount; ++index) { + totalValue += values.nextValue(); + } + totalCount += docCount; + } + } + if (totalCount < 1) { + return missingValue; + } + return totalCount > 1 ? divideUnsignedAndRoundUp(totalValue, totalCount) : totalValue; + } + }, + + /** + * Pick the median of the values. + */ + MEDIAN { + @Override + protected long pick(SortedNumericDocValues values) throws IOException { + int count = values.docValueCount(); + long firstValue = values.nextValue(); + if (count == 1) { + return firstValue; + } else if (count == 2) { + long total = firstValue + values.nextValue(); + return (total >>> 1) + (total & 1); + } else if (firstValue >= 0) { + for (int i = 1; i < (count - 1) / 2; ++i) { + values.nextValue(); + } + if (count % 2 == 0) { + long total = values.nextValue() + values.nextValue(); + return (total >>> 1) + (total & 1); + } else { + return values.nextValue(); + } + } + + final long[] docValues = new long[count]; + docValues[0] = firstValue; + int firstPositiveIndex = 0; + for (int i = 1; i < count; ++i) { + docValues[i] = values.nextValue(); + if (docValues[i] >= 0 && firstPositiveIndex == 0) { + firstPositiveIndex = i; + } + } + final int mid = ((count - 1) / 2 + firstPositiveIndex) % count; + if (count % 2 == 0) { + long total = docValues[mid] + docValues[(mid + 1) % count]; + return (total >>> 1) + (total & 1); + } else { + return docValues[mid]; + } + } + }, + + /** + * Pick the lowest value. + */ + MIN { + @Override + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + final long min = values.nextValue(); + if (count == 1 || min > 0) { + return min; + } + for (int i = 1; i < count; ++i) { + long val = values.nextValue(); + if (val >= 0) { + return val; + } + } + return min; + } + + @Override + protected long pick( + SortedNumericDocValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + boolean hasValue = false; + long minValue = Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + final long docMin = pick(values); + minValue = Long.compareUnsigned(docMin, minValue) < 0 ? docMin : minValue; + hasValue = true; + } + } + return hasValue ? minValue : missingValue; + } + }, + + /** + * Pick the highest value. + */ + MAX { + @Override + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + long max = values.nextValue(); + long val; + for (int i = 1; i < count; ++i) { + val = values.nextValue(); + if (max < 0 && val >= 0) { + return max; + } + max = val; + } + return max; + } + + @Override + protected long pick( + SortedNumericDocValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + boolean hasValue = false; + long maxValue = Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + final long docMax = pick(values); + maxValue = Long.compareUnsigned(maxValue, docMax) < 0 ? docMax : maxValue; + hasValue = true; + } + } + return hasValue ? maxValue : missingValue; + } + }; + + /** + * A case insensitive version of {@link #valueOf(String)} + * + * @throws IllegalArgumentException if the given string doesn't match a sort mode or is null. + */ + private static UnsignedLongMultiValueMode fromString(String sortMode) { + try { + return valueOf(sortMode.toUpperCase(Locale.ROOT)); + } catch (Exception e) { + throw new IllegalArgumentException("Illegal sort mode: " + sortMode); + } + } + + /** + * Convert a {@link MultiValueMode} to a {@link UnsignedLongMultiValueMode}. + */ + public static UnsignedLongMultiValueMode toUnsignedSortMode(MultiValueMode sortMode) { + return fromString(sortMode.name()); + } + + /** + * Return a {@link NumericDocValues} instance that can be used to sort documents + * with this mode and the provided values. When a document has no value, + * missingValue is returned. + *

+ * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX + */ + public NumericDocValues select(final SortedNumericDocValues values) { + final NumericDocValues singleton = DocValues.unwrapSingleton(values); + if (singleton != null) { + return singleton; + } else { + return new AbstractNumericDocValues() { + + private long value; + + @Override + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + value = pick(values); + return true; + } + return false; + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public long longValue() throws IOException { + return value; + } + }; + } + } + + protected long pick(SortedNumericDocValues values) throws IOException { + throw new IllegalArgumentException("Unsupported sort mode: " + this); + } + + /** + * Return a {@link NumericDocValues} instance that can be used to sort root documents + * with this mode, the provided values and filters for root/inner documents. + *

+ * For every root document, the values of its inner documents will be aggregated. + * If none of the inner documents has a value, then missingValue is returned. + *

+ * Allowed Modes: SUM, AVG, MIN, MAX + *

+ * NOTE: Calling the returned instance on docs that are not root docs is illegal + * The returned instance can only be evaluate the current and upcoming docs + */ + public NumericDocValues select( + final SortedNumericDocValues values, + final long missingValue, + final BitSet parentDocs, + final DocIdSetIterator childDocs, + int maxDoc, + int maxChildren + ) throws IOException { + if (parentDocs == null || childDocs == null) { + return FieldData.replaceMissing(DocValues.emptyNumeric(), missingValue); + } + + return new AbstractNumericDocValues() { + + int lastSeenParentDoc = -1; + long lastEmittedValue = missingValue; + + @Override + public boolean advanceExact(int parentDoc) throws IOException { + assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs"; + if (parentDoc == lastSeenParentDoc) { + return true; + } else if (parentDoc == 0) { + lastEmittedValue = missingValue; + return true; + } + final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); + final int firstChildDoc; + if (childDocs.docID() > prevParentDoc) { + firstChildDoc = childDocs.docID(); + } else { + firstChildDoc = childDocs.advance(prevParentDoc + 1); + } + + lastSeenParentDoc = parentDoc; + lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren); + return true; + } + + @Override + public int docID() { + return lastSeenParentDoc; + } + + @Override + public long longValue() { + return lastEmittedValue; + } + }; + } + + protected long pick( + SortedNumericDocValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + throw new IllegalArgumentException("Unsupported sort mode: " + this); + } + + /** + * Copied from {@link Long#divideUnsigned(long, long)} and {@link Long#remainderUnsigned(long, long)} + */ + private static long divideUnsignedAndRoundUp(long dividend, long divisor) { + assert divisor > 0; + final long q = (dividend >>> 1) / divisor << 1; + final long r = dividend - q * divisor; + final long quotient = q + ((r | ~(r - divisor)) >>> (Long.SIZE - 1)); + final long rem = r - ((~(r - divisor) >> (Long.SIZE - 1)) & divisor); + return quotient + Math.round((double) rem / divisor); + } +} diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index 9db5817450cd0..c7b9c9f445484 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -41,6 +41,7 @@ public class UnsignedLongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource { private final IndexNumericFieldData indexFieldData; + private final UnsignedLongMultiValueMode unsignedSortMode; public UnsignedLongValuesComparatorSource( IndexNumericFieldData indexFieldData, @@ -50,6 +51,7 @@ public UnsignedLongValuesComparatorSource( ) { super(missingValue, sortMode, nested); this.indexFieldData = indexFieldData; + this.unsignedSortMode = UnsignedLongMultiValueMode.toUnsignedSortMode(sortMode); } @Override @@ -66,12 +68,12 @@ private SortedNumericDocValues loadDocValues(LeafReaderContext context) { private NumericDocValues getNumericDocValues(LeafReaderContext context, BigInteger missingValue) throws IOException { final SortedNumericDocValues values = loadDocValues(context); if (nested == null) { - return FieldData.replaceMissing(sortMode.select(values), missingValue); + return FieldData.replaceMissing(unsignedSortMode.select(values), missingValue); } final BitSet rootDocs = nested.rootDocs(context); final DocIdSetIterator innerDocs = nested.innerDocs(context); final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE; - return sortMode.select(values, missingValue.longValue(), rootDocs, innerDocs, context.reader().maxDoc(), maxChildren); + return unsignedSortMode.select(values, missingValue.longValue(), rootDocs, innerDocs, context.reader().maxDoc(), maxChildren); } @Override diff --git a/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java b/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java new file mode 100644 index 0000000000000..afa562862cfe9 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java @@ -0,0 +1,271 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.fielddata.fieldcomparator; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.common.Numbers; +import org.opensearch.index.fielddata.AbstractNumericDocValues; +import org.opensearch.index.fielddata.AbstractSortedNumericDocValues; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.util.Arrays; +import java.util.function.Supplier; + +public class UnsignedLongMultiValueModeTests extends OpenSearchTestCase { + private static FixedBitSet randomRootDocs(int maxDoc) { + FixedBitSet set = new FixedBitSet(maxDoc); + for (int i = 0; i < maxDoc; ++i) { + if (randomBoolean()) { + set.set(i); + } + } + // the last doc must be a root doc + set.set(maxDoc - 1); + return set; + } + + private static FixedBitSet randomInnerDocs(FixedBitSet rootDocs) { + FixedBitSet innerDocs = new FixedBitSet(rootDocs.length()); + for (int i = 0; i < innerDocs.length(); ++i) { + if (!rootDocs.get(i) && randomBoolean()) { + innerDocs.set(i); + } + } + return innerDocs; + } + + public void testSingleValuedLongs() throws Exception { + final int numDocs = scaledRandomIntBetween(1, 100); + final long[] array = new long[numDocs]; + final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); + for (int i = 0; i < array.length; ++i) { + if (randomBoolean()) { + array[i] = randomUnsignedLong().longValue(); + if (docsWithValue != null) { + docsWithValue.set(i); + } + } else if (docsWithValue != null && randomBoolean()) { + docsWithValue.set(i); + } + } + + final Supplier multiValues = () -> DocValues.singleton(new AbstractNumericDocValues() { + int docId = -1; + + @Override + public boolean advanceExact(int target) throws IOException { + this.docId = target; + return docsWithValue == null || docsWithValue.get(docId); + } + + @Override + public int docID() { + return docId; + } + + @Override + public long longValue() { + return array[docId]; + } + }); + verifySortedNumeric(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs)); + } + + public void testMultiValuedLongs() throws Exception { + final int numDocs = scaledRandomIntBetween(1, 100); + final long[][] array = new long[numDocs][]; + for (int i = 0; i < numDocs; ++i) { + final long[] values = new long[randomInt(4)]; + for (int j = 0; j < values.length; ++j) { + values[j] = randomUnsignedLong().longValue(); + } + Arrays.sort(values); + array[i] = values; + } + final Supplier multiValues = () -> new AbstractSortedNumericDocValues() { + int doc; + int i; + + @Override + public long nextValue() { + return array[doc][i++]; + } + + @Override + public boolean advanceExact(int doc) { + this.doc = doc; + i = 0; + return array[doc].length > 0; + } + + @Override + public int docValueCount() { + return array[doc].length; + } + }; + verifySortedNumeric(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs)); + } + + private void verifySortedNumeric(Supplier supplier, int maxDoc) throws IOException { + for (UnsignedLongMultiValueMode mode : UnsignedLongMultiValueMode.values()) { + SortedNumericDocValues values = supplier.get(); + final NumericDocValues selected = mode.select(values); + for (int i = 0; i < maxDoc; ++i) { + Long actual = null; + if (selected.advanceExact(i)) { + actual = selected.longValue(); + verifyLongValueCanCalledMoreThanOnce(selected, actual); + } + + BigInteger expected = null; + if (values.advanceExact(i)) { + int numValues = values.docValueCount(); + if (mode == UnsignedLongMultiValueMode.MAX) { + expected = Numbers.MIN_UNSIGNED_LONG_VALUE; + } else if (mode == UnsignedLongMultiValueMode.MIN) { + expected = Numbers.MAX_UNSIGNED_LONG_VALUE; + } else { + expected = BigInteger.ZERO; + } + for (int j = 0; j < numValues; ++j) { + if (mode == UnsignedLongMultiValueMode.SUM || mode == UnsignedLongMultiValueMode.AVG) { + expected = expected.add(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == UnsignedLongMultiValueMode.MIN) { + expected = expected.min(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == UnsignedLongMultiValueMode.MAX) { + expected = expected.max(Numbers.toUnsignedBigInteger(values.nextValue())); + } + } + if (mode == UnsignedLongMultiValueMode.AVG) { + expected = Numbers.toUnsignedBigInteger(expected.longValue()); + expected = numValues > 1 + ? new BigDecimal(expected).divide(new BigDecimal(numValues), RoundingMode.HALF_UP).toBigInteger() + : expected; + } else if (mode == UnsignedLongMultiValueMode.MEDIAN) { + final Long[] docValues = new Long[numValues]; + for (int j = 0; j < numValues; ++j) { + docValues[j] = values.nextValue(); + } + Arrays.sort(docValues, Long::compareUnsigned); + int value = numValues / 2; + if (numValues % 2 == 0) { + expected = Numbers.toUnsignedBigInteger(docValues[value - 1]) + .add(Numbers.toUnsignedBigInteger(docValues[value])); + expected = Numbers.toUnsignedBigInteger(expected.longValue()); + expected = new BigDecimal(expected).divide(new BigDecimal(2), RoundingMode.HALF_UP).toBigInteger(); + } else { + expected = Numbers.toUnsignedBigInteger(docValues[value]); + } + } + } + + final Long expectedLong = expected == null ? null : expected.longValue(); + assertEquals(mode.toString() + " docId=" + i, expectedLong, actual); + } + } + } + + private void verifyLongValueCanCalledMoreThanOnce(NumericDocValues values, long expected) throws IOException { + for (int j = 0, numCall = randomIntBetween(1, 10); j < numCall; j++) { + assertEquals(expected, values.longValue()); + } + } + + private void verifySortedNumeric( + Supplier supplier, + int maxDoc, + FixedBitSet rootDocs, + FixedBitSet innerDocs, + int maxChildren + ) throws IOException { + for (long missingValue : new long[] { 0, randomUnsignedLong().longValue() }) { + for (UnsignedLongMultiValueMode mode : new UnsignedLongMultiValueMode[] { + UnsignedLongMultiValueMode.MIN, + UnsignedLongMultiValueMode.MAX, + UnsignedLongMultiValueMode.SUM, + UnsignedLongMultiValueMode.AVG }) { + SortedNumericDocValues values = supplier.get(); + final NumericDocValues selected = mode.select( + values, + missingValue, + rootDocs, + new BitSetIterator(innerDocs, 0L), + maxDoc, + maxChildren + ); + int prevRoot = -1; + for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { + assertTrue(selected.advanceExact(root)); + final long actual = selected.longValue(); + verifyLongValueCanCalledMoreThanOnce(selected, actual); + + BigInteger expected = BigInteger.ZERO; + if (mode == UnsignedLongMultiValueMode.MAX) { + expected = Numbers.MIN_UNSIGNED_LONG_VALUE; + } else if (mode == UnsignedLongMultiValueMode.MIN) { + expected = Numbers.MAX_UNSIGNED_LONG_VALUE; + } + int numValues = 0; + int count = 0; + for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit( + child + 1 + )) { + if (values.advanceExact(child)) { + if (++count > maxChildren) { + break; + } + for (int j = 0; j < values.docValueCount(); ++j) { + if (mode == UnsignedLongMultiValueMode.SUM || mode == UnsignedLongMultiValueMode.AVG) { + expected = expected.add(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == UnsignedLongMultiValueMode.MIN) { + expected = expected.min(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == UnsignedLongMultiValueMode.MAX) { + expected = expected.max(Numbers.toUnsignedBigInteger(values.nextValue())); + } + ++numValues; + } + } + } + final long expectedLong; + if (numValues == 0) { + expectedLong = missingValue; + } else if (mode == UnsignedLongMultiValueMode.AVG) { + expected = Numbers.toUnsignedBigInteger(expected.longValue()); + expected = numValues > 1 + ? new BigDecimal(expected).divide(new BigDecimal(numValues), RoundingMode.HALF_UP).toBigInteger() + : expected; + expectedLong = expected.longValue(); + } else { + expectedLong = expected.longValue(); + } + + assertEquals(mode.toString() + " docId=" + root, expectedLong, actual); + + prevRoot = root; + } + } + } + } +} From b1f8f7ab10c70b5c83fc7d1dccc179a7740525ae Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 2 Dec 2024 13:17:29 -0500 Subject: [PATCH 02/10] Add initial rest-api-spec tests Signed-off-by: Andriy Redko --- .../test/search/280_unsigned_long.yml | 61 +++++++++++++++ .../rest-api-spec/test/search/290_long.yml | 74 +++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml new file mode 100644 index 0000000000000..4dafae31f23aa --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml @@ -0,0 +1,61 @@ +setup: + - do: + indices.create: + index: unsigned_long_sort + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + field: + type: unsigned_long + +--- +"test sorting against unsigned_long only fields": + - skip: + version: " - 2.19.99" + reason: "this change is added in 3.0.0" + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "1" } }' + - '{"field" : [ 13835058055282163712, 1 ] }' + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "2" } }' + - '{"field" : [ 13835058055282163713, 2 ] }' + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "3" } }' + - '{"field" : [ 13835058055282163714, 3 ] }' + + - do: + search: + index: unsigned_long_sort + body: + size: 5 + sort: [{ field: { mode: max, order: desc } } ] + - match: {hits.total.value: 3 } + - length: {hits.hits: 3 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163714, 3 ] } + - match: { hits.hits.0.sort: [ 13835058055282163714 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163713, 2 ] } + - match: { hits.hits.1.sort: [ 13835058055282163713 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.2.sort: [ 13835058055282163712 ] } + + - do: + search: + index: unsigned_long_sort + body: + size: 5 + sort: [{ field: { mode: max, order: asc } } ] + - match: {hits.total.value: 3 } + - length: {hits.hits: 3 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.0.sort: [ 13835058055282163712 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163713, 2 ] } + - match: { hits.hits.1.sort: [ 13835058055282163713 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163714, 3 ] } + - match: { hits.hits.2.sort: [ 13835058055282163714 ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml new file mode 100644 index 0000000000000..49e79cce88fce --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml @@ -0,0 +1,74 @@ +setup: + - do: + indices.create: + index: long_sort + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + field: + type: long + +--- +"test sorting against long only fields": + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "long_sort", "_id" : "1" } }' + - '{"field" : [ 55282163712, 1 ] }' + - '{ "index" : { "_index" : "long_sort", "_id" : "2" } }' + - '{"field" : [ 55282163713, 2 ] }' + - '{ "index" : { "_index" : "long_sort", "_id" : "3" } }' + - '{"field" : [ 55282163714, 3 ] }' + + - do: + search: + index: long_sort + body: + size: 5 + sort: [{ field: { mode: max, order: desc } } ] + - match: {hits.total.value: 3 } + - length: {hits.hits: 3 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 55282163714, 3 ] } + - match: { hits.hits.0.sort: [ 55282163714 ] } + - match: { hits.hits.1._source.field: [ 55282163713, 2 ] } + - match: { hits.hits.1.sort: [ 55282163713 ] } + - match: { hits.hits.2._source.field: [ 55282163712, 1 ] } + - match: { hits.hits.2.sort: [ 55282163712 ] } + + - do: + search: + index: long_sort + body: + size: 5 + sort: [{ field: { mode: min, order: desc } } ] + - match: {hits.total.value: 3 } + - length: {hits.hits: 3 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 55282163714, 3 ] } + - match: { hits.hits.0.sort: [ 3 ] } + - match: { hits.hits.1._source.field: [ 55282163713, 2 ] } + - match: { hits.hits.1.sort: [ 2 ] } + - match: { hits.hits.2._source.field: [ 55282163712, 1 ] } + - match: { hits.hits.2.sort: [ 1 ] } + + - do: + search: + index: long_sort + body: + size: 5 + sort: [{ field: { mode: max, order: asc } } ] + - match: {hits.total.value: 3 } + - length: {hits.hits: 3 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 55282163712, 1 ] } + - match: { hits.hits.0.sort: [ 55282163712 ] } + - match: { hits.hits.1._source.field: [ 55282163713, 2 ] } + - match: { hits.hits.1.sort: [ 55282163713 ] } + - match: { hits.hits.2._source.field: [ 55282163714, 3 ] } + - match: { hits.hits.2.sort: [ 55282163714 ] } From 17940b9b23098e5cb4272da9da931788414df2d7 Mon Sep 17 00:00:00 2001 From: panguixin Date: Thu, 12 Dec 2024 19:01:37 +0800 Subject: [PATCH 03/10] add more rest tests Signed-off-by: panguixin --- .../test/search/260_sort_double.yml | 136 ++++++++++++++ .../test/search/260_sort_long.yml | 137 ++++++++++++++ .../test/search/260_sort_unsigned_long.yml | 167 ++++++++++++++++++ .../test/search/280_unsigned_long.yml | 61 ------- .../rest-api-spec/test/search/290_long.yml | 74 -------- .../rest/yaml/section/MatchAssertion.java | 20 +++ 6 files changed, 460 insertions(+), 135 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml new file mode 100644 index 0000000000000..654d3f6c58a61 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml @@ -0,0 +1,136 @@ +setup: + - do: + indices.create: + index: double_sort + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + field: + type: double + +--- +"test sorting against double only fields": + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "double_sort", "_id" : "1" } }' + - '{"field" : [ 900719925474099.1, 1.1 ] }' + - '{ "index" : { "_index" : "double_sort", "_id" : "2" } }' + - '{"field" : [ 900719925474099.2, 900719925474099.3 ] }' + - '{ "index" : { "_index" : "double_sort", "_id" : "3" } }' + - '{"field" : [ 450359962737049.4, 3.5, 4.6 ] }' + - '{ "index" : { "_index" : "double_sort", "_id" : "4" } }' + - '{"field" : [ 450359962737049.7, 5.8, -1.9, -2.0 ] }' + + - do: + search: + index: double_sort + body: + size: 5 + sort: [{ field: { mode: max, order: desc } } ] + - match: {hits.total.value: 4 } + - length: {hits.hits: 4 } + - match: { hits.hits.0._index: double_sort } + - match: { hits.hits.0._source.field: [ 900719925474099.2, 900719925474099.2 ] } + - match: { hits.hits.0.sort: [ 900719925474099.2 ] } + - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } + - match: { hits.hits.1.sort: [ 900719925474099.1 ] } + - match: { hits.hits.2._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } + - match: { hits.hits.2.sort: [ 450359962737049.7 ] } + - match: { hits.hits.3._source.field: [ 450359962737049.4, 3.5, 4.6 ] } + - match: { hits.hits.3.sort: [ 450359962737049.4 ] } + + - do: + search: + index: double_sort + body: + size: 5 + sort: [ { field: { mode: max, order: asc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: double_sort } + - match: { hits.hits.0._source.field: [ 450359962737049.4, 3.5, 4.6 ] } + - match: { hits.hits.0.sort: [ 450359962737049.4 ] } + - match: { hits.hits.1._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } + - match: { hits.hits.1.sort: [ 450359962737049.7 ] } + - match: { hits.hits.2._source.field: [ 900719925474099.1, 1.1 ] } + - match: { hits.hits.2.sort: [ 900719925474099.1 ] } + - match: { hits.hits.3._source.field: [ 900719925474099.2, 900719925474099.2 ] } + - match: { hits.hits.3.sort: [ 900719925474099.2 ] } + + - do: + search: + index: double_sort + body: + size: 5 + sort: [ { field: { mode: min, order: asc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: double_sort } + - match: { hits.hits.0._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } + - match: { hits.hits.0.sort: [ -2.0 ] } + - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } + - match: { hits.hits.1.sort: [ 1.1 ] } + - match: { hits.hits.2._source.field: [ 450359962737049.4, 3.5, 4.6 ] } + - match: { hits.hits.2.sort: [ 3.5 ] } + - match: { hits.hits.3._source.field: [ 900719925474099.2, 900719925474099.2 ] } + - match: { hits.hits.3.sort: [ 900719925474099.2 ] } + + - do: + search: + index: double_sort + body: + size: 5 + sort: [ { field: { mode: median, order: desc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: double_sort } + - match: { hits.hits.0._source.field: [ 900719925474099.2, 900719925474099.2 ] } + - match: { hits.hits.0.sort: [ 900719925474099.2 ] } + - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } + - match: { hits.hits.1.sort: [ 450359962737050.1 ] } + - match: { hits.hits.2._source.field: [ 450359962737049.4, 3.5, 4.6 ] } + - match: { hits.hits.2.sort: [ 4.6 ] } + - match: { hits.hits.3._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } + - match: { hits.hits.3.sort: [ 1.95 ] } + + - do: + search: + index: double_sort + body: + size: 5 + sort: [ { field: { mode: avg, order: asc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: double_sort } + - match: { hits.hits.0._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } + - match: { hits.hits.0.sort: [ 112589990684262.89 ] } + - match: { hits.hits.1._source.field: [ 450359962737049.4, 3.5, 4.6 ] } + - match: { hits.hits.1.sort: [ 150119987579019.16 ] } + - match: { hits.hits.2._source.field: [ 900719925474099.1, 1.1 ] } + - match: { hits.hits.2.sort: [ 450359962737050.1 ] } + - match: { hits.hits.3._source.field: [ 900719925474099.2, 900719925474099.2 ] } + - match: { hits.hits.3.sort: [ 900719925474099.2 ] } + + - do: + search: + index: double_sort + body: + size: 5 + sort: [ { field: { mode: sum, order: desc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: double_sort } + - match: { hits.hits.0._source.field: [ 900719925474099.2, 900719925474099.2 ] } + - match: { hits.hits.0.sort: [ 1801439850948198.5 ] } + - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } + - match: { hits.hits.1.sort: [ 900719925474100.2 ] } + - match: { hits.hits.2._source.field: [ 450359962737049.4, 3.5, 4.6 ] } + - match: { hits.hits.2.sort: [ 450359962737057.5 ] } + - match: { hits.hits.3._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } + - match: { hits.hits.3.sort: [ 450359962737051.56 ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml new file mode 100644 index 0000000000000..b71e38bccd40c --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml @@ -0,0 +1,137 @@ +setup: + - do: + indices.create: + index: long_sort + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + field: + type: long + +--- +"test sorting against long only fields": + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "long_sort", "_id" : "1" } }' + - '{"field" : [ 9223372036854775807, 1 ] }' + - '{ "index" : { "_index" : "long_sort", "_id" : "2" } }' + - '{"field" : [ 922337203685477777, 2 ] }' + - '{ "index" : { "_index" : "long_sort", "_id" : "3" } }' + - '{"field" : [ 2147483647, 3, 4 ] }' + - '{ "index" : { "_index" : "long_sort", "_id" : "4" } }' + - '{"field" : [ 2147483648, 5, -1, -2 ] }' + + - do: + search: + index: long_sort + body: + size: 5 + sort: [{ field: { mode: max, order: desc } } ] + - match: {hits.total.value: 4 } + - length: {hits.hits: 4 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 9223372036854775807, 1 ] } + - match: { hits.hits.0.sort: [ 9223372036854775807 ] } + - match: { hits.hits.1._source.field: [ 922337203685477777, 2 ] } + - match: { hits.hits.1.sort: [ 922337203685477777 ] } + - match: { hits.hits.2._source.field: [ 2147483648, 5, -1, -2 ] } + - match: { hits.hits.2.sort: [ 2147483648 ] } + - match: { hits.hits.3._source.field: [ 2147483647, 3, 4 ] } + - match: { hits.hits.3.sort: [ 2147483647 ] } + + - do: + search: + index: long_sort + body: + size: 5 + sort: [ { field: { mode: max, order: asc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 2147483647, 3, 4 ] } + - match: { hits.hits.0.sort: [ 2147483647 ] } + - match: { hits.hits.1._source.field: [ 2147483648, 5, -1, -2 ] } + - match: { hits.hits.1.sort: [ 2147483648 ] } + - match: { hits.hits.2._source.field: [ 922337203685477777, 2 ] } + - match: { hits.hits.2.sort: [ 922337203685477777 ] } + - match: { hits.hits.3._source.field: [ 9223372036854775807, 1 ] } + - match: { hits.hits.3.sort: [ 9223372036854775807 ] } + + + - do: + search: + index: long_sort + body: + size: 5 + sort: [{ field: { mode: min, order: desc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 2147483647, 3, 4 ] } + - match: { hits.hits.0.sort: [ 3 ] } + - match: { hits.hits.1._source.field: [ 922337203685477777, 2 ] } + - match: { hits.hits.1.sort: [ 2 ] } + - match: { hits.hits.2._source.field: [ 9223372036854775807, 1 ] } + - match: { hits.hits.2.sort: [ 1 ] } + - match: { hits.hits.3._source.field: [ 2147483648, 5, -1, -2 ] } + - match: { hits.hits.3.sort: [ -2 ] } + + - do: + search: + index: long_sort + body: + size: 5 + sort: [ { field: { mode: median, order: asc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 2147483648, 5, -1, -2 ] } + - match: { hits.hits.0.sort: [ 2 ] } + - match: { hits.hits.1._source.field: [ 2147483647, 3, 4 ] } + - match: { hits.hits.1.sort: [ 4 ] } + - match: { hits.hits.2._source.field: [ 922337203685477777, 2 ] } + - match: { hits.hits.2.sort: [ 461168601842738880 ] } + - match: { hits.hits.3._source.field: [ 9223372036854775807, 1 ] } + - match: { hits.hits.3.sort: [ 4611686018427387904 ] } + + - do: + search: + index: long_sort + body: + size: 5 + sort: [ { field: { mode: avg, order: desc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 922337203685477777, 2 ] } + - match: { hits.hits.0.sort: [ 461168601842738880 ] } + - match: { hits.hits.1._source.field: [ 2147483647, 3, 4 ] } + - match: { hits.hits.1.sort: [ 715827885 ] } + - match: { hits.hits.2._source.field: [ 2147483648, 5, -1, -2 ] } + - match: { hits.hits.2.sort: [ 536870913 ] } + - match: { hits.hits.3._source.field: [ 9223372036854775807, 1 ] } + - match: { hits.hits.3.sort: [ -4611686018427387904 ] } + + - do: + search: + index: long_sort + body: + size: 5 + sort: [ { field: { mode: sum, order: asc } } ] + - match: { hits.total.value: 4 } + - length: { hits.hits: 4 } + - match: { hits.hits.0._index: long_sort } + - match: { hits.hits.0._source.field: [ 9223372036854775807, 1 ] } + - match: { hits.hits.0.sort: [ -9223372036854775808 ] } + - match: { hits.hits.1._source.field: [ 2147483648, 5, -1, -2 ] } + - match: { hits.hits.1.sort: [ 2147483650 ] } + - match: { hits.hits.2._source.field: [ 2147483647, 3, 4 ] } + - match: { hits.hits.2.sort: [ 2147483654 ] } + - match: { hits.hits.3._source.field: [ 922337203685477777, 2 ] } + - match: { hits.hits.3.sort: [ 922337203685477779 ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml new file mode 100644 index 0000000000000..fb3b1f994fd8c --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml @@ -0,0 +1,167 @@ +setup: + - do: + indices.create: + index: unsigned_long_sort + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + field: + type: unsigned_long + +--- +"test sorting against unsigned_long only fields": + - skip: + version: " - 2.19.99" + reason: "this change is added in 3.0.0" + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "1" } }' + - '{"field" : [ 13835058055282163712, 1 ] }' + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "2" } }' + - '{"field" : [ 13835058055282163713, 13835058055282163714 ] }' + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "3" } }' + - '{"field" : [ 13835058055282163715, 13835058055282163716, 2 ] }' + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "4" } }' + - '{"field" : [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] }' + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "5" } }' + - '{"field" : [ 13835058055282163720, 13835058055282163721, 3, 4 ] }' + - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "6" } }' + - '{"field" : [ 13835058055282163722, 5, 6, 7 ] }' + + - do: + search: + index: unsigned_long_sort + body: + size: 10 + sort: [{ field: { mode: max, order: desc } } ] + - match: {hits.total.value: 6 } + - length: {hits.hits: 6 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163722, 5, 6, 7 ] } + - match: { hits.hits.0.sort: [ 13835058055282163722 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } + - match: { hits.hits.1.sort: [ 13835058055282163721 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } + - match: { hits.hits.2.sort: [ 13835058055282163719 ] } + - match: { hits.hits.3._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } + - match: { hits.hits.3.sort: [ 13835058055282163716 ] } + - match: { hits.hits.4._source.field: [ 13835058055282163713, 13835058055282163714 ] } + - match: { hits.hits.4.sort: [ 13835058055282163714 ] } + - match: { hits.hits.5._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.5.sort: [ 13835058055282163712 ] } + + - do: + search: + index: unsigned_long_sort + body: + size: 10 + sort: [{ field: { mode: max, order: asc } } ] + - match: {hits.total.value: 6 } + - length: {hits.hits: 6 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.0.sort: [ 13835058055282163712 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163713, 13835058055282163714 ] } + - match: { hits.hits.1.sort: [ 13835058055282163714 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } + - match: { hits.hits.2.sort: [ 13835058055282163716 ] } + - match: { hits.hits.3._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } + - match: { hits.hits.3.sort: [ 13835058055282163719 ] } + - match: { hits.hits.4._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } + - match: { hits.hits.4.sort: [ 13835058055282163721 ] } + - match: { hits.hits.5._source.field: [ 13835058055282163722, 5, 6, 7 ] } + - match: { hits.hits.5.sort: [ 13835058055282163722 ] } + + - do: + search: + index: unsigned_long_sort + body: + size: 10 + sort: [ { field: { mode: median, order: asc } } ] + - match: { hits.total.value: 6 } + - length: { hits.hits: 6 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163722, 5, 6, 7 ] } + - match: { hits.hits.0.sort: [ 7 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163713, 13835058055282163714 ] } + - match: { hits.hits.1.sort: [ 4611686018427387906 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.2.sort: [ 6917529027641081857 ] } + - match: { hits.hits.3._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } + - match: { hits.hits.3.sort: [ 6917529027641081862 ] } + - match: { hits.hits.4._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } + - match: { hits.hits.4.sort: [ 13835058055282163715 ] } + - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } + - match: { hits.hits.5.sort: [ 13835058055282163718 ] } + + - do: + search: + index: unsigned_long_sort + body: + size: 10 + sort: [ { field: { mode: sum, order: desc } } ] + - match: { hits.total.value: 6 } + - length: { hits.hits: 6 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163722, 5, 6, 7 ] } + - match: { hits.hits.0.sort: [ 13835058055282163740 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.1.sort: [ 13835058055282163713 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } + - match: { hits.hits.2.sort: [ 9223372036854775832 ] } + - match: { hits.hits.3._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } + - match: { hits.hits.3.sort: [ 9223372036854775817 ] } + - match: { hits.hits.4._source.field: [ 13835058055282163713, 13835058055282163714 ] } + - match: { hits.hits.4.sort: [ 9223372036854775811 ] } + - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } + - match: { hits.hits.5.sort: [ 4611686018427387922 ] } + + - do: + search: + index: unsigned_long_sort + body: + size: 10 + sort: [ { field: { mode: avg, order: desc } } ] + - match: { hits.total.value: 6 } + - length: { hits.hits: 6 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.0.sort: [ 6917529027641081857 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163713, 13835058055282163714 ] } + - match: { hits.hits.1.sort: [ 4611686018427387906 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163722, 5, 6, 7 ] } + - match: { hits.hits.2.sort: [ 3458764513820540935 ] } + - match: { hits.hits.3._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } + - match: { hits.hits.3.sort: [ 3074457345618258606 ] } + - match: { hits.hits.4._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } + - match: { hits.hits.4.sort: [ 2305843009213693958 ] } + - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } + - match: { hits.hits.5.sort: [ 1537228672809129307 ] } + + - do: + search: + index: unsigned_long_sort + body: + size: 10 + sort: [ { field: { mode: min, order: asc } } ] + - match: { hits.total.value: 6 } + - length: { hits.hits: 6 } + - match: { hits.hits.0._index: unsigned_long_sort } + - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } + - match: { hits.hits.0.sort: [ 1 ] } + - match: { hits.hits.1._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } + - match: { hits.hits.1.sort: [ 2 ] } + - match: { hits.hits.2._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } + - match: { hits.hits.2.sort: [ 3 ] } + - match: { hits.hits.3._source.field: [ 13835058055282163722, 5, 6, 7 ] } + - match: { hits.hits.3.sort: [ 5 ] } + - match: { hits.hits.4._source.field: [ 13835058055282163713, 13835058055282163714 ] } + - match: { hits.hits.4.sort: [ 13835058055282163713 ] } + - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } + - match: { hits.hits.5.sort: [ 13835058055282163717 ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml deleted file mode 100644 index 4dafae31f23aa..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/280_unsigned_long.yml +++ /dev/null @@ -1,61 +0,0 @@ -setup: - - do: - indices.create: - index: unsigned_long_sort - body: - settings: - number_of_shards: 3 - number_of_replicas: 0 - mappings: - properties: - field: - type: unsigned_long - ---- -"test sorting against unsigned_long only fields": - - skip: - version: " - 2.19.99" - reason: "this change is added in 3.0.0" - - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "1" } }' - - '{"field" : [ 13835058055282163712, 1 ] }' - - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "2" } }' - - '{"field" : [ 13835058055282163713, 2 ] }' - - '{ "index" : { "_index" : "unsigned_long_sort", "_id" : "3" } }' - - '{"field" : [ 13835058055282163714, 3 ] }' - - - do: - search: - index: unsigned_long_sort - body: - size: 5 - sort: [{ field: { mode: max, order: desc } } ] - - match: {hits.total.value: 3 } - - length: {hits.hits: 3 } - - match: { hits.hits.0._index: unsigned_long_sort } - - match: { hits.hits.0._source.field: [ 13835058055282163714, 3 ] } - - match: { hits.hits.0.sort: [ 13835058055282163714 ] } - - match: { hits.hits.1._source.field: [ 13835058055282163713, 2 ] } - - match: { hits.hits.1.sort: [ 13835058055282163713 ] } - - match: { hits.hits.2._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.2.sort: [ 13835058055282163712 ] } - - - do: - search: - index: unsigned_long_sort - body: - size: 5 - sort: [{ field: { mode: max, order: asc } } ] - - match: {hits.total.value: 3 } - - length: {hits.hits: 3 } - - match: { hits.hits.0._index: unsigned_long_sort } - - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.0.sort: [ 13835058055282163712 ] } - - match: { hits.hits.1._source.field: [ 13835058055282163713, 2 ] } - - match: { hits.hits.1.sort: [ 13835058055282163713 ] } - - match: { hits.hits.2._source.field: [ 13835058055282163714, 3 ] } - - match: { hits.hits.2.sort: [ 13835058055282163714 ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml deleted file mode 100644 index 49e79cce88fce..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/290_long.yml +++ /dev/null @@ -1,74 +0,0 @@ -setup: - - do: - indices.create: - index: long_sort - body: - settings: - number_of_shards: 3 - number_of_replicas: 0 - mappings: - properties: - field: - type: long - ---- -"test sorting against long only fields": - - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "long_sort", "_id" : "1" } }' - - '{"field" : [ 55282163712, 1 ] }' - - '{ "index" : { "_index" : "long_sort", "_id" : "2" } }' - - '{"field" : [ 55282163713, 2 ] }' - - '{ "index" : { "_index" : "long_sort", "_id" : "3" } }' - - '{"field" : [ 55282163714, 3 ] }' - - - do: - search: - index: long_sort - body: - size: 5 - sort: [{ field: { mode: max, order: desc } } ] - - match: {hits.total.value: 3 } - - length: {hits.hits: 3 } - - match: { hits.hits.0._index: long_sort } - - match: { hits.hits.0._source.field: [ 55282163714, 3 ] } - - match: { hits.hits.0.sort: [ 55282163714 ] } - - match: { hits.hits.1._source.field: [ 55282163713, 2 ] } - - match: { hits.hits.1.sort: [ 55282163713 ] } - - match: { hits.hits.2._source.field: [ 55282163712, 1 ] } - - match: { hits.hits.2.sort: [ 55282163712 ] } - - - do: - search: - index: long_sort - body: - size: 5 - sort: [{ field: { mode: min, order: desc } } ] - - match: {hits.total.value: 3 } - - length: {hits.hits: 3 } - - match: { hits.hits.0._index: long_sort } - - match: { hits.hits.0._source.field: [ 55282163714, 3 ] } - - match: { hits.hits.0.sort: [ 3 ] } - - match: { hits.hits.1._source.field: [ 55282163713, 2 ] } - - match: { hits.hits.1.sort: [ 2 ] } - - match: { hits.hits.2._source.field: [ 55282163712, 1 ] } - - match: { hits.hits.2.sort: [ 1 ] } - - - do: - search: - index: long_sort - body: - size: 5 - sort: [{ field: { mode: max, order: asc } } ] - - match: {hits.total.value: 3 } - - length: {hits.hits: 3 } - - match: { hits.hits.0._index: long_sort } - - match: { hits.hits.0._source.field: [ 55282163712, 1 ] } - - match: { hits.hits.0.sort: [ 55282163712 ] } - - match: { hits.hits.1._source.field: [ 55282163713, 2 ] } - - match: { hits.hits.1.sort: [ 55282163713 ] } - - match: { hits.hits.2._source.field: [ 55282163714, 3 ] } - - match: { hits.hits.2.sort: [ 55282163714 ] } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java index 77d8f3154729e..606689d89ddb6 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java @@ -40,6 +40,7 @@ import org.opensearch.test.hamcrest.RegexMatcher; import java.io.IOException; +import java.util.ArrayList; import java.util.regex.Pattern; import static org.hamcrest.Matchers.equalTo; @@ -110,6 +111,25 @@ protected void doAssert(Object actualValue, Object expectedValue) { } if (expectedValue.equals(actualValue) == false) { + if (expectedValue instanceof ArrayList + && actualValue instanceof ArrayList + && ((ArrayList) expectedValue).get(0) instanceof Number) { + ArrayList expectedList = (ArrayList) expectedValue; + ArrayList actualList = (ArrayList) actualValue; + if (expectedList.size() == 1 + && actualList.size() == 1 + && expectedList.get(0) instanceof Number + && actualList.get(0) instanceof Number) { + // BigInteger 1 is equal to Integer 1 + assertThat( + "field [" + getField() + "] doesn't match the expected value", + ((Number) actualList.get(0)).longValue(), + equalTo(((Number) expectedList.get(0)).longValue()) + ); + return; + } + } + NotEqualMessageBuilder message = new NotEqualMessageBuilder(); message.compare(getField(), true, actualValue, expectedValue); throw new AssertionError(getField() + " didn't match expected value:\n" + message); From 6550f602b0fa071e88706f02f67c834279211bc3 Mon Sep 17 00:00:00 2001 From: panguixin Date: Thu, 12 Dec 2024 21:07:15 +0800 Subject: [PATCH 04/10] fix Signed-off-by: panguixin --- .../rest/yaml/section/MatchAssertion.java | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java index 606689d89ddb6..b46ea23f8cd77 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java @@ -111,22 +111,25 @@ protected void doAssert(Object actualValue, Object expectedValue) { } if (expectedValue.equals(actualValue) == false) { - if (expectedValue instanceof ArrayList - && actualValue instanceof ArrayList - && ((ArrayList) expectedValue).get(0) instanceof Number) { - ArrayList expectedList = (ArrayList) expectedValue; - ArrayList actualList = (ArrayList) actualValue; - if (expectedList.size() == 1 - && actualList.size() == 1 - && expectedList.get(0) instanceof Number - && actualList.get(0) instanceof Number) { - // BigInteger 1 is equal to Integer 1 - assertThat( - "field [" + getField() + "] doesn't match the expected value", - ((Number) actualList.get(0)).longValue(), - equalTo(((Number) expectedList.get(0)).longValue()) - ); - return; + if (expectedValue instanceof ArrayList && actualValue instanceof ArrayList) { + ArrayList expectedList = (ArrayList) expectedValue; + ArrayList actualList = (ArrayList) actualValue; + if (expectedList.size() == actualList.size()) { + boolean pass = true; + for (int i = 0; i < expectedList.size(); i++) { + // BigInteger 1 is equal to Integer 1 + Object expected = expectedList.get(i); + Object actual = actualList.get(i); + if (expected instanceof Number == false + || actual instanceof Number == false + || ((Number) expected).longValue() != ((Number) actual).longValue()) { + pass = false; + break; + } + } + if (pass) { + return; + } } } From 92843d04b2d569b616e1ea6ade9d3de21e6337b0 Mon Sep 17 00:00:00 2001 From: panguixin Date: Mon, 16 Dec 2024 20:49:55 +0800 Subject: [PATCH 05/10] fix Signed-off-by: panguixin --- .../test/search/260_sort_double.yml | 46 ++++++------ .../test/search/260_sort_long.yml | 42 +++++------ .../test/search/260_sort_unsigned_long.yml | 72 +++++++++---------- .../rest/yaml/section/MatchAssertion.java | 23 ------ 4 files changed, 80 insertions(+), 103 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml index 654d3f6c58a61..eccafaf96dd23 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_double.yml @@ -37,13 +37,13 @@ setup: - length: {hits.hits: 4 } - match: { hits.hits.0._index: double_sort } - match: { hits.hits.0._source.field: [ 900719925474099.2, 900719925474099.2 ] } - - match: { hits.hits.0.sort: [ 900719925474099.2 ] } + - match: { hits.hits.0.sort.0: 900719925474099.2 } - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } - - match: { hits.hits.1.sort: [ 900719925474099.1 ] } + - match: { hits.hits.1.sort.0: 900719925474099.1 } - match: { hits.hits.2._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } - - match: { hits.hits.2.sort: [ 450359962737049.7 ] } + - match: { hits.hits.2.sort.0: 450359962737049.7 } - match: { hits.hits.3._source.field: [ 450359962737049.4, 3.5, 4.6 ] } - - match: { hits.hits.3.sort: [ 450359962737049.4 ] } + - match: { hits.hits.3.sort.0: 450359962737049.4 } - do: search: @@ -55,13 +55,13 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: double_sort } - match: { hits.hits.0._source.field: [ 450359962737049.4, 3.5, 4.6 ] } - - match: { hits.hits.0.sort: [ 450359962737049.4 ] } + - match: { hits.hits.0.sort.0: 450359962737049.4 } - match: { hits.hits.1._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } - - match: { hits.hits.1.sort: [ 450359962737049.7 ] } + - match: { hits.hits.1.sort.0: 450359962737049.7 } - match: { hits.hits.2._source.field: [ 900719925474099.1, 1.1 ] } - - match: { hits.hits.2.sort: [ 900719925474099.1 ] } + - match: { hits.hits.2.sort.0: 900719925474099.1 } - match: { hits.hits.3._source.field: [ 900719925474099.2, 900719925474099.2 ] } - - match: { hits.hits.3.sort: [ 900719925474099.2 ] } + - match: { hits.hits.3.sort.0: 900719925474099.2 } - do: search: @@ -75,11 +75,11 @@ setup: - match: { hits.hits.0._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } - match: { hits.hits.0.sort: [ -2.0 ] } - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } - - match: { hits.hits.1.sort: [ 1.1 ] } + - match: { hits.hits.1.sort.0: 1.1 } - match: { hits.hits.2._source.field: [ 450359962737049.4, 3.5, 4.6 ] } - - match: { hits.hits.2.sort: [ 3.5 ] } + - match: { hits.hits.2.sort.0: 3.5 } - match: { hits.hits.3._source.field: [ 900719925474099.2, 900719925474099.2 ] } - - match: { hits.hits.3.sort: [ 900719925474099.2 ] } + - match: { hits.hits.3.sort.0: 900719925474099.2 } - do: search: @@ -91,13 +91,13 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: double_sort } - match: { hits.hits.0._source.field: [ 900719925474099.2, 900719925474099.2 ] } - - match: { hits.hits.0.sort: [ 900719925474099.2 ] } + - match: { hits.hits.0.sort.0: 900719925474099.2 } - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } - - match: { hits.hits.1.sort: [ 450359962737050.1 ] } + - match: { hits.hits.1.sort.0: 450359962737050.1 } - match: { hits.hits.2._source.field: [ 450359962737049.4, 3.5, 4.6 ] } - - match: { hits.hits.2.sort: [ 4.6 ] } + - match: { hits.hits.2.sort.0: 4.6 } - match: { hits.hits.3._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } - - match: { hits.hits.3.sort: [ 1.95 ] } + - match: { hits.hits.3.sort.0: 1.95 } - do: search: @@ -109,13 +109,13 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: double_sort } - match: { hits.hits.0._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } - - match: { hits.hits.0.sort: [ 112589990684262.89 ] } + - match: { hits.hits.0.sort.0: 112589990684262.89 } - match: { hits.hits.1._source.field: [ 450359962737049.4, 3.5, 4.6 ] } - - match: { hits.hits.1.sort: [ 150119987579019.16 ] } + - match: { hits.hits.1.sort.0: 150119987579019.16 } - match: { hits.hits.2._source.field: [ 900719925474099.1, 1.1 ] } - - match: { hits.hits.2.sort: [ 450359962737050.1 ] } + - match: { hits.hits.2.sort.0: 450359962737050.1 } - match: { hits.hits.3._source.field: [ 900719925474099.2, 900719925474099.2 ] } - - match: { hits.hits.3.sort: [ 900719925474099.2 ] } + - match: { hits.hits.3.sort.0: 900719925474099.2 } - do: search: @@ -127,10 +127,10 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: double_sort } - match: { hits.hits.0._source.field: [ 900719925474099.2, 900719925474099.2 ] } - - match: { hits.hits.0.sort: [ 1801439850948198.5 ] } + - match: { hits.hits.0.sort.0: 1801439850948198.5 } - match: { hits.hits.1._source.field: [ 900719925474099.1, 1.1 ] } - - match: { hits.hits.1.sort: [ 900719925474100.2 ] } + - match: { hits.hits.1.sort.0: 900719925474100.2 } - match: { hits.hits.2._source.field: [ 450359962737049.4, 3.5, 4.6 ] } - - match: { hits.hits.2.sort: [ 450359962737057.5 ] } + - match: { hits.hits.2.sort.0: 450359962737057.5 } - match: { hits.hits.3._source.field: [ 450359962737049.7, 5.8, -1.9, -2.0 ] } - - match: { hits.hits.3.sort: [ 450359962737051.56 ] } + - match: { hits.hits.3.sort.0: 450359962737051.56 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml index b71e38bccd40c..f354dff6cbf02 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_long.yml @@ -37,13 +37,13 @@ setup: - length: {hits.hits: 4 } - match: { hits.hits.0._index: long_sort } - match: { hits.hits.0._source.field: [ 9223372036854775807, 1 ] } - - match: { hits.hits.0.sort: [ 9223372036854775807 ] } + - match: { hits.hits.0.sort.0: 9223372036854775807 } - match: { hits.hits.1._source.field: [ 922337203685477777, 2 ] } - - match: { hits.hits.1.sort: [ 922337203685477777 ] } + - match: { hits.hits.1.sort.0: 922337203685477777 } - match: { hits.hits.2._source.field: [ 2147483648, 5, -1, -2 ] } - - match: { hits.hits.2.sort: [ 2147483648 ] } + - match: { hits.hits.2.sort.0: 2147483648 } - match: { hits.hits.3._source.field: [ 2147483647, 3, 4 ] } - - match: { hits.hits.3.sort: [ 2147483647 ] } + - match: { hits.hits.3.sort.0: 2147483647 } - do: search: @@ -55,13 +55,13 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: long_sort } - match: { hits.hits.0._source.field: [ 2147483647, 3, 4 ] } - - match: { hits.hits.0.sort: [ 2147483647 ] } + - match: { hits.hits.0.sort.0: 2147483647 } - match: { hits.hits.1._source.field: [ 2147483648, 5, -1, -2 ] } - - match: { hits.hits.1.sort: [ 2147483648 ] } + - match: { hits.hits.1.sort.0: 2147483648 } - match: { hits.hits.2._source.field: [ 922337203685477777, 2 ] } - - match: { hits.hits.2.sort: [ 922337203685477777 ] } + - match: { hits.hits.2.sort.0: 922337203685477777 } - match: { hits.hits.3._source.field: [ 9223372036854775807, 1 ] } - - match: { hits.hits.3.sort: [ 9223372036854775807 ] } + - match: { hits.hits.3.sort.0: 9223372036854775807 } - do: @@ -74,11 +74,11 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: long_sort } - match: { hits.hits.0._source.field: [ 2147483647, 3, 4 ] } - - match: { hits.hits.0.sort: [ 3 ] } + - match: { hits.hits.0.sort.0: 3 } - match: { hits.hits.1._source.field: [ 922337203685477777, 2 ] } - - match: { hits.hits.1.sort: [ 2 ] } + - match: { hits.hits.1.sort.0: 2 } - match: { hits.hits.2._source.field: [ 9223372036854775807, 1 ] } - - match: { hits.hits.2.sort: [ 1 ] } + - match: { hits.hits.2.sort.0: 1 } - match: { hits.hits.3._source.field: [ 2147483648, 5, -1, -2 ] } - match: { hits.hits.3.sort: [ -2 ] } @@ -92,13 +92,13 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: long_sort } - match: { hits.hits.0._source.field: [ 2147483648, 5, -1, -2 ] } - - match: { hits.hits.0.sort: [ 2 ] } + - match: { hits.hits.0.sort.0: 2 } - match: { hits.hits.1._source.field: [ 2147483647, 3, 4 ] } - - match: { hits.hits.1.sort: [ 4 ] } + - match: { hits.hits.1.sort.0: 4 } - match: { hits.hits.2._source.field: [ 922337203685477777, 2 ] } - - match: { hits.hits.2.sort: [ 461168601842738880 ] } + - match: { hits.hits.2.sort.0: 461168601842738880 } - match: { hits.hits.3._source.field: [ 9223372036854775807, 1 ] } - - match: { hits.hits.3.sort: [ 4611686018427387904 ] } + - match: { hits.hits.3.sort.0: 4611686018427387904 } - do: search: @@ -110,11 +110,11 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._index: long_sort } - match: { hits.hits.0._source.field: [ 922337203685477777, 2 ] } - - match: { hits.hits.0.sort: [ 461168601842738880 ] } + - match: { hits.hits.0.sort.0: 461168601842738880 } - match: { hits.hits.1._source.field: [ 2147483647, 3, 4 ] } - - match: { hits.hits.1.sort: [ 715827885 ] } + - match: { hits.hits.1.sort.0: 715827885 } - match: { hits.hits.2._source.field: [ 2147483648, 5, -1, -2 ] } - - match: { hits.hits.2.sort: [ 536870913 ] } + - match: { hits.hits.2.sort.0: 536870913 } - match: { hits.hits.3._source.field: [ 9223372036854775807, 1 ] } - match: { hits.hits.3.sort: [ -4611686018427387904 ] } @@ -130,8 +130,8 @@ setup: - match: { hits.hits.0._source.field: [ 9223372036854775807, 1 ] } - match: { hits.hits.0.sort: [ -9223372036854775808 ] } - match: { hits.hits.1._source.field: [ 2147483648, 5, -1, -2 ] } - - match: { hits.hits.1.sort: [ 2147483650 ] } + - match: { hits.hits.1.sort.0: 2147483650 } - match: { hits.hits.2._source.field: [ 2147483647, 3, 4 ] } - - match: { hits.hits.2.sort: [ 2147483654 ] } + - match: { hits.hits.2.sort.0: 2147483654 } - match: { hits.hits.3._source.field: [ 922337203685477777, 2 ] } - - match: { hits.hits.3.sort: [ 922337203685477779 ] } + - match: { hits.hits.3.sort.0: 922337203685477779 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml index fb3b1f994fd8c..056b2f58b2229 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/260_sort_unsigned_long.yml @@ -44,17 +44,17 @@ setup: - length: {hits.hits: 6 } - match: { hits.hits.0._index: unsigned_long_sort } - match: { hits.hits.0._source.field: [ 13835058055282163722, 5, 6, 7 ] } - - match: { hits.hits.0.sort: [ 13835058055282163722 ] } + - match: { hits.hits.0.sort.0: 13835058055282163722 } - match: { hits.hits.1._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } - - match: { hits.hits.1.sort: [ 13835058055282163721 ] } + - match: { hits.hits.1.sort.0: 13835058055282163721 } - match: { hits.hits.2._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } - - match: { hits.hits.2.sort: [ 13835058055282163719 ] } + - match: { hits.hits.2.sort.0: 13835058055282163719 } - match: { hits.hits.3._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } - - match: { hits.hits.3.sort: [ 13835058055282163716 ] } + - match: { hits.hits.3.sort.0: 13835058055282163716 } - match: { hits.hits.4._source.field: [ 13835058055282163713, 13835058055282163714 ] } - - match: { hits.hits.4.sort: [ 13835058055282163714 ] } + - match: { hits.hits.4.sort.0: 13835058055282163714 } - match: { hits.hits.5._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.5.sort: [ 13835058055282163712 ] } + - match: { hits.hits.5.sort.0: 13835058055282163712 } - do: search: @@ -66,17 +66,17 @@ setup: - length: {hits.hits: 6 } - match: { hits.hits.0._index: unsigned_long_sort } - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.0.sort: [ 13835058055282163712 ] } + - match: { hits.hits.0.sort.0: 13835058055282163712 } - match: { hits.hits.1._source.field: [ 13835058055282163713, 13835058055282163714 ] } - - match: { hits.hits.1.sort: [ 13835058055282163714 ] } + - match: { hits.hits.1.sort.0: 13835058055282163714 } - match: { hits.hits.2._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } - - match: { hits.hits.2.sort: [ 13835058055282163716 ] } + - match: { hits.hits.2.sort.0: 13835058055282163716 } - match: { hits.hits.3._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } - - match: { hits.hits.3.sort: [ 13835058055282163719 ] } + - match: { hits.hits.3.sort.0: 13835058055282163719 } - match: { hits.hits.4._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } - - match: { hits.hits.4.sort: [ 13835058055282163721 ] } + - match: { hits.hits.4.sort.0: 13835058055282163721 } - match: { hits.hits.5._source.field: [ 13835058055282163722, 5, 6, 7 ] } - - match: { hits.hits.5.sort: [ 13835058055282163722 ] } + - match: { hits.hits.5.sort.0: 13835058055282163722 } - do: search: @@ -88,17 +88,17 @@ setup: - length: { hits.hits: 6 } - match: { hits.hits.0._index: unsigned_long_sort } - match: { hits.hits.0._source.field: [ 13835058055282163722, 5, 6, 7 ] } - - match: { hits.hits.0.sort: [ 7 ] } + - match: { hits.hits.0.sort.0: 7 } - match: { hits.hits.1._source.field: [ 13835058055282163713, 13835058055282163714 ] } - - match: { hits.hits.1.sort: [ 4611686018427387906 ] } + - match: { hits.hits.1.sort.0: 4611686018427387906 } - match: { hits.hits.2._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.2.sort: [ 6917529027641081857 ] } + - match: { hits.hits.2.sort.0: 6917529027641081857 } - match: { hits.hits.3._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } - - match: { hits.hits.3.sort: [ 6917529027641081862 ] } + - match: { hits.hits.3.sort.0: 6917529027641081862 } - match: { hits.hits.4._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } - - match: { hits.hits.4.sort: [ 13835058055282163715 ] } + - match: { hits.hits.4.sort.0: 13835058055282163715 } - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } - - match: { hits.hits.5.sort: [ 13835058055282163718 ] } + - match: { hits.hits.5.sort.0: 13835058055282163718 } - do: search: @@ -110,17 +110,17 @@ setup: - length: { hits.hits: 6 } - match: { hits.hits.0._index: unsigned_long_sort } - match: { hits.hits.0._source.field: [ 13835058055282163722, 5, 6, 7 ] } - - match: { hits.hits.0.sort: [ 13835058055282163740 ] } + - match: { hits.hits.0.sort.0: 13835058055282163740 } - match: { hits.hits.1._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.1.sort: [ 13835058055282163713 ] } + - match: { hits.hits.1.sort.0: 13835058055282163713 } - match: { hits.hits.2._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } - - match: { hits.hits.2.sort: [ 9223372036854775832 ] } + - match: { hits.hits.2.sort.0: 9223372036854775832 } - match: { hits.hits.3._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } - - match: { hits.hits.3.sort: [ 9223372036854775817 ] } + - match: { hits.hits.3.sort.0: 9223372036854775817 } - match: { hits.hits.4._source.field: [ 13835058055282163713, 13835058055282163714 ] } - - match: { hits.hits.4.sort: [ 9223372036854775811 ] } + - match: { hits.hits.4.sort.0: 9223372036854775811 } - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } - - match: { hits.hits.5.sort: [ 4611686018427387922 ] } + - match: { hits.hits.5.sort.0: 4611686018427387922 } - do: search: @@ -132,17 +132,17 @@ setup: - length: { hits.hits: 6 } - match: { hits.hits.0._index: unsigned_long_sort } - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.0.sort: [ 6917529027641081857 ] } + - match: { hits.hits.0.sort.0: 6917529027641081857 } - match: { hits.hits.1._source.field: [ 13835058055282163713, 13835058055282163714 ] } - - match: { hits.hits.1.sort: [ 4611686018427387906 ] } + - match: { hits.hits.1.sort.0: 4611686018427387906 } - match: { hits.hits.2._source.field: [ 13835058055282163722, 5, 6, 7 ] } - - match: { hits.hits.2.sort: [ 3458764513820540935 ] } + - match: { hits.hits.2.sort.0: 3458764513820540935 } - match: { hits.hits.3._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } - - match: { hits.hits.3.sort: [ 3074457345618258606 ] } + - match: { hits.hits.3.sort.0: 3074457345618258606 } - match: { hits.hits.4._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } - - match: { hits.hits.4.sort: [ 2305843009213693958 ] } + - match: { hits.hits.4.sort.0: 2305843009213693958 } - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } - - match: { hits.hits.5.sort: [ 1537228672809129307 ] } + - match: { hits.hits.5.sort.0: 1537228672809129307 } - do: search: @@ -154,14 +154,14 @@ setup: - length: { hits.hits: 6 } - match: { hits.hits.0._index: unsigned_long_sort } - match: { hits.hits.0._source.field: [ 13835058055282163712, 1 ] } - - match: { hits.hits.0.sort: [ 1 ] } + - match: { hits.hits.0.sort.0: 1 } - match: { hits.hits.1._source.field: [ 13835058055282163715, 13835058055282163716, 2 ] } - - match: { hits.hits.1.sort: [ 2 ] } + - match: { hits.hits.1.sort.0: 2 } - match: { hits.hits.2._source.field: [ 13835058055282163720, 13835058055282163721, 3, 4 ] } - - match: { hits.hits.2.sort: [ 3 ] } + - match: { hits.hits.2.sort.0: 3 } - match: { hits.hits.3._source.field: [ 13835058055282163722, 5, 6, 7 ] } - - match: { hits.hits.3.sort: [ 5 ] } + - match: { hits.hits.3.sort.0: 5 } - match: { hits.hits.4._source.field: [ 13835058055282163713, 13835058055282163714 ] } - - match: { hits.hits.4.sort: [ 13835058055282163713 ] } + - match: { hits.hits.4.sort.0: 13835058055282163713 } - match: { hits.hits.5._source.field: [ 13835058055282163717, 13835058055282163718, 13835058055282163719 ] } - - match: { hits.hits.5.sort: [ 13835058055282163717 ] } + - match: { hits.hits.5.sort.0: 13835058055282163717 } diff --git a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java index b46ea23f8cd77..77d8f3154729e 100644 --- a/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java +++ b/test/framework/src/main/java/org/opensearch/test/rest/yaml/section/MatchAssertion.java @@ -40,7 +40,6 @@ import org.opensearch.test.hamcrest.RegexMatcher; import java.io.IOException; -import java.util.ArrayList; import java.util.regex.Pattern; import static org.hamcrest.Matchers.equalTo; @@ -111,28 +110,6 @@ protected void doAssert(Object actualValue, Object expectedValue) { } if (expectedValue.equals(actualValue) == false) { - if (expectedValue instanceof ArrayList && actualValue instanceof ArrayList) { - ArrayList expectedList = (ArrayList) expectedValue; - ArrayList actualList = (ArrayList) actualValue; - if (expectedList.size() == actualList.size()) { - boolean pass = true; - for (int i = 0; i < expectedList.size(); i++) { - // BigInteger 1 is equal to Integer 1 - Object expected = expectedList.get(i); - Object actual = actualList.get(i); - if (expected instanceof Number == false - || actual instanceof Number == false - || ((Number) expected).longValue() != ((Number) actual).longValue()) { - pass = false; - break; - } - } - if (pass) { - return; - } - } - } - NotEqualMessageBuilder message = new NotEqualMessageBuilder(); message.compare(getField(), true, actualValue, expectedValue); throw new AssertionError(getField() + " didn't match expected value:\n" + message); From 94ef87740a1957ad708f6aecb05e9405eee067ec Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 19 Dec 2024 14:49:21 -0500 Subject: [PATCH 06/10] Extend MultiValueMode with dedicated support of unsigned_long doc values Signed-off-by: Andriy Redko --- .../opensearch/index/fielddata/FieldData.java | 12 + ...gletonSortedNumericUnsignedLongValues.java | 78 ++++ .../SortedNumericUnsignedLongValues.java | 86 ++++ .../UnsignedLongMultiValueMode.java | 400 ------------------ .../UnsignedLongValuesComparatorSource.java | 14 +- .../org/opensearch/search/MultiValueMode.java | 328 ++++++++++++++ .../UnsignedLongMultiValueModeTests.java | 271 ------------ .../search/MultiValueModeTests.java | 230 ++++++++++ 8 files changed, 741 insertions(+), 678 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java create mode 100644 server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java delete mode 100644 server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java delete mode 100644 server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java index 6db6bbccacae5..61641f65489ae 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java @@ -311,6 +311,18 @@ public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues valu return null; } + /** + * Returns a single-valued view of the {@link SortedNumericDoubleValues}, + * if it was previously wrapped with {@link DocValues#singleton(NumericDocValues)}, + * or null. + */ + public static SortedNumericDocValues unwrapSingleton(SortedNumericUnsignedLongValues values) { + if (values instanceof SingletonSortedNumericUnsignedLongValues) { + return ((SingletonSortedNumericUnsignedLongValues) values).getNumericUnsignedLongValues(); + } + return null; + } + /** * Returns a multi-valued view over the provided {@link GeoPointValues}. */ diff --git a/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java b/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java new file mode 100644 index 0000000000000..05929d08a142c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.fielddata; + +import org.apache.lucene.index.SortedNumericDocValues; + +import java.io.IOException; + +/** + * Clone of {@link SortedNumericDocValues} for double values. + * + * @opensearch.internal + */ +public final class SingletonSortedNumericUnsignedLongValues extends SortedNumericUnsignedLongValues { + private final SortedNumericDocValues values; + + public SingletonSortedNumericUnsignedLongValues(SortedNumericDocValues values) { + this.values = values; + } + + @Override + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); + } + + @Override + public long nextValue() throws IOException { + return values.nextValue(); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + + public int advance(int target) throws IOException { + return values.advance(target); + } + + public int docID() { + return values.docID(); + } + + /** Return the wrapped values. */ + public SortedNumericDocValues getNumericUnsignedLongValues() { + return values; + } +} diff --git a/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java b/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java new file mode 100644 index 0000000000000..b22e732280fa2 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.fielddata; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.opensearch.common.annotation.PublicApi; + +import java.io.IOException; + +/** + * Clone of {@link SortedNumericDocValues} for unsigned long values. + * + * @opensearch.api + */ +@PublicApi(since = "1.0.0") +public abstract class SortedNumericUnsignedLongValues { + + /** Sole constructor. (For invocation by subclass + * constructors, typically implicit.) */ + protected SortedNumericUnsignedLongValues() {} + + /** Advance the iterator to exactly {@code target} and return whether + * {@code target} has a value. + * {@code target} must be greater than or equal to the current + * doc ID and must be a valid doc ID, ie. ≥ 0 and + * < {@code maxDoc}.*/ + public abstract boolean advanceExact(int target) throws IOException; + + /** + * Iterates to the next value in the current document. Do not call this more than + * {@link #docValueCount} times for the document. + */ + public abstract long nextValue() throws IOException; + + /** + * Retrieves the number of values for the current document. This must always + * be greater than zero. + * It is illegal to call this method after {@link #advanceExact(int)} + * returned {@code false}. + */ + public abstract int docValueCount(); + + /** + * Advances to the first beyond the current whose document number is greater than or equal to + * target, and returns the document number itself. Exhausts the iterator and returns {@link + * org.apache.lucene.search.DocIdSetIterator#NO_MORE_DOCS} if target is greater than the highest document number in the set. + * + * This method is being used by {@link org.apache.lucene.search.comparators.NumericComparator.NumericLeafComparator} when point values optimization kicks + * in and is implemented by most numeric types. + */ + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + public abstract int docID(); +} diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java deleted file mode 100644 index 4e20460f5c820..0000000000000 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueMode.java +++ /dev/null @@ -1,400 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BitSet; -import org.opensearch.common.Numbers; -import org.opensearch.index.fielddata.AbstractNumericDocValues; -import org.opensearch.index.fielddata.FieldData; -import org.opensearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.Locale; - -/** - * Defines what values to pick in the case a document contains multiple values for an unsigned long field. - * - * @opensearch.internal - */ -enum UnsignedLongMultiValueMode { - /** - * Pick the sum of all the values. - */ - SUM { - @Override - protected long pick(SortedNumericDocValues values) throws IOException { - final int count = values.docValueCount(); - long total = 0; - for (int index = 0; index < count; ++index) { - total += values.nextValue(); - } - return total; - } - - @Override - protected long pick( - SortedNumericDocValues values, - long missingValue, - DocIdSetIterator docItr, - int startDoc, - int endDoc, - int maxChildren - ) throws IOException { - int totalCount = 0; - long totalValue = 0; - int count = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - if (values.advanceExact(doc)) { - if (++count > maxChildren) { - break; - } - - final int docCount = values.docValueCount(); - for (int index = 0; index < docCount; ++index) { - totalValue += values.nextValue(); - } - totalCount += docCount; - } - } - return totalCount > 0 ? totalValue : missingValue; - } - }, - - /** - * Pick the average of all the values. - */ - AVG { - @Override - protected long pick(SortedNumericDocValues values) throws IOException { - final int count = values.docValueCount(); - long total = 0; - for (int index = 0; index < count; ++index) { - total += values.nextValue(); - } - return count > 1 ? divideUnsignedAndRoundUp(total, count) : total; - } - - @Override - protected long pick( - SortedNumericDocValues values, - long missingValue, - DocIdSetIterator docItr, - int startDoc, - int endDoc, - int maxChildren - ) throws IOException { - int totalCount = 0; - long totalValue = 0; - int count = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - if (values.advanceExact(doc)) { - if (++count > maxChildren) { - break; - } - final int docCount = values.docValueCount(); - for (int index = 0; index < docCount; ++index) { - totalValue += values.nextValue(); - } - totalCount += docCount; - } - } - if (totalCount < 1) { - return missingValue; - } - return totalCount > 1 ? divideUnsignedAndRoundUp(totalValue, totalCount) : totalValue; - } - }, - - /** - * Pick the median of the values. - */ - MEDIAN { - @Override - protected long pick(SortedNumericDocValues values) throws IOException { - int count = values.docValueCount(); - long firstValue = values.nextValue(); - if (count == 1) { - return firstValue; - } else if (count == 2) { - long total = firstValue + values.nextValue(); - return (total >>> 1) + (total & 1); - } else if (firstValue >= 0) { - for (int i = 1; i < (count - 1) / 2; ++i) { - values.nextValue(); - } - if (count % 2 == 0) { - long total = values.nextValue() + values.nextValue(); - return (total >>> 1) + (total & 1); - } else { - return values.nextValue(); - } - } - - final long[] docValues = new long[count]; - docValues[0] = firstValue; - int firstPositiveIndex = 0; - for (int i = 1; i < count; ++i) { - docValues[i] = values.nextValue(); - if (docValues[i] >= 0 && firstPositiveIndex == 0) { - firstPositiveIndex = i; - } - } - final int mid = ((count - 1) / 2 + firstPositiveIndex) % count; - if (count % 2 == 0) { - long total = docValues[mid] + docValues[(mid + 1) % count]; - return (total >>> 1) + (total & 1); - } else { - return docValues[mid]; - } - } - }, - - /** - * Pick the lowest value. - */ - MIN { - @Override - protected long pick(SortedNumericDocValues values) throws IOException { - final int count = values.docValueCount(); - final long min = values.nextValue(); - if (count == 1 || min > 0) { - return min; - } - for (int i = 1; i < count; ++i) { - long val = values.nextValue(); - if (val >= 0) { - return val; - } - } - return min; - } - - @Override - protected long pick( - SortedNumericDocValues values, - long missingValue, - DocIdSetIterator docItr, - int startDoc, - int endDoc, - int maxChildren - ) throws IOException { - boolean hasValue = false; - long minValue = Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG; - int count = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - if (values.advanceExact(doc)) { - if (++count > maxChildren) { - break; - } - final long docMin = pick(values); - minValue = Long.compareUnsigned(docMin, minValue) < 0 ? docMin : minValue; - hasValue = true; - } - } - return hasValue ? minValue : missingValue; - } - }, - - /** - * Pick the highest value. - */ - MAX { - @Override - protected long pick(SortedNumericDocValues values) throws IOException { - final int count = values.docValueCount(); - long max = values.nextValue(); - long val; - for (int i = 1; i < count; ++i) { - val = values.nextValue(); - if (max < 0 && val >= 0) { - return max; - } - max = val; - } - return max; - } - - @Override - protected long pick( - SortedNumericDocValues values, - long missingValue, - DocIdSetIterator docItr, - int startDoc, - int endDoc, - int maxChildren - ) throws IOException { - boolean hasValue = false; - long maxValue = Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG; - int count = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - if (values.advanceExact(doc)) { - if (++count > maxChildren) { - break; - } - final long docMax = pick(values); - maxValue = Long.compareUnsigned(maxValue, docMax) < 0 ? docMax : maxValue; - hasValue = true; - } - } - return hasValue ? maxValue : missingValue; - } - }; - - /** - * A case insensitive version of {@link #valueOf(String)} - * - * @throws IllegalArgumentException if the given string doesn't match a sort mode or is null. - */ - private static UnsignedLongMultiValueMode fromString(String sortMode) { - try { - return valueOf(sortMode.toUpperCase(Locale.ROOT)); - } catch (Exception e) { - throw new IllegalArgumentException("Illegal sort mode: " + sortMode); - } - } - - /** - * Convert a {@link MultiValueMode} to a {@link UnsignedLongMultiValueMode}. - */ - public static UnsignedLongMultiValueMode toUnsignedSortMode(MultiValueMode sortMode) { - return fromString(sortMode.name()); - } - - /** - * Return a {@link NumericDocValues} instance that can be used to sort documents - * with this mode and the provided values. When a document has no value, - * missingValue is returned. - *

- * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX - */ - public NumericDocValues select(final SortedNumericDocValues values) { - final NumericDocValues singleton = DocValues.unwrapSingleton(values); - if (singleton != null) { - return singleton; - } else { - return new AbstractNumericDocValues() { - - private long value; - - @Override - public boolean advanceExact(int target) throws IOException { - if (values.advanceExact(target)) { - value = pick(values); - return true; - } - return false; - } - - @Override - public int docID() { - return values.docID(); - } - - @Override - public long longValue() throws IOException { - return value; - } - }; - } - } - - protected long pick(SortedNumericDocValues values) throws IOException { - throw new IllegalArgumentException("Unsupported sort mode: " + this); - } - - /** - * Return a {@link NumericDocValues} instance that can be used to sort root documents - * with this mode, the provided values and filters for root/inner documents. - *

- * For every root document, the values of its inner documents will be aggregated. - * If none of the inner documents has a value, then missingValue is returned. - *

- * Allowed Modes: SUM, AVG, MIN, MAX - *

- * NOTE: Calling the returned instance on docs that are not root docs is illegal - * The returned instance can only be evaluate the current and upcoming docs - */ - public NumericDocValues select( - final SortedNumericDocValues values, - final long missingValue, - final BitSet parentDocs, - final DocIdSetIterator childDocs, - int maxDoc, - int maxChildren - ) throws IOException { - if (parentDocs == null || childDocs == null) { - return FieldData.replaceMissing(DocValues.emptyNumeric(), missingValue); - } - - return new AbstractNumericDocValues() { - - int lastSeenParentDoc = -1; - long lastEmittedValue = missingValue; - - @Override - public boolean advanceExact(int parentDoc) throws IOException { - assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs"; - if (parentDoc == lastSeenParentDoc) { - return true; - } else if (parentDoc == 0) { - lastEmittedValue = missingValue; - return true; - } - final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); - final int firstChildDoc; - if (childDocs.docID() > prevParentDoc) { - firstChildDoc = childDocs.docID(); - } else { - firstChildDoc = childDocs.advance(prevParentDoc + 1); - } - - lastSeenParentDoc = parentDoc; - lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren); - return true; - } - - @Override - public int docID() { - return lastSeenParentDoc; - } - - @Override - public long longValue() { - return lastEmittedValue; - } - }; - } - - protected long pick( - SortedNumericDocValues values, - long missingValue, - DocIdSetIterator docItr, - int startDoc, - int endDoc, - int maxChildren - ) throws IOException { - throw new IllegalArgumentException("Unsupported sort mode: " + this); - } - - /** - * Copied from {@link Long#divideUnsigned(long, long)} and {@link Long#remainderUnsigned(long, long)} - */ - private static long divideUnsignedAndRoundUp(long dividend, long divisor) { - assert divisor > 0; - final long q = (dividend >>> 1) / divisor << 1; - final long r = dividend - q * divisor; - final long quotient = q + ((r | ~(r - divisor)) >>> (Long.SIZE - 1)); - final long rem = r - ((~(r - divisor) >> (Long.SIZE - 1)) & divisor); - return quotient + Math.round((double) rem / divisor); - } -} diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index c7b9c9f445484..c9076e773ba29 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -24,6 +24,8 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.fielddata.LeafNumericFieldData; +import org.opensearch.index.fielddata.SingletonSortedNumericUnsignedLongValues; +import org.opensearch.index.fielddata.SortedNumericUnsignedLongValues; import org.opensearch.index.search.comparators.UnsignedLongComparator; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; @@ -41,7 +43,6 @@ public class UnsignedLongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource { private final IndexNumericFieldData indexFieldData; - private final UnsignedLongMultiValueMode unsignedSortMode; public UnsignedLongValuesComparatorSource( IndexNumericFieldData indexFieldData, @@ -51,7 +52,6 @@ public UnsignedLongValuesComparatorSource( ) { super(missingValue, sortMode, nested); this.indexFieldData = indexFieldData; - this.unsignedSortMode = UnsignedLongMultiValueMode.toUnsignedSortMode(sortMode); } @Override @@ -59,21 +59,21 @@ public SortField.Type reducedType() { return SortField.Type.LONG; } - private SortedNumericDocValues loadDocValues(LeafReaderContext context) { + private SortedNumericUnsignedLongValues loadDocValues(LeafReaderContext context) { final LeafNumericFieldData data = indexFieldData.load(context); SortedNumericDocValues values = data.getLongValues(); - return values; + return new SingletonSortedNumericUnsignedLongValues(values); } private NumericDocValues getNumericDocValues(LeafReaderContext context, BigInteger missingValue) throws IOException { - final SortedNumericDocValues values = loadDocValues(context); + final SortedNumericUnsignedLongValues values = loadDocValues(context); if (nested == null) { - return FieldData.replaceMissing(unsignedSortMode.select(values), missingValue); + return FieldData.replaceMissing(sortMode.select(values), missingValue); } final BitSet rootDocs = nested.rootDocs(context); final DocIdSetIterator innerDocs = nested.innerDocs(context); final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE; - return unsignedSortMode.select(values, missingValue.longValue(), rootDocs, innerDocs, context.reader().maxDoc(), maxChildren); + return sortMode.select(values, missingValue.longValue(), rootDocs, innerDocs, context.reader().maxDoc(), maxChildren); } @Override diff --git a/server/src/main/java/org/opensearch/search/MultiValueMode.java b/server/src/main/java/org/opensearch/search/MultiValueMode.java index a99da674836f2..b9df357016788 100644 --- a/server/src/main/java/org/opensearch/search/MultiValueMode.java +++ b/server/src/main/java/org/opensearch/search/MultiValueMode.java @@ -42,6 +42,7 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import org.opensearch.common.Numbers; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -53,6 +54,7 @@ import org.opensearch.index.fielddata.NumericDoubleValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; +import org.opensearch.index.fielddata.SortedNumericUnsignedLongValues; import java.io.IOException; import java.util.Locale; @@ -143,6 +145,44 @@ protected double pick( return totalCount > 0 ? totalValue : missingValue; } + + @Override + protected long pick(SortedNumericUnsignedLongValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); + } + return total; + } + + @Override + protected long pick( + SortedNumericUnsignedLongValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + int totalCount = 0; + long totalValue = 0; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + + final int docCount = values.docValueCount(); + for (int index = 0; index < docCount; ++index) { + totalValue += values.nextValue(); + } + totalCount += docCount; + } + } + return totalCount > 0 ? totalValue : missingValue; + } }, /** @@ -228,6 +268,46 @@ protected double pick( } return totalValue / totalCount; } + + @Override + protected long pick(SortedNumericUnsignedLongValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); + } + return count > 1 ? divideUnsignedAndRoundUp(total, count) : total; + } + + @Override + protected long pick( + SortedNumericUnsignedLongValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + int totalCount = 0; + long totalValue = 0; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + final int docCount = values.docValueCount(); + for (int index = 0; index < docCount; ++index) { + totalValue += values.nextValue(); + } + totalCount += docCount; + } + } + if (totalCount < 1) { + return missingValue; + } + return totalCount > 1 ? divideUnsignedAndRoundUp(totalValue, totalCount) : totalValue; + } }, /** @@ -259,6 +339,45 @@ protected double pick(SortedNumericDoubleValues values) throws IOException { return values.nextValue(); } } + + @Override + protected long pick(SortedNumericUnsignedLongValues values) throws IOException { + int count = values.docValueCount(); + long firstValue = values.nextValue(); + if (count == 1) { + return firstValue; + } else if (count == 2) { + long total = firstValue + values.nextValue(); + return (total >>> 1) + (total & 1); + } else if (firstValue >= 0) { + for (int i = 1; i < (count - 1) / 2; ++i) { + values.nextValue(); + } + if (count % 2 == 0) { + long total = values.nextValue() + values.nextValue(); + return (total >>> 1) + (total & 1); + } else { + return values.nextValue(); + } + } + + final long[] docValues = new long[count]; + docValues[0] = firstValue; + int firstPositiveIndex = 0; + for (int i = 1; i < count; ++i) { + docValues[i] = values.nextValue(); + if (docValues[i] >= 0 && firstPositiveIndex == 0) { + firstPositiveIndex = i; + } + } + final int mid = ((count - 1) / 2 + firstPositiveIndex) % count; + if (count % 2 == 0) { + long total = docValues[mid] + docValues[(mid + 1) % count]; + return (total >>> 1) + (total & 1); + } else { + return docValues[mid]; + } + } }, /** @@ -382,6 +501,47 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc return hasValue ? ord : -1; } + + @Override + protected long pick(SortedNumericUnsignedLongValues values) throws IOException { + final int count = values.docValueCount(); + final long min = values.nextValue(); + if (count == 1 || min > 0) { + return min; + } + for (int i = 1; i < count; ++i) { + long val = values.nextValue(); + if (val >= 0) { + return val; + } + } + return min; + } + + @Override + protected long pick( + SortedNumericUnsignedLongValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + boolean hasValue = false; + long minValue = Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + final long docMin = pick(values); + minValue = Long.compareUnsigned(docMin, minValue) < 0 ? docMin : minValue; + hasValue = true; + } + } + return hasValue ? minValue : missingValue; + } }, /** @@ -525,6 +685,46 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc } return ord; } + + @Override + protected long pick(SortedNumericUnsignedLongValues values) throws IOException { + final int count = values.docValueCount(); + long max = values.nextValue(); + long val; + for (int i = 1; i < count; ++i) { + val = values.nextValue(); + if (max < 0 && val >= 0) { + return max; + } + max = val; + } + return max; + } + + @Override + protected long pick( + SortedNumericUnsignedLongValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + boolean hasValue = false; + long maxValue = Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG; + int count = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + if (++count > maxChildren) { + break; + } + final long docMax = pick(values); + maxValue = Long.compareUnsigned(maxValue, docMax) < 0 ? docMax : maxValue; + hasValue = true; + } + } + return hasValue ? maxValue : missingValue; + } }; /** @@ -1032,6 +1232,122 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc throw new IllegalArgumentException("Unsupported sort mode: " + this); } + /** + * Return a {@link NumericDoubleValues} instance that can be used to sort documents + * with this mode and the provided values. When a document has no value, + * missingValue is returned. + *

+ * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX + */ + public NumericDocValues select(final SortedNumericUnsignedLongValues values) { + final SortedNumericDocValues sortedNumericDocValues = FieldData.unwrapSingleton(values); + final NumericDocValues singleton = DocValues.unwrapSingleton(sortedNumericDocValues); + if (singleton != null) { + return singleton; + } else { + return new AbstractNumericDocValues() { + + private long value; + + @Override + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + value = pick(values); + return true; + } + return false; + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public long longValue() throws IOException { + return value; + } + }; + } + } + + protected long pick(SortedNumericUnsignedLongValues values) throws IOException { + throw new IllegalArgumentException("Unsupported sort mode: " + this); + } + + /** + * Return a {@link SortedDocValues} instance that can be used to sort root documents + * with this mode, the provided values and filters for root/inner documents. + *

+ * For every root document, the values of its inner documents will be aggregated. + *

+ * Allowed Modes: MIN, MAX + *

+ * NOTE: Calling the returned instance on docs that are not root docs is illegal + * The returned instance can only be evaluate the current and upcoming docs + */ + public NumericDocValues select( + final SortedNumericUnsignedLongValues values, + final long missingValue, + final BitSet parentDocs, + final DocIdSetIterator childDocs, + int maxDoc, + int maxChildren + ) throws IOException { + if (parentDocs == null || childDocs == null) { + return FieldData.replaceMissing(DocValues.emptyNumeric(), missingValue); + } + + return new AbstractNumericDocValues() { + + int lastSeenParentDoc = -1; + long lastEmittedValue = missingValue; + + @Override + public boolean advanceExact(int parentDoc) throws IOException { + assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs"; + if (parentDoc == lastSeenParentDoc) { + return true; + } else if (parentDoc == 0) { + lastEmittedValue = missingValue; + return true; + } + final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); + final int firstChildDoc; + if (childDocs.docID() > prevParentDoc) { + firstChildDoc = childDocs.docID(); + } else { + firstChildDoc = childDocs.advance(prevParentDoc + 1); + } + + lastSeenParentDoc = parentDoc; + lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren); + return true; + } + + @Override + public int docID() { + return lastSeenParentDoc; + } + + @Override + public long longValue() { + return lastEmittedValue; + } + }; + } + + protected long pick( + SortedNumericUnsignedLongValues values, + long missingValue, + DocIdSetIterator docItr, + int startDoc, + int endDoc, + int maxChildren + ) throws IOException { + throw new IllegalArgumentException("Unsupported sort mode: " + this); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeEnum(this); @@ -1040,4 +1356,16 @@ public void writeTo(StreamOutput out) throws IOException { public static MultiValueMode readMultiValueModeFrom(StreamInput in) throws IOException { return in.readEnum(MultiValueMode.class); } + + /** + * Copied from {@link Long#divideUnsigned(long, long)} and {@link Long#remainderUnsigned(long, long)} + */ + private static long divideUnsignedAndRoundUp(long dividend, long divisor) { + assert divisor > 0; + final long q = (dividend >>> 1) / divisor << 1; + final long r = dividend - q * divisor; + final long quotient = q + ((r | ~(r - divisor)) >>> (Long.SIZE - 1)); + final long rem = r - ((~(r - divisor) >> (Long.SIZE - 1)) & divisor); + return quotient + Math.round((double) rem / divisor); + } } diff --git a/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java b/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java deleted file mode 100644 index afa562862cfe9..0000000000000 --- a/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.fielddata.fieldcomparator; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.util.BitSetIterator; -import org.apache.lucene.util.FixedBitSet; -import org.opensearch.common.Numbers; -import org.opensearch.index.fielddata.AbstractNumericDocValues; -import org.opensearch.index.fielddata.AbstractSortedNumericDocValues; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.util.Arrays; -import java.util.function.Supplier; - -public class UnsignedLongMultiValueModeTests extends OpenSearchTestCase { - private static FixedBitSet randomRootDocs(int maxDoc) { - FixedBitSet set = new FixedBitSet(maxDoc); - for (int i = 0; i < maxDoc; ++i) { - if (randomBoolean()) { - set.set(i); - } - } - // the last doc must be a root doc - set.set(maxDoc - 1); - return set; - } - - private static FixedBitSet randomInnerDocs(FixedBitSet rootDocs) { - FixedBitSet innerDocs = new FixedBitSet(rootDocs.length()); - for (int i = 0; i < innerDocs.length(); ++i) { - if (!rootDocs.get(i) && randomBoolean()) { - innerDocs.set(i); - } - } - return innerDocs; - } - - public void testSingleValuedLongs() throws Exception { - final int numDocs = scaledRandomIntBetween(1, 100); - final long[] array = new long[numDocs]; - final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); - for (int i = 0; i < array.length; ++i) { - if (randomBoolean()) { - array[i] = randomUnsignedLong().longValue(); - if (docsWithValue != null) { - docsWithValue.set(i); - } - } else if (docsWithValue != null && randomBoolean()) { - docsWithValue.set(i); - } - } - - final Supplier multiValues = () -> DocValues.singleton(new AbstractNumericDocValues() { - int docId = -1; - - @Override - public boolean advanceExact(int target) throws IOException { - this.docId = target; - return docsWithValue == null || docsWithValue.get(docId); - } - - @Override - public int docID() { - return docId; - } - - @Override - public long longValue() { - return array[docId]; - } - }); - verifySortedNumeric(multiValues, numDocs); - final FixedBitSet rootDocs = randomRootDocs(numDocs); - final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE); - verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs)); - } - - public void testMultiValuedLongs() throws Exception { - final int numDocs = scaledRandomIntBetween(1, 100); - final long[][] array = new long[numDocs][]; - for (int i = 0; i < numDocs; ++i) { - final long[] values = new long[randomInt(4)]; - for (int j = 0; j < values.length; ++j) { - values[j] = randomUnsignedLong().longValue(); - } - Arrays.sort(values); - array[i] = values; - } - final Supplier multiValues = () -> new AbstractSortedNumericDocValues() { - int doc; - int i; - - @Override - public long nextValue() { - return array[doc][i++]; - } - - @Override - public boolean advanceExact(int doc) { - this.doc = doc; - i = 0; - return array[doc].length > 0; - } - - @Override - public int docValueCount() { - return array[doc].length; - } - }; - verifySortedNumeric(multiValues, numDocs); - final FixedBitSet rootDocs = randomRootDocs(numDocs); - final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE); - verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs)); - } - - private void verifySortedNumeric(Supplier supplier, int maxDoc) throws IOException { - for (UnsignedLongMultiValueMode mode : UnsignedLongMultiValueMode.values()) { - SortedNumericDocValues values = supplier.get(); - final NumericDocValues selected = mode.select(values); - for (int i = 0; i < maxDoc; ++i) { - Long actual = null; - if (selected.advanceExact(i)) { - actual = selected.longValue(); - verifyLongValueCanCalledMoreThanOnce(selected, actual); - } - - BigInteger expected = null; - if (values.advanceExact(i)) { - int numValues = values.docValueCount(); - if (mode == UnsignedLongMultiValueMode.MAX) { - expected = Numbers.MIN_UNSIGNED_LONG_VALUE; - } else if (mode == UnsignedLongMultiValueMode.MIN) { - expected = Numbers.MAX_UNSIGNED_LONG_VALUE; - } else { - expected = BigInteger.ZERO; - } - for (int j = 0; j < numValues; ++j) { - if (mode == UnsignedLongMultiValueMode.SUM || mode == UnsignedLongMultiValueMode.AVG) { - expected = expected.add(Numbers.toUnsignedBigInteger(values.nextValue())); - } else if (mode == UnsignedLongMultiValueMode.MIN) { - expected = expected.min(Numbers.toUnsignedBigInteger(values.nextValue())); - } else if (mode == UnsignedLongMultiValueMode.MAX) { - expected = expected.max(Numbers.toUnsignedBigInteger(values.nextValue())); - } - } - if (mode == UnsignedLongMultiValueMode.AVG) { - expected = Numbers.toUnsignedBigInteger(expected.longValue()); - expected = numValues > 1 - ? new BigDecimal(expected).divide(new BigDecimal(numValues), RoundingMode.HALF_UP).toBigInteger() - : expected; - } else if (mode == UnsignedLongMultiValueMode.MEDIAN) { - final Long[] docValues = new Long[numValues]; - for (int j = 0; j < numValues; ++j) { - docValues[j] = values.nextValue(); - } - Arrays.sort(docValues, Long::compareUnsigned); - int value = numValues / 2; - if (numValues % 2 == 0) { - expected = Numbers.toUnsignedBigInteger(docValues[value - 1]) - .add(Numbers.toUnsignedBigInteger(docValues[value])); - expected = Numbers.toUnsignedBigInteger(expected.longValue()); - expected = new BigDecimal(expected).divide(new BigDecimal(2), RoundingMode.HALF_UP).toBigInteger(); - } else { - expected = Numbers.toUnsignedBigInteger(docValues[value]); - } - } - } - - final Long expectedLong = expected == null ? null : expected.longValue(); - assertEquals(mode.toString() + " docId=" + i, expectedLong, actual); - } - } - } - - private void verifyLongValueCanCalledMoreThanOnce(NumericDocValues values, long expected) throws IOException { - for (int j = 0, numCall = randomIntBetween(1, 10); j < numCall; j++) { - assertEquals(expected, values.longValue()); - } - } - - private void verifySortedNumeric( - Supplier supplier, - int maxDoc, - FixedBitSet rootDocs, - FixedBitSet innerDocs, - int maxChildren - ) throws IOException { - for (long missingValue : new long[] { 0, randomUnsignedLong().longValue() }) { - for (UnsignedLongMultiValueMode mode : new UnsignedLongMultiValueMode[] { - UnsignedLongMultiValueMode.MIN, - UnsignedLongMultiValueMode.MAX, - UnsignedLongMultiValueMode.SUM, - UnsignedLongMultiValueMode.AVG }) { - SortedNumericDocValues values = supplier.get(); - final NumericDocValues selected = mode.select( - values, - missingValue, - rootDocs, - new BitSetIterator(innerDocs, 0L), - maxDoc, - maxChildren - ); - int prevRoot = -1; - for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - assertTrue(selected.advanceExact(root)); - final long actual = selected.longValue(); - verifyLongValueCanCalledMoreThanOnce(selected, actual); - - BigInteger expected = BigInteger.ZERO; - if (mode == UnsignedLongMultiValueMode.MAX) { - expected = Numbers.MIN_UNSIGNED_LONG_VALUE; - } else if (mode == UnsignedLongMultiValueMode.MIN) { - expected = Numbers.MAX_UNSIGNED_LONG_VALUE; - } - int numValues = 0; - int count = 0; - for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit( - child + 1 - )) { - if (values.advanceExact(child)) { - if (++count > maxChildren) { - break; - } - for (int j = 0; j < values.docValueCount(); ++j) { - if (mode == UnsignedLongMultiValueMode.SUM || mode == UnsignedLongMultiValueMode.AVG) { - expected = expected.add(Numbers.toUnsignedBigInteger(values.nextValue())); - } else if (mode == UnsignedLongMultiValueMode.MIN) { - expected = expected.min(Numbers.toUnsignedBigInteger(values.nextValue())); - } else if (mode == UnsignedLongMultiValueMode.MAX) { - expected = expected.max(Numbers.toUnsignedBigInteger(values.nextValue())); - } - ++numValues; - } - } - } - final long expectedLong; - if (numValues == 0) { - expectedLong = missingValue; - } else if (mode == UnsignedLongMultiValueMode.AVG) { - expected = Numbers.toUnsignedBigInteger(expected.longValue()); - expected = numValues > 1 - ? new BigDecimal(expected).divide(new BigDecimal(numValues), RoundingMode.HALF_UP).toBigInteger() - : expected; - expectedLong = expected.longValue(); - } else { - expectedLong = expected.longValue(); - } - - assertEquals(mode.toString() + " docId=" + root, expectedLong, actual); - - prevRoot = root; - } - } - } - } -} diff --git a/server/src/test/java/org/opensearch/search/MultiValueModeTests.java b/server/src/test/java/org/opensearch/search/MultiValueModeTests.java index 948d2cffceabe..e011dd0bcf6c0 100644 --- a/server/src/test/java/org/opensearch/search/MultiValueModeTests.java +++ b/server/src/test/java/org/opensearch/search/MultiValueModeTests.java @@ -41,6 +41,7 @@ import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; +import org.opensearch.common.Numbers; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.fielddata.AbstractBinaryDocValues; @@ -52,9 +53,13 @@ import org.opensearch.index.fielddata.NumericDoubleValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; +import org.opensearch.index.fielddata.SortedNumericUnsignedLongValues; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; import java.util.Arrays; import static org.hamcrest.Matchers.equalTo; @@ -776,6 +781,96 @@ public int docValueCount() { verifySortedSet(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs)); } + public void testSingleValuedUnsignedLongs() throws Exception { + final int numDocs = scaledRandomIntBetween(1, 100); + final long[] array = new long[numDocs]; + final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); + for (int i = 0; i < array.length; ++i) { + if (randomBoolean()) { + array[i] = randomUnsignedLong().longValue(); + if (docsWithValue != null) { + docsWithValue.set(i); + } + } else if (docsWithValue != null && randomBoolean()) { + docsWithValue.set(i); + } + } + + final Supplier multiValues = () -> new SortedNumericUnsignedLongValues() { + int docId = -1; + + @Override + public boolean advanceExact(int target) throws IOException { + this.docId = target; + return docsWithValue == null || docsWithValue.get(docId); + } + + @Override + public int docID() { + return docId; + } + + @Override + public long nextValue() { + return array[docId]; + } + + @Override + public int docValueCount() { + return 1; + } + }; + verifySortedUnsignedLong(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verifySortedUnsignedLong(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE); + verifySortedUnsignedLong(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs)); + } + + public void testMultiValuedUnsignedLongs() throws Exception { + final int numDocs = scaledRandomIntBetween(1, 100); + final long[][] array = new long[numDocs][]; + for (int i = 0; i < numDocs; ++i) { + final long[] values = new long[randomInt(4)]; + for (int j = 0; j < values.length; ++j) { + values[j] = randomUnsignedLong().longValue(); + } + Arrays.sort(values); + array[i] = values; + } + final Supplier multiValues = () -> new SortedNumericUnsignedLongValues() { + int doc; + int i; + + @Override + public long nextValue() { + return array[doc][i++]; + } + + @Override + public boolean advanceExact(int doc) { + this.doc = doc; + i = 0; + return array[doc].length > 0; + } + + @Override + public int docValueCount() { + return array[doc].length; + } + + @Override + public int docID() { + return doc; + } + }; + verifySortedUnsignedLong(multiValues, numDocs); + final FixedBitSet rootDocs = randomRootDocs(numDocs); + final FixedBitSet innerDocs = randomInnerDocs(rootDocs); + verifySortedUnsignedLong(multiValues, numDocs, rootDocs, innerDocs, Integer.MAX_VALUE); + verifySortedUnsignedLong(multiValues, numDocs, rootDocs, innerDocs, randomIntBetween(1, numDocs)); + } + private void verifySortedSet(Supplier supplier, int maxDoc) throws IOException { for (MultiValueMode mode : new MultiValueMode[] { MultiValueMode.MIN, MultiValueMode.MAX }) { SortedSetDocValues values = supplier.get(); @@ -857,6 +952,141 @@ private void verifySortedSet( } } + private void verifySortedUnsignedLong(Supplier supplier, int maxDoc) throws IOException { + for (MultiValueMode mode : MultiValueMode.values()) { + SortedNumericUnsignedLongValues values = supplier.get(); + final NumericDocValues selected = mode.select(values); + for (int i = 0; i < maxDoc; ++i) { + Long actual = null; + if (selected.advanceExact(i)) { + actual = selected.longValue(); + verifyLongValueCanCalledMoreThanOnce(selected, actual); + } + + BigInteger expected = null; + if (values.advanceExact(i)) { + int numValues = values.docValueCount(); + if (mode == MultiValueMode.MAX) { + expected = Numbers.MIN_UNSIGNED_LONG_VALUE; + } else if (mode == MultiValueMode.MIN) { + expected = Numbers.MAX_UNSIGNED_LONG_VALUE; + } else { + expected = BigInteger.ZERO; + } + for (int j = 0; j < numValues; ++j) { + if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { + expected = expected.add(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == MultiValueMode.MIN) { + expected = expected.min(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == MultiValueMode.MAX) { + expected = expected.max(Numbers.toUnsignedBigInteger(values.nextValue())); + } + } + if (mode == MultiValueMode.AVG) { + expected = Numbers.toUnsignedBigInteger(expected.longValue()); + expected = numValues > 1 + ? new BigDecimal(expected).divide(new BigDecimal(numValues), RoundingMode.HALF_UP).toBigInteger() + : expected; + } else if (mode == MultiValueMode.MEDIAN) { + final Long[] docValues = new Long[numValues]; + for (int j = 0; j < numValues; ++j) { + docValues[j] = values.nextValue(); + } + Arrays.sort(docValues, Long::compareUnsigned); + int value = numValues / 2; + if (numValues % 2 == 0) { + expected = Numbers.toUnsignedBigInteger(docValues[value - 1]) + .add(Numbers.toUnsignedBigInteger(docValues[value])); + expected = Numbers.toUnsignedBigInteger(expected.longValue()); + expected = new BigDecimal(expected).divide(new BigDecimal(2), RoundingMode.HALF_UP).toBigInteger(); + } else { + expected = Numbers.toUnsignedBigInteger(docValues[value]); + } + } + } + + final Long expectedLong = expected == null ? null : expected.longValue(); + assertEquals(mode.toString() + " docId=" + i, expectedLong, actual); + } + } + } + + private void verifySortedUnsignedLong( + Supplier supplier, + int maxDoc, + FixedBitSet rootDocs, + FixedBitSet innerDocs, + int maxChildren + ) throws IOException { + for (long missingValue : new long[] { 0, randomUnsignedLong().longValue() }) { + for (MultiValueMode mode : new MultiValueMode[] { + MultiValueMode.MIN, + MultiValueMode.MAX, + MultiValueMode.SUM, + MultiValueMode.AVG }) { + SortedNumericUnsignedLongValues values = supplier.get(); + final NumericDocValues selected = mode.select( + values, + missingValue, + rootDocs, + new BitSetIterator(innerDocs, 0L), + maxDoc, + maxChildren + ); + int prevRoot = -1; + for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { + assertTrue(selected.advanceExact(root)); + final long actual = selected.longValue(); + verifyLongValueCanCalledMoreThanOnce(selected, actual); + + BigInteger expected = BigInteger.ZERO; + if (mode == MultiValueMode.MAX) { + expected = Numbers.MIN_UNSIGNED_LONG_VALUE; + } else if (mode == MultiValueMode.MIN) { + expected = Numbers.MAX_UNSIGNED_LONG_VALUE; + } + int numValues = 0; + int count = 0; + for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit( + child + 1 + )) { + if (values.advanceExact(child)) { + if (++count > maxChildren) { + break; + } + for (int j = 0; j < values.docValueCount(); ++j) { + if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { + expected = expected.add(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == MultiValueMode.MIN) { + expected = expected.min(Numbers.toUnsignedBigInteger(values.nextValue())); + } else if (mode == MultiValueMode.MAX) { + expected = expected.max(Numbers.toUnsignedBigInteger(values.nextValue())); + } + ++numValues; + } + } + } + final long expectedLong; + if (numValues == 0) { + expectedLong = missingValue; + } else if (mode == MultiValueMode.AVG) { + expected = Numbers.toUnsignedBigInteger(expected.longValue()); + expected = numValues > 1 + ? new BigDecimal(expected).divide(new BigDecimal(numValues), RoundingMode.HALF_UP).toBigInteger() + : expected; + expectedLong = expected.longValue(); + } else { + expectedLong = expected.longValue(); + } + + assertEquals(mode.toString() + " docId=" + root, expectedLong, actual); + + prevRoot = root; + } + } + } + } + public void testValidOrdinals() { assertThat(MultiValueMode.SUM.ordinal(), equalTo(0)); assertThat(MultiValueMode.AVG.ordinal(), equalTo(1)); From 29c7e46d6a06d868155fd4aa59e84daf65c6c1eb Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 19 Dec 2024 15:32:10 -0500 Subject: [PATCH 07/10] Add CHANGELOG.md, minor cleanups Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + .../java/org/opensearch/index/fielddata/FieldData.java | 7 +++++++ .../SingletonSortedNumericUnsignedLongValues.java | 10 +++++++--- .../UnsignedLongValuesComparatorSource.java | 5 +---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a77833468ab57..48cc84b20d555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Skip remote-repositories validations for node-joins when RepositoriesService is not in sync with cluster-state ([#16763](https://github.com/opensearch-project/OpenSearch/pull/16763)) - Fix _list/shards API failing when closed indices are present ([#16606](https://github.com/opensearch-project/OpenSearch/pull/16606)) - Fix remote shards balance ([#15335](https://github.com/opensearch-project/OpenSearch/pull/15335)) +- Fix multi-value sort for unsigned long ([#16732](https://github.com/opensearch-project/OpenSearch/pull/16732)) ### Security diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java index 61641f65489ae..61e48d05a6263 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java @@ -292,6 +292,13 @@ public static SortedNumericDocValues castToLong(final SortedNumericDoubleValues } } + /** + * Returns a multi-valued view over the provided {@link NumericDoubleValues}. + */ + public static SortedNumericUnsignedLongValues singleton(SortedNumericDocValues values) { + return new SingletonSortedNumericUnsignedLongValues(values); + } + /** * Returns a multi-valued view over the provided {@link NumericDoubleValues}. */ diff --git a/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java b/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java index 05929d08a142c..040b93c2f482f 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java +++ b/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java @@ -37,14 +37,18 @@ import java.io.IOException; /** - * Clone of {@link SortedNumericDocValues} for double values. + * Exposes multi-valued view over a single-valued instance. + *

+ * This can be used if you want to have one multi-valued implementation + * that works for single or multi-valued types. * * @opensearch.internal */ -public final class SingletonSortedNumericUnsignedLongValues extends SortedNumericUnsignedLongValues { + +final class SingletonSortedNumericUnsignedLongValues extends SortedNumericUnsignedLongValues { private final SortedNumericDocValues values; - public SingletonSortedNumericUnsignedLongValues(SortedNumericDocValues values) { + SingletonSortedNumericUnsignedLongValues(SortedNumericDocValues values) { this.values = values; } diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index c9076e773ba29..fe3270d09c422 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.LeafFieldComparator; @@ -24,7 +23,6 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.fielddata.LeafNumericFieldData; -import org.opensearch.index.fielddata.SingletonSortedNumericUnsignedLongValues; import org.opensearch.index.fielddata.SortedNumericUnsignedLongValues; import org.opensearch.index.search.comparators.UnsignedLongComparator; import org.opensearch.search.DocValueFormat; @@ -61,8 +59,7 @@ public SortField.Type reducedType() { private SortedNumericUnsignedLongValues loadDocValues(LeafReaderContext context) { final LeafNumericFieldData data = indexFieldData.load(context); - SortedNumericDocValues values = data.getLongValues(); - return new SingletonSortedNumericUnsignedLongValues(values); + return FieldData.singleton(data.getLongValues()); } private NumericDocValues getNumericDocValues(LeafReaderContext context, BigInteger missingValue) throws IOException { From 9c62128a6bd92e919bcbb073ade409e78f4fdb55 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 20 Dec 2024 17:02:37 -0500 Subject: [PATCH 08/10] Correct the license headers Signed-off-by: Andriy Redko --- ...gletonSortedNumericUnsignedLongValues.java | 24 ------------------- .../SortedNumericUnsignedLongValues.java | 24 ------------------- 2 files changed, 48 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java b/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java index 040b93c2f482f..f2ec4d7589b64 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java +++ b/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.fielddata; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java b/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java index b22e732280fa2..aaa09c3f0dcf9 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java +++ b/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.index.fielddata; import org.apache.lucene.index.SortedNumericDocValues; From cfbf241becfdf6830db0391863bf0cd40a48fb7b Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 20 Dec 2024 17:05:15 -0500 Subject: [PATCH 09/10] Correct the @PublicApi version Signed-off-by: Andriy Redko --- .../index/fielddata/SortedNumericUnsignedLongValues.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java b/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java index aaa09c3f0dcf9..fa4c5152b9f90 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java +++ b/server/src/main/java/org/opensearch/index/fielddata/SortedNumericUnsignedLongValues.java @@ -18,7 +18,7 @@ * * @opensearch.api */ -@PublicApi(since = "1.0.0") +@PublicApi(since = "2.19.0") public abstract class SortedNumericUnsignedLongValues { /** Sole constructor. (For invocation by subclass From bbb18582641b756394c9dcdb2527f2d54300c684 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 2 Jan 2025 13:29:54 -0500 Subject: [PATCH 10/10] Replace SingletonSortedNumericUnsignedLongValues with LongToSortedNumericUnsignedLongValues (as per review comments) Signed-off-by: Andriy Redko --- .../opensearch/index/fielddata/FieldData.java | 19 ------------------- ...ongToSortedNumericUnsignedLongValues.java} | 11 ++++------- .../UnsignedLongValuesComparatorSource.java | 3 ++- .../org/opensearch/search/MultiValueMode.java | 7 ++++++- 4 files changed, 12 insertions(+), 28 deletions(-) rename server/src/main/java/org/opensearch/index/fielddata/{SingletonSortedNumericUnsignedLongValues.java => LongToSortedNumericUnsignedLongValues.java} (74%) diff --git a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java index 61e48d05a6263..6db6bbccacae5 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/FieldData.java @@ -292,13 +292,6 @@ public static SortedNumericDocValues castToLong(final SortedNumericDoubleValues } } - /** - * Returns a multi-valued view over the provided {@link NumericDoubleValues}. - */ - public static SortedNumericUnsignedLongValues singleton(SortedNumericDocValues values) { - return new SingletonSortedNumericUnsignedLongValues(values); - } - /** * Returns a multi-valued view over the provided {@link NumericDoubleValues}. */ @@ -318,18 +311,6 @@ public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues valu return null; } - /** - * Returns a single-valued view of the {@link SortedNumericDoubleValues}, - * if it was previously wrapped with {@link DocValues#singleton(NumericDocValues)}, - * or null. - */ - public static SortedNumericDocValues unwrapSingleton(SortedNumericUnsignedLongValues values) { - if (values instanceof SingletonSortedNumericUnsignedLongValues) { - return ((SingletonSortedNumericUnsignedLongValues) values).getNumericUnsignedLongValues(); - } - return null; - } - /** * Returns a multi-valued view over the provided {@link GeoPointValues}. */ diff --git a/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java b/server/src/main/java/org/opensearch/index/fielddata/LongToSortedNumericUnsignedLongValues.java similarity index 74% rename from server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java rename to server/src/main/java/org/opensearch/index/fielddata/LongToSortedNumericUnsignedLongValues.java index f2ec4d7589b64..eb8d8f1667218 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/SingletonSortedNumericUnsignedLongValues.java +++ b/server/src/main/java/org/opensearch/index/fielddata/LongToSortedNumericUnsignedLongValues.java @@ -13,18 +13,15 @@ import java.io.IOException; /** - * Exposes multi-valued view over a single-valued instance. - *

- * This can be used if you want to have one multi-valued implementation - * that works for single or multi-valued types. + * Wraps long-based {@link SortedNumericDocValues} as unsigned long ones + * (primarily used by {@link org.opensearch.search.MultiValueMode} * * @opensearch.internal */ - -final class SingletonSortedNumericUnsignedLongValues extends SortedNumericUnsignedLongValues { +public final class LongToSortedNumericUnsignedLongValues extends SortedNumericUnsignedLongValues { private final SortedNumericDocValues values; - SingletonSortedNumericUnsignedLongValues(SortedNumericDocValues values) { + public LongToSortedNumericUnsignedLongValues(SortedNumericDocValues values) { this.values = values; } diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java index fe3270d09c422..6fc85bd0b2689 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java @@ -23,6 +23,7 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.fielddata.LeafNumericFieldData; +import org.opensearch.index.fielddata.LongToSortedNumericUnsignedLongValues; import org.opensearch.index.fielddata.SortedNumericUnsignedLongValues; import org.opensearch.index.search.comparators.UnsignedLongComparator; import org.opensearch.search.DocValueFormat; @@ -59,7 +60,7 @@ public SortField.Type reducedType() { private SortedNumericUnsignedLongValues loadDocValues(LeafReaderContext context) { final LeafNumericFieldData data = indexFieldData.load(context); - return FieldData.singleton(data.getLongValues()); + return new LongToSortedNumericUnsignedLongValues(data.getLongValues()); } private NumericDocValues getNumericDocValues(LeafReaderContext context, BigInteger missingValue) throws IOException { diff --git a/server/src/main/java/org/opensearch/search/MultiValueMode.java b/server/src/main/java/org/opensearch/search/MultiValueMode.java index b9df357016788..fa2e776eca67a 100644 --- a/server/src/main/java/org/opensearch/search/MultiValueMode.java +++ b/server/src/main/java/org/opensearch/search/MultiValueMode.java @@ -51,6 +51,7 @@ import org.opensearch.index.fielddata.AbstractNumericDocValues; import org.opensearch.index.fielddata.AbstractSortedDocValues; import org.opensearch.index.fielddata.FieldData; +import org.opensearch.index.fielddata.LongToSortedNumericUnsignedLongValues; import org.opensearch.index.fielddata.NumericDoubleValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; @@ -1240,7 +1241,11 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX */ public NumericDocValues select(final SortedNumericUnsignedLongValues values) { - final SortedNumericDocValues sortedNumericDocValues = FieldData.unwrapSingleton(values); + SortedNumericDocValues sortedNumericDocValues = null; + if (values instanceof LongToSortedNumericUnsignedLongValues) { + sortedNumericDocValues = ((LongToSortedNumericUnsignedLongValues) values).getNumericUnsignedLongValues(); + } + final NumericDocValues singleton = DocValues.unwrapSingleton(sortedNumericDocValues); if (singleton != null) { return singleton;