From 302036595373aabaefdb05de766ba6fadfabb2ff Mon Sep 17 00:00:00 2001 From: Chip Kent <5250374+chipkent@users.noreply.github.com> Date: Mon, 6 May 2024 11:49:45 -0600 Subject: [PATCH 1/7] Fix inconsistencies in aggregations and query library functions. (#5368) Aggregation operations in query library functions and built-in query aggregations are inconsistent. This PR makes them consistent. Query library functions were changed. * `percentile` now returns the primitive type. * `sum` returns a widened type of `double` for floating point inputs or `long` for integer inputs. * `product` returns a widened type of `double` for floating point inputs or `long` for integer inputs. * `cumsum` returns a widened type of `double[]` for floating point inputs or `long[]` for integer inputs. * `cumprod` returns a widened type of `double[]` for floating point inputs or `long[]` for integer inputs. * `wsum` returns a widened type of `long` for all integer inputs and `double` for inputs containing floating points. Note: Because the types have changed, the NULL return values have changed as well. Resolves #4023 --- engine/function/src/templates/Numeric.ftl | 364 +++++++++++++++--- engine/function/src/templates/TestNumeric.ftl | 237 +++++++++--- .../deephaven/libs/GroovyStaticImports.java | 228 +++++------ .../table/impl/updateby/TestCumProd.java | 2 +- .../table/impl/updateby/TestCumSum.java | 2 +- 5 files changed, 617 insertions(+), 216 deletions(-) diff --git a/engine/function/src/templates/Numeric.ftl b/engine/function/src/templates/Numeric.ftl index 5db8a41b0cf..9853bd4ebb8 100644 --- a/engine/function/src/templates/Numeric.ftl +++ b/engine/function/src/templates/Numeric.ftl @@ -1243,9 +1243,9 @@ public class Numeric { * @param values values. * @return percentile, or null value in the Deephaven convention if values is null or empty. */ - public static double percentile(double percentile, ${pt.primitive}... values) { + public static ${pt.primitive} percentile(double percentile, ${pt.primitive}... values) { if (values == null || values.length == 0) { - return NULL_DOUBLE; + return ${pt.null}; } return percentile(percentile, new ${pt.vectorDirect}(values)); @@ -1258,9 +1258,9 @@ public class Numeric { * @param values values. * @return percentile, or null value in the Deephaven convention if values is null or empty. */ - public static double percentile(double percentile, ${pt.vector} values) { + public static ${pt.primitive} percentile(double percentile, ${pt.vector} values) { if (values == null || values.isEmpty()) { - return NULL_DOUBLE; + return ${pt.null}; } if (percentile < 0 || percentile > 1) { @@ -1481,9 +1481,10 @@ public class Numeric { * @param values values. * @return sum of non-null values. */ - public static ${pt.primitive} sum(${pt.vector} values) { + <#if pt.valueType.isFloat > + public static double sum(${pt.vector} values) { if (values == null) { - return ${pt.null}; + return NULL_DOUBLE; } double sum = 0; @@ -1491,19 +1492,40 @@ public class Numeric { try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); - <#if pt.valueType.isFloat > - if (isNaN(c)) { - return ${pt.boxed}.NaN; + + if (isNaN(c) || isNaN(sum)) { + return Double.NaN; } - + + if (!isNull(c)) { + sum += c; + } + } + } + + return sum; + } + <#else> + public static long sum(${pt.vector} values) { + if (values == null) { + return NULL_LONG; + } + + long sum = 0; + + try ( final ${pt.vectorIterator} vi = values.iterator() ) { + while ( vi.hasNext() ) { + final ${pt.primitive} c = vi.${pt.iteratorNext}(); + if (!isNull(c)) { sum += c; } } } - return (${pt.primitive}) (sum); + return sum; } + /** * Returns the sum. Null values are excluded. @@ -1511,13 +1533,23 @@ public class Numeric { * @param values values. * @return sum of non-null values. */ - public static ${pt.primitive} sum(${pt.primitive}... values) { + <#if pt.valueType.isFloat > + public static double sum(${pt.primitive}... values) { if (values == null) { - return ${pt.null}; + return NULL_DOUBLE; } return sum(new ${pt.vectorDirect}(values)); } + <#else> + public static long sum(${pt.primitive}... values) { + if (values == null) { + return NULL_LONG; + } + + return sum(new ${pt.vectorDirect}(values)); + } + /** * Returns the product. Null values are excluded. @@ -1525,40 +1557,65 @@ public class Numeric { * @param values values. * @return product of non-null values. */ - public static ${pt.primitive} product(${pt.vector} values) { + <#if pt.valueType.isFloat > + public static double product(${pt.vector} values) { if (values == null) { - return ${pt.null}; + return NULL_DOUBLE; } - ${pt.primitive} prod = 1; + double prod = 1; int count = 0; - <#if pt.valueType.isFloat > - long zeroCount = 0; - long infCount = 0; - + boolean hasZero = false; + boolean hasInf = false; try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); - <#if pt.valueType.isFloat > + if (isNaN(c)) { - return ${pt.boxed}.NaN; + return Double.NaN; } else if (Double.isInfinite(c)) { - if (zeroCount > 0) { - return ${pt.boxed}.NaN; + if (hasZero) { + return Double.NaN; } - infCount++; + hasInf = true; } else if (c == 0) { - if (infCount > 0) { - return ${pt.boxed}.NaN; + if (hasInf) { + return Double.NaN; } - zeroCount++; + hasZero = true; } + + if (!isNull(c)) { + count++; + prod *= c; + } + } + } + + if (count == 0) { + return NULL_DOUBLE; + } + + return hasZero ? 0 : prod; + } <#else> + public static long product(${pt.vector} values) { + if (values == null) { + return NULL_LONG; + } + + long prod = 1; + int count = 0; + + try ( final ${pt.vectorIterator} vi = values.iterator() ) { + while ( vi.hasNext() ) { + final ${pt.primitive} c = vi.${pt.iteratorNext}(); + if (c == 0) { return 0; } - + if (!isNull(c)) { count++; prod *= c; @@ -1567,15 +1624,12 @@ public class Numeric { } if (count == 0) { - return ${pt.null}; + return NULL_LONG; } - <#if pt.valueType.isFloat > - return zeroCount > 0 ? 0 : (${pt.primitive}) (prod); - <#else> - return (${pt.primitive}) (prod); - + return prod; } + /** * Returns the product. Null values are excluded. @@ -1583,13 +1637,23 @@ public class Numeric { * @param values values. * @return product of non-null values. */ - public static ${pt.primitive} product(${pt.primitive}... values) { + <#if pt.valueType.isFloat > + public static double product(${pt.primitive}... values) { if (values == null) { - return ${pt.null}; + return NULL_DOUBLE; + } + + return product(new ${pt.vectorDirect}(values)); + } + <#else> + public static long product(${pt.primitive}... values) { + if (values == null) { + return NULL_LONG; } return product(new ${pt.vectorDirect}(values)); } + /** * Returns the differences between elements in the input vector separated by a stride. @@ -1792,9 +1856,15 @@ public class Numeric { * @param values values. * @return cumulative sum of non-null values. */ - public static ${pt.primitive}[] cumsum(${pt.boxed}[] values) { + <#if pt.valueType.isFloat > + public static double[] cumsum(${pt.boxed}[] values) { + return cumsum(unbox(values)); + } + <#else> + public static long[] cumsum(${pt.boxed}[] values) { return cumsum(unbox(values)); } + /** * Returns the cumulative sum. Null values are excluded. @@ -1802,13 +1872,23 @@ public class Numeric { * @param values values. * @return cumulative sum of non-null values. */ - public static ${pt.primitive}[] cumsum(${pt.primitive}... values) { + <#if pt.valueType.isFloat > + public static double[] cumsum(${pt.primitive}... values) { if (values == null) { return null; } return cumsum(new ${pt.vectorDirect}(values)); } + <#else> + public static long[] cumsum(${pt.primitive}... values) { + if (values == null) { + return null; + } + + return cumsum(new ${pt.vectorDirect}(values)); + } + /** * Returns the cumulative sum. Null values are excluded. @@ -1816,39 +1896,80 @@ public class Numeric { * @param values values. * @return cumulative sum of non-null values. */ - public static ${pt.primitive}[] cumsum(${pt.vector} values) { + <#if pt.valueType.isFloat > + public static double[] cumsum(${pt.vector} values) { if (values == null) { return null; } if (values.isEmpty()) { - return new ${pt.primitive}[0]; + return new double[0]; } final int n = values.intSize("cumsum"); - ${pt.primitive}[] result = new ${pt.primitive}[n]; + final double[] result = new double[n]; try ( final ${pt.vectorIterator} vi = values.iterator() ) { - result[0] = vi.${pt.iteratorNext}(); + final ${pt.primitive} v0 = vi.${pt.iteratorNext}(); + result[0] = isNull(v0) ? NULL_DOUBLE : v0; int i = 1; while (vi.hasNext()) { final ${pt.primitive} v = vi.${pt.iteratorNext}(); + + if (isNaN(v) || isNaN(result[i - 1])) { + Arrays.fill(result, i, n, Double.NaN); + return result; + } else if (isNull(result[i - 1])) { + result[i] = v; + } else if (isNull(v)) { + result[i] = result[i - 1]; + } else { + result[i] = result[i - 1] + v; + } + i++; + } + } + + return result; + } + <#else> + public static long[] cumsum(${pt.vector} values) { + if (values == null) { + return null; + } + + if (values.isEmpty()) { + return new long[0]; + } + + final int n = values.intSize("cumsum"); + final long[] result = new long[n]; + + try ( final ${pt.vectorIterator} vi = values.iterator() ) { + final ${pt.primitive} v0 = vi.${pt.iteratorNext}(); + result[0] = isNull(v0) ? NULL_LONG : v0; + int i = 1; + + while (vi.hasNext()) { + final ${pt.primitive} v = vi.${pt.iteratorNext}(); + if (isNull(result[i - 1])) { result[i] = v; } else if (isNull(v)) { result[i] = result[i - 1]; } else { - result[i] = (${pt.primitive}) (result[i - 1] + v); + result[i] = result[i - 1] + v; } - + i++; } } return result; } + /** * Returns the cumulative product. Null values are excluded. @@ -1856,9 +1977,15 @@ public class Numeric { * @param values values. * @return cumulative product of non-null values. */ - public static ${pt.primitive}[] cumprod(${pt.boxed}[] values) { + <#if pt.valueType.isFloat > + public static double[] cumprod(${pt.boxed}[] values) { + return cumprod(unbox(values)); + } + <#else> + public static long[] cumprod(${pt.boxed}[] values) { return cumprod(unbox(values)); } + /** * Returns the cumulative product. Null values are excluded. @@ -1866,13 +1993,23 @@ public class Numeric { * @param values values. * @return cumulative product of non-null values. */ - public static ${pt.primitive}[] cumprod(${pt.primitive}... values) { + <#if pt.valueType.isFloat > + public static double[] cumprod(${pt.primitive}... values) { + if (values == null) { + return null; + } + + return cumprod(new ${pt.vectorDirect}(values)); + } + <#else> + public static long[] cumprod(${pt.primitive}... values) { if (values == null) { return null; } return cumprod(new ${pt.vectorDirect}(values)); } + /** * Returns the cumulative product. Null values are excluded. @@ -1880,39 +2017,80 @@ public class Numeric { * @param values values. * @return cumulative product of non-null values. */ - public static ${pt.primitive}[] cumprod(${pt.vector} values) { + <#if pt.valueType.isFloat > + public static double[] cumprod(${pt.vector} values) { if (values == null) { return null; } if (values.isEmpty()) { - return new ${pt.primitive}[0]; + return new double[0]; } final int n = values.intSize("cumprod"); - ${pt.primitive}[] result = new ${pt.primitive}[n]; + double[] result = new double[n]; try ( final ${pt.vectorIterator} vi = values.iterator() ) { - result[0] = vi.${pt.iteratorNext}(); + final ${pt.primitive} v0 = vi.${pt.iteratorNext}(); + result[0] = isNull(v0) ? NULL_DOUBLE : v0; int i = 1; while (vi.hasNext()) { final ${pt.primitive} v = vi.${pt.iteratorNext}(); + + if (isNaN(v) || isNaN(result[i - 1])) { + Arrays.fill(result, i, n, Double.NaN); + return result; + } else if (isNull(result[i - 1])) { + result[i] = v; + } else if (isNull(v)) { + result[i] = result[i - 1]; + } else { + result[i] = result[i - 1] * v; + } + i++; + } + } + + return result; + } + <#else> + public static long[] cumprod(${pt.vector} values) { + if (values == null) { + return null; + } + + if (values.isEmpty()) { + return new long[0]; + } + + final int n = values.intSize("cumprod"); + long[] result = new long[n]; + + try ( final ${pt.vectorIterator} vi = values.iterator() ) { + final ${pt.primitive} v0 = vi.${pt.iteratorNext}(); + result[0] = isNull(v0) ? NULL_LONG : v0; + int i = 1; + + while (vi.hasNext()) { + final ${pt.primitive} v = vi.${pt.iteratorNext}(); + if (isNull(result[i - 1])) { result[i] = v; } else if (isNull(v)) { result[i] = result[i - 1]; } else { - result[i] = (${pt.primitive}) (result[i - 1] * v); + result[i] = result[i - 1] * v; } - + i++; } } return result; } + /** * Returns the absolute value. @@ -2319,13 +2497,23 @@ public class Numeric { * @param weights weights * @return weighted sum of non-null values. */ - public static double wsum(${pt.primitive}[] values, ${pt2.primitive}[] weights) { + <#if pt.valueType.isInteger && pt2.valueType.isInteger > + public static long wsum(${pt.primitive}[] values, ${pt2.primitive}[] weights) { + if (values == null || weights == null) { + return NULL_LONG; + } + + return wsum(new ${pt.vectorDirect}(values), new ${pt2.vector}Direct(weights)); + } + <#else> + public static double wsum(${pt.primitive}[] values, ${pt2.primitive}[] weights) { if (values == null || weights == null) { return NULL_DOUBLE; } return wsum(new ${pt.vectorDirect}(values), new ${pt2.vector}Direct(weights)); } + /** * Returns the weighted sum. Null values are excluded. @@ -2334,6 +2522,15 @@ public class Numeric { * @param weights weights * @return weighted sum of non-null values. */ + <#if pt.valueType.isInteger && pt2.valueType.isInteger > + public static long wsum(${pt.primitive}[] values, ${pt2.vector} weights) { + if (values == null || weights == null) { + return NULL_LONG; + } + + return wsum(new ${pt.vectorDirect}(values), weights); + } + <#else> public static double wsum(${pt.primitive}[] values, ${pt2.vector} weights) { if (values == null || weights == null) { return NULL_DOUBLE; @@ -2341,6 +2538,7 @@ public class Numeric { return wsum(new ${pt.vectorDirect}(values), weights); } + /** * Returns the weighted sum. Null values are excluded. @@ -2349,6 +2547,15 @@ public class Numeric { * @param weights weights * @return weighted sum of non-null values. */ + <#if pt.valueType.isInteger && pt2.valueType.isInteger > + public static long wsum(${pt.vector} values, ${pt2.primitive}[] weights) { + if (values == null || weights == null) { + return NULL_LONG; + } + + return wsum(values, new ${pt2.vector}Direct(weights)); + } + <#else> public static double wsum(${pt.vector} values, ${pt2.primitive}[] weights) { if (values == null || weights == null) { return NULL_DOUBLE; @@ -2356,6 +2563,7 @@ public class Numeric { return wsum(values, new ${pt2.vector}Direct(weights)); } + /** * Returns the weighted sum. Null values are excluded. @@ -2364,6 +2572,37 @@ public class Numeric { * @param weights weights * @return weighted sum of non-null values. */ + <#if pt.valueType.isInteger && pt2.valueType.isInteger > + public static long wsum(${pt.vector} values, ${pt2.vector} weights) { + if (values == null || weights == null) { + return NULL_LONG; + } + + final long n = values.size(); + + if (n != weights.size()) { + throw new IllegalArgumentException("Incompatible input sizes: " + values.size() + ", " + weights.size()); + } + + long vsum = 0; + + try ( + final ${pt.vectorIterator} vi = values.iterator(); + final ${pt2.vectorIterator} wi = weights.iterator() + ) { + while (vi.hasNext()) { + final ${pt.primitive} c = vi.${pt.iteratorNext}(); + final ${pt2.primitive} w = wi.${pt2.iteratorNext}(); + + if (!isNull(c) && !isNull(w)) { + vsum += c * (long) w; + } + } + } + + return vsum; + } + <#else> public static double wsum(${pt.vector} values, ${pt2.vector} weights) { if (values == null || weights == null) { return NULL_DOUBLE; @@ -2384,21 +2623,36 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); final ${pt2.primitive} w = wi.${pt2.iteratorNext}(); + + if (isNaN(vsum)) { + return Double.NaN; + } + + <#if pt.valueType.isFloat > if (isNaN(c)) { return Double.NaN; } + + + <#if pt2.valueType.isFloat > if (isNaN(w)) { return Double.NaN; } + if (!isNull(c) && !isNull(w)) { - vsum += c * w; + <#if pt.valueType.isFloat > + vsum += (double) c * w; + <#else> + vsum += c * (double) w; + } } } return vsum; } + /** * Returns the weighted average. Null values are excluded. diff --git a/engine/function/src/templates/TestNumeric.ftl b/engine/function/src/templates/TestNumeric.ftl index 6918a9bbd77..41855aa6710 100644 --- a/engine/function/src/templates/TestNumeric.ftl +++ b/engine/function/src/templates/TestNumeric.ftl @@ -570,7 +570,17 @@ public class TestNumeric extends BaseArrayTestCase { assertTrue(Math.abs(0 - sum(new ${pt.vectorDirect}())) == 0.0); assertTrue(Math.abs(0 - sum(new ${pt.vectorDirect}(${pt.null}))) == 0.0); assertTrue(Math.abs(20 - sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{5, ${pt.null}, 15}))) == 0.0); - assertEquals(${pt.null}, sum((${pt.vector}) null)); + <#if pt.valueType.isFloat > + assertEquals(NULL_DOUBLE, sum((${pt.vector}) null)); + assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, 6}))); + assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY}))); + assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, 6}))); + assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY}))); + assertEquals(Double.NaN, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY}))); + <#else> + assertEquals(NULL_LONG, sum((${pt.vector}) null)); + + } public void test${pt.boxed}Sum2() { @@ -578,7 +588,16 @@ public class TestNumeric extends BaseArrayTestCase { assertTrue(Math.abs(0 - sum(new ${pt.primitive}[]{})) == 0.0); assertTrue(Math.abs(0 - sum(new ${pt.primitive}[]{${pt.null}})) == 0.0); assertTrue(Math.abs(20 - sum(new ${pt.primitive}[]{5, ${pt.null}, 15})) == 0.0); - assertEquals(${pt.null}, sum((${pt.primitive}[]) null)); + <#if pt.valueType.isFloat > + assertEquals(NULL_DOUBLE, sum((${pt.primitive}[]) null)); + assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, 6})); + assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY})); + assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, 6})); + assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY})); + assertEquals(Double.NaN, sum(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY})); + <#else> + assertEquals(NULL_LONG, sum((${pt.primitive}[]) null)); + } // public void test${pt.boxed}SumObjectVector() { @@ -608,48 +627,77 @@ public class TestNumeric extends BaseArrayTestCase { // } public void test${pt.boxed}Product() { + <#if pt.valueType.isFloat > + final double nullResult = NULL_DOUBLE; + final double zeroValue = 0.0; + <#else> + final long nullResult = NULL_LONG; + final long zeroValue = 0; + + assertTrue(Math.abs(120 - product(new ${pt.primitive}[]{4, 5, 6})) == 0.0); - assertEquals(${pt.null}, product(new ${pt.primitive}[]{})); - assertEquals(${pt.null}, product(new ${pt.primitive}[]{${pt.null}})); + assertEquals(nullResult, product(new ${pt.primitive}[]{})); + assertEquals(nullResult, product(new ${pt.primitive}[]{${pt.null}})); assertTrue(Math.abs(75 - product(new ${pt.primitive}[]{5, ${pt.null}, 15})) == 0.0); - assertEquals(${pt.null}, product((${pt.primitive}[]) null)); + assertEquals(nullResult, product((${pt.primitive}[]) null)); + assertEquals(zeroValue, product(new ${pt.primitive}[]{4, 0, 5, 6})); assertTrue(Math.abs(120 - product(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, 5, 6}))) == 0.0); - assertEquals(${pt.null}, product(new ${pt.vectorDirect}())); - assertEquals(${pt.null}, product(new ${pt.vectorDirect}(${pt.null}))); + assertEquals(nullResult, product(new ${pt.vectorDirect}())); + assertEquals(nullResult, product(new ${pt.vectorDirect}(${pt.null}))); assertTrue(Math.abs(75 - product(new ${pt.vectorDirect}(new ${pt.primitive}[]{5, ${pt.null}, 15}))) == 0.0); - assertEquals(${pt.null}, product((${pt.vector}) null)); + assertEquals(nullResult, product((${pt.vector}) null)); + assertEquals(zeroValue, product(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, 0, 5, 6}))); + + + <#if pt.valueType.isFloat > + assertEquals(Double.POSITIVE_INFINITY, product(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, 6})); + assertEquals(Double.POSITIVE_INFINITY, product(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY})); + assertEquals(Double.NEGATIVE_INFINITY, product(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, 6})); + assertEquals(Double.POSITIVE_INFINITY, product(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY})); + assertEquals(Double.NEGATIVE_INFINITY, product(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY})); + + assertEquals(Double.POSITIVE_INFINITY, product(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, 6}))); + assertEquals(Double.POSITIVE_INFINITY, product(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY}))); + assertEquals(Double.NEGATIVE_INFINITY, product(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, 6}))); + assertEquals(Double.POSITIVE_INFINITY, product(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY}))); + assertEquals(Double.NEGATIVE_INFINITY, product(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY}))); + } <#if pt.valueType.isFloat > public void test${pt.boxed}ProductOverflowAndNaN() { + <#if pt.primitive == "double" > + final ${pt.primitive} LARGE_VALUE = Math.nextDown(${pt.boxed}.MAX_VALUE); final ${pt.primitive}[] overflow = new ${pt.primitive}[]{1, LARGE_VALUE, LARGE_VALUE}; - final ${pt.primitive} overflowProduct = product(overflow); - assertTrue(${pt.boxed}.isInfinite(overflowProduct) && overflowProduct > 0); + final double overflowProduct = product(overflow); + assertTrue(Double.isInfinite(overflowProduct) && overflowProduct > 0); final ${pt.primitive}[] negOverflow = new ${pt.primitive}[]{1, LARGE_VALUE, -LARGE_VALUE}; - final ${pt.primitive} negOverflowProduct = product(negOverflow); - assertTrue(${pt.boxed}.isInfinite(negOverflowProduct) && negOverflowProduct < 0); + final double negOverflowProduct = product(negOverflow); + assertTrue(Double.isInfinite(negOverflowProduct) && negOverflowProduct < 0); final ${pt.primitive}[] overflowWithZero = new ${pt.primitive}[]{1, LARGE_VALUE, LARGE_VALUE, 0}; assertTrue(Math.abs(product(overflowWithZero)) == 0.0); + + final ${pt.primitive}[] normalWithNaN = new ${pt.primitive}[]{1, 2, 3, ${pt.boxed}.NaN, 4, 5}; - assertTrue(${pt.boxed}.isNaN(product(normalWithNaN))); + assertTrue(Double.isNaN(product(normalWithNaN))); final ${pt.primitive}[] posInfAndZero = new ${pt.primitive}[]{1, ${pt.boxed}.POSITIVE_INFINITY, 0}; - assertTrue(${pt.boxed}.isNaN(product(posInfAndZero))); + assertTrue(Double.isNaN(product(posInfAndZero))); final ${pt.primitive}[] negInfAndZero = new ${pt.primitive}[]{1, ${pt.boxed}.NEGATIVE_INFINITY, 0}; - assertTrue(${pt.boxed}.isNaN(product(negInfAndZero))); + assertTrue(Double.isNaN(product(negInfAndZero))); final ${pt.primitive}[] zeroAndPosInf = new ${pt.primitive}[]{1, 0, ${pt.boxed}.POSITIVE_INFINITY}; - assertTrue(${pt.boxed}.isNaN(product(zeroAndPosInf))); + assertTrue(Double.isNaN(product(zeroAndPosInf))); final ${pt.primitive}[] zeroAndNegInf = new ${pt.primitive}[]{1, 0, ${pt.boxed}.NEGATIVE_INFINITY}; - assertTrue(${pt.boxed}.isNaN(product(zeroAndNegInf))); + assertTrue(Double.isNaN(product(zeroAndNegInf))); } @@ -747,41 +795,101 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(new ${pt.primitive}[]{5, 5, 5, 5, 5}, cummax((${pt.primitive})5, (${pt.primitive})4, (${pt.primitive})3, (${pt.primitive})2, (${pt.primitive})1)); } +<#if pt.valueType.isFloat > public void test${pt.boxed}CumSumArray() { - assertEquals(new ${pt.primitive}[]{1, 3, 6, 10, 15}, cumsum(new ${pt.primitive}[]{1, 2, 3, 4, 5})); - assertEquals(new ${pt.primitive}[]{1, 3, 6, 6, 11}, cumsum(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); - assertEquals(new ${pt.primitive}[]{${pt.null}, 2, 5, 9, 14}, cumsum(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); - assertEquals(new ${pt.primitive}[0], cumsum(new ${pt.primitive}[0])); - assertEquals(new ${pt.primitive}[0], cumsum(new ${pt.boxed}[0])); + assertEquals(new double[]{1, 3, 6, 10, 15}, cumsum(new ${pt.primitive}[]{1, 2, 3, 4, 5})); + assertEquals(new double[]{1, 3, 6, 6, 11}, cumsum(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); + assertEquals(new double[]{NULL_DOUBLE, 2, 5, 9, 14}, cumsum(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); + assertEquals(new double[0], cumsum(new ${pt.primitive}[0])); + assertEquals(new double[0], cumsum(new ${pt.boxed}[0])); assertEquals(null, cumsum((${pt.primitive}[]) null)); - assertEquals(new ${pt.primitive}[]{1, 3, 6, 10, 15}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); - assertEquals(new ${pt.primitive}[]{1, 3, 6, 6, 11}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); - assertEquals(new ${pt.primitive}[]{${pt.null}, 2, 5, 9, 14}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); - assertEquals(new ${pt.primitive}[0], cumsum(new ${pt.vectorDirect}())); + assertEquals(new double[]{1, 3, 6, 10, 15}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); + assertEquals(new double[]{1, 3, 6, 6, 11}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); + assertEquals(new double[]{NULL_DOUBLE, 2, 5, 9, 14}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); + assertEquals(new double[0], cumsum(new ${pt.vectorDirect}())); assertEquals(null, cumsum((${pt.vector}) null)); // check that functions can be resolved with varargs - assertEquals(new ${pt.primitive}[]{1, 3, 6, 10, 15}, cumsum((${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})3, (${pt.primitive})4, (${pt.primitive})5)); + assertEquals(new double[]{1, 3, 6, 10, 15}, cumsum((${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})3, (${pt.primitive})4, (${pt.primitive})5)); + + assertEquals(new double[]{1, 3, 6, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, 5})); + assertEquals(new double[]{1, 3, 6, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, -5})); + assertEquals(new double[]{1, 3, 6, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY})); + assertEquals(new double[]{1, 3, 6, Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, 5})); + assertEquals(new double[]{1, 3, 6, Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, -5})); + assertEquals(new double[]{1, 3, 6, Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY})); + assertEquals(new double[]{1, 3, 6, Double.POSITIVE_INFINITY, Double.NaN}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY})); + assertEquals(new double[]{1, 3, 6, Double.NEGATIVE_INFINITY, Double.NaN}, cumsum(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY})); + } +<#else> + public void test${pt.boxed}CumSumArray() { + assertEquals(new long[]{1, 3, 6, 10, 15}, cumsum(new ${pt.primitive}[]{1, 2, 3, 4, 5})); + assertEquals(new long[]{1, 3, 6, 6, 11}, cumsum(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); + assertEquals(new long[]{NULL_LONG, 2, 5, 9, 14}, cumsum(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); + assertEquals(new long[0], cumsum(new ${pt.primitive}[0])); + assertEquals(new long[0], cumsum(new ${pt.boxed}[0])); + assertEquals(null, cumsum((${pt.primitive}[]) null)); + + assertEquals(new long[]{1, 3, 6, 10, 15}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); + assertEquals(new long[]{1, 3, 6, 6, 11}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); + assertEquals(new long[]{NULL_LONG, 2, 5, 9, 14}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); + assertEquals(new long[0], cumsum(new ${pt.vectorDirect}())); + assertEquals(null, cumsum((${pt.vector}) null)); + + // check that functions can be resolved with varargs + assertEquals(new long[]{1, 3, 6, 10, 15}, cumsum((${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})3, (${pt.primitive})4, (${pt.primitive})5)); } + + +<#if pt.valueType.isFloat > + public void test${pt.boxed}CumProdArray() { + assertEquals(new double[]{1, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{1, 2, 3, 4, 5})); + assertEquals(new double[]{1, 2, 6, 6, 30}, cumprod(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); + assertEquals(new double[]{NULL_DOUBLE, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); + assertEquals(new double[0], cumprod(new ${pt.primitive}[0])); + assertEquals(new double[0], cumprod(new ${pt.boxed}[0])); + assertEquals(null, cumprod((${pt.primitive}[]) null)); + assertEquals(new double[]{1, Double.NaN, Double.NaN, Double.NaN, Double.NaN}, cumprod(new ${pt.primitive}[]{1, Float.NaN, 3, 4, 5})); + assertEquals(new double[]{1, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); + assertEquals(new double[]{1, 2, 6, 6, 30}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); + assertEquals(new double[]{NULL_DOUBLE, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); + assertEquals(new double[0], cumprod(new ${pt.vectorDirect}())); + assertEquals(null, cumprod((${pt.vector}) null)); + assertEquals(new double[]{1, Double.NaN, Double.NaN, Double.NaN, Double.NaN}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, Float.NaN, 3, 4, 5}))); + + // check that functions can be resolved with varargs + assertEquals(new double[]{1, 2, 6, 24, 120}, cumprod((${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})3, (${pt.primitive})4, (${pt.primitive})5)); + + assertEquals(new double[]{1, 2, 6, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, 5})); + assertEquals(new double[]{1, 2, 6, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, -5})); + assertEquals(new double[]{1, 2, 6, Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY})); + assertEquals(new double[]{1, 2, 6, Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, 5})); + assertEquals(new double[]{1, 2, 6, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, -5})); + assertEquals(new double[]{1, 2, 6, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY})); + assertEquals(new double[]{1, 2, 6, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY})); + assertEquals(new double[]{1, 2, 6, Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY}, cumprod(new ${pt.primitive}[]{1, 2, 3, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY})); + } +<#else> public void test${pt.boxed}CumProdArray() { - assertEquals(new ${pt.primitive}[]{1, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{1, 2, 3, 4, 5})); - assertEquals(new ${pt.primitive}[]{1, 2, 6, 6, 30}, cumprod(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); - assertEquals(new ${pt.primitive}[]{${pt.null}, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); - assertEquals(new ${pt.primitive}[0], cumprod(new ${pt.primitive}[0])); - assertEquals(new ${pt.primitive}[0], cumprod(new ${pt.boxed}[0])); + assertEquals(new long[]{1, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{1, 2, 3, 4, 5})); + assertEquals(new long[]{1, 2, 6, 6, 30}, cumprod(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); + assertEquals(new long[]{NULL_LONG, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); + assertEquals(new long[0], cumprod(new ${pt.primitive}[0])); + assertEquals(new long[0], cumprod(new ${pt.boxed}[0])); assertEquals(null, cumprod((${pt.primitive}[]) null)); - assertEquals(new ${pt.primitive}[]{1, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); - assertEquals(new ${pt.primitive}[]{1, 2, 6, 6, 30}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); - assertEquals(new ${pt.primitive}[]{${pt.null}, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); - assertEquals(new ${pt.primitive}[0], cumprod(new ${pt.vectorDirect}())); + assertEquals(new long[]{1, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); + assertEquals(new long[]{1, 2, 6, 6, 30}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); + assertEquals(new long[]{NULL_LONG, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); + assertEquals(new long[0], cumprod(new ${pt.vectorDirect}())); assertEquals(null, cumprod((${pt.vector}) null)); // check that functions can be resolved with varargs - assertEquals(new ${pt.primitive}[]{1, 2, 6, 24, 120}, cumprod((${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})3, (${pt.primitive})4, (${pt.primitive})5)); + assertEquals(new long[]{1, 2, 6, 24, 120}, cumprod((${pt.primitive})1, (${pt.primitive})2, (${pt.primitive})3, (${pt.primitive})4, (${pt.primitive})5)); } + public void test${pt.boxed}Abs() { ${pt.primitive} value = -5; @@ -992,15 +1100,15 @@ public class TestNumeric extends BaseArrayTestCase { } public void test${pt.boxed}Percentile() { - assertEquals(2.0, percentile(0.00, new ${pt.primitive}[]{4,2,3})); - assertEquals(3.0, percentile(0.50, new ${pt.primitive}[]{4,2,3})); - assertEquals(NULL_DOUBLE, percentile(0.25, (${pt.primitive}[])null)); - assertEquals(NULL_DOUBLE, percentile(0.25, new ${pt.primitive}[]{})); + assertEquals((${pt.primitive})2, percentile(0.00, new ${pt.primitive}[]{4,2,3})); + assertEquals((${pt.primitive})3, percentile(0.50, new ${pt.primitive}[]{4,2,3})); + assertEquals(${pt.null}, percentile(0.25, (${pt.primitive}[])null)); + assertEquals(${pt.null}, percentile(0.25, new ${pt.primitive}[]{})); - assertEquals(2.0, percentile(0.00, new ${pt.vectorDirect}(new ${pt.primitive}[]{4,2,3}))); - assertEquals(3.0, percentile(0.50, new ${pt.vectorDirect}(new ${pt.primitive}[]{4,2,3}))); - assertEquals(NULL_DOUBLE, percentile(0.25, (${pt.vector}) null)); - assertEquals(NULL_DOUBLE, percentile(0.50, new ${pt.vectorDirect}(new ${pt.primitive}[]{}))); + assertEquals((${pt.primitive})2, percentile(0.00, new ${pt.vectorDirect}(new ${pt.primitive}[]{4,2,3}))); + assertEquals((${pt.primitive})3, percentile(0.50, new ${pt.vectorDirect}(new ${pt.primitive}[]{4,2,3}))); + assertEquals(${pt.null}, percentile(0.25, (${pt.vector}) null)); + assertEquals(${pt.null}, percentile(0.50, new ${pt.vectorDirect}(new ${pt.primitive}[]{}))); try { percentile(-1, new ${pt.primitive}[]{4,2,3}); @@ -1022,6 +1130,30 @@ public class TestNumeric extends BaseArrayTestCase { <#list primitiveTypes as pt2> <#if pt2.valueType.isNumber > + <#if pt.valueType.isInteger && pt2.valueType.isInteger > + assertEquals(1*4+2*5+3*6, wsum(new ${pt.primitive}[]{1,2,3,${pt.null},5}, new ${pt2.primitive}[]{4,5,6,7,${pt2.null}})); + assertEquals(NULL_LONG, wsum((${pt.primitive}[])null, new ${pt2.primitive}[]{4,5,6})); + assertEquals(NULL_LONG, wsum(new ${pt.primitive}[]{1,2,3}, (${pt2.primitive}[])null)); + + assertEquals(1*4+2*5+3*6, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.primitive}[]{4,5,6,7,${pt2.null}})); + assertEquals(NULL_LONG, wsum((${pt.vector}) null, new ${pt2.primitive}[]{4,5,6})); + assertEquals(NULL_LONG, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3}), (${pt2.primitive}[])null)); + + assertEquals(1*4+2*5+3*6, wsum(new ${pt.primitive}[]{1,2,3,${pt.null},5}, new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,7,${pt2.null}}))); + assertEquals(NULL_LONG, wsum((${pt.primitive}[])null, new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6}))); + assertEquals(NULL_LONG, wsum(new ${pt.primitive}[]{1,2,3}, (${pt2.vector}) null)); + + assertEquals(1*4+2*5+3*6, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,7,${pt2.null}}))); + assertEquals(NULL_LONG, wsum((${pt.vector}) null, new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6}))); + assertEquals(NULL_LONG, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3}), (${pt2.vector}) null)); + + try { + wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5})); + fail("Mismatched arguments"); + } catch(IllegalArgumentException e){ + // pass + } + <#else> assertEquals(1.0*4.0+2.0*5.0+3.0*6.0, wsum(new ${pt.primitive}[]{1,2,3,${pt.null},5}, new ${pt2.primitive}[]{4,5,6,7,${pt2.null}})); assertEquals(NULL_DOUBLE, wsum((${pt.primitive}[])null, new ${pt2.primitive}[]{4,5,6})); assertEquals(NULL_DOUBLE, wsum(new ${pt.primitive}[]{1,2,3}, (${pt2.primitive}[])null)); @@ -1038,12 +1170,27 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(NULL_DOUBLE, wsum((${pt.vector}) null, new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6}))); assertEquals(NULL_DOUBLE, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3}), (${pt2.vector}) null)); + <#if pt.valueType.isFloat > + assertEquals(Double.NaN, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},Float.NaN}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,7,${pt2.null}}))); + assertEquals(Double.POSITIVE_INFINITY, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},Float.POSITIVE_INFINITY}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,7,8}))); + assertEquals(Double.NEGATIVE_INFINITY, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},Float.NEGATIVE_INFINITY}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,7,8}))); + assertEquals(Double.NaN, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,Float.POSITIVE_INFINITY,Float.NEGATIVE_INFINITY}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,7,8}))); + + + <#if pt2.valueType.isFloat > + assertEquals(Double.NaN, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,Float.NaN,${pt2.null}}))); + assertEquals(Double.POSITIVE_INFINITY, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,4,5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,Float.POSITIVE_INFINITY,${pt2.null}}))); + assertEquals(Double.NEGATIVE_INFINITY, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,4,5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,Float.NEGATIVE_INFINITY,${pt2.null}}))); + assertEquals(Double.NaN, wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,4,5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5,6,Float.NEGATIVE_INFINITY,Float.POSITIVE_INFINITY}))); + + try { wsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1,2,3,${pt.null},5}), new ${pt2.vectorDirect}(new ${pt2.primitive}[]{4,5})); fail("Mismatched arguments"); } catch(IllegalArgumentException e){ // pass } + diff --git a/engine/table/src/main/java/io/deephaven/libs/GroovyStaticImports.java b/engine/table/src/main/java/io/deephaven/libs/GroovyStaticImports.java index 9ad4146b6f5..fcdd8102b47 100644 --- a/engine/table/src/main/java/io/deephaven/libs/GroovyStaticImports.java +++ b/engine/table/src/main/java/io/deephaven/libs/GroovyStaticImports.java @@ -2015,112 +2015,112 @@ public class GroovyStaticImports { public static short[] cummin( io.deephaven.vector.ShortVector values ) {return Numeric.cummin( values );} /** @see io.deephaven.function.Numeric#cumprod(byte[]) */ - public static byte[] cumprod( byte... values ) {return Numeric.cumprod( values );} + public static long[] cumprod( byte... values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(double[]) */ public static double[] cumprod( double... values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(float[]) */ - public static float[] cumprod( float... values ) {return Numeric.cumprod( values );} + public static double[] cumprod( float... values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(int[]) */ - public static int[] cumprod( int... values ) {return Numeric.cumprod( values );} + public static long[] cumprod( int... values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(long[]) */ public static long[] cumprod( long... values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(java.lang.Byte[]) */ - public static byte[] cumprod( java.lang.Byte[] values ) {return Numeric.cumprod( values );} + public static long[] cumprod( java.lang.Byte[] values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(java.lang.Double[]) */ public static double[] cumprod( java.lang.Double[] values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(java.lang.Float[]) */ - public static float[] cumprod( java.lang.Float[] values ) {return Numeric.cumprod( values );} + public static double[] cumprod( java.lang.Float[] values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(java.lang.Integer[]) */ - public static int[] cumprod( java.lang.Integer[] values ) {return Numeric.cumprod( values );} + public static long[] cumprod( java.lang.Integer[] values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(java.lang.Long[]) */ public static long[] cumprod( java.lang.Long[] values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(java.lang.Short[]) */ - public static short[] cumprod( java.lang.Short[] values ) {return Numeric.cumprod( values );} + public static long[] cumprod( java.lang.Short[] values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(short[]) */ - public static short[] cumprod( short... values ) {return Numeric.cumprod( values );} + public static long[] cumprod( short... values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(io.deephaven.vector.ByteVector) */ - public static byte[] cumprod( io.deephaven.vector.ByteVector values ) {return Numeric.cumprod( values );} + public static long[] cumprod( io.deephaven.vector.ByteVector values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(io.deephaven.vector.DoubleVector) */ public static double[] cumprod( io.deephaven.vector.DoubleVector values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(io.deephaven.vector.FloatVector) */ - public static float[] cumprod( io.deephaven.vector.FloatVector values ) {return Numeric.cumprod( values );} + public static double[] cumprod( io.deephaven.vector.FloatVector values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(io.deephaven.vector.IntVector) */ - public static int[] cumprod( io.deephaven.vector.IntVector values ) {return Numeric.cumprod( values );} + public static long[] cumprod( io.deephaven.vector.IntVector values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(io.deephaven.vector.LongVector) */ public static long[] cumprod( io.deephaven.vector.LongVector values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumprod(io.deephaven.vector.ShortVector) */ - public static short[] cumprod( io.deephaven.vector.ShortVector values ) {return Numeric.cumprod( values );} + public static long[] cumprod( io.deephaven.vector.ShortVector values ) {return Numeric.cumprod( values );} /** @see io.deephaven.function.Numeric#cumsum(byte[]) */ - public static byte[] cumsum( byte... values ) {return Numeric.cumsum( values );} + public static long[] cumsum( byte... values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(double[]) */ public static double[] cumsum( double... values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(float[]) */ - public static float[] cumsum( float... values ) {return Numeric.cumsum( values );} + public static double[] cumsum( float... values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(int[]) */ - public static int[] cumsum( int... values ) {return Numeric.cumsum( values );} + public static long[] cumsum( int... values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(long[]) */ public static long[] cumsum( long... values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(java.lang.Byte[]) */ - public static byte[] cumsum( java.lang.Byte[] values ) {return Numeric.cumsum( values );} + public static long[] cumsum( java.lang.Byte[] values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(java.lang.Double[]) */ public static double[] cumsum( java.lang.Double[] values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(java.lang.Float[]) */ - public static float[] cumsum( java.lang.Float[] values ) {return Numeric.cumsum( values );} + public static double[] cumsum( java.lang.Float[] values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(java.lang.Integer[]) */ - public static int[] cumsum( java.lang.Integer[] values ) {return Numeric.cumsum( values );} + public static long[] cumsum( java.lang.Integer[] values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(java.lang.Long[]) */ public static long[] cumsum( java.lang.Long[] values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(java.lang.Short[]) */ - public static short[] cumsum( java.lang.Short[] values ) {return Numeric.cumsum( values );} + public static long[] cumsum( java.lang.Short[] values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(short[]) */ - public static short[] cumsum( short... values ) {return Numeric.cumsum( values );} + public static long[] cumsum( short... values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(io.deephaven.vector.ByteVector) */ - public static byte[] cumsum( io.deephaven.vector.ByteVector values ) {return Numeric.cumsum( values );} + public static long[] cumsum( io.deephaven.vector.ByteVector values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(io.deephaven.vector.DoubleVector) */ public static double[] cumsum( io.deephaven.vector.DoubleVector values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(io.deephaven.vector.FloatVector) */ - public static float[] cumsum( io.deephaven.vector.FloatVector values ) {return Numeric.cumsum( values );} + public static double[] cumsum( io.deephaven.vector.FloatVector values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(io.deephaven.vector.IntVector) */ - public static int[] cumsum( io.deephaven.vector.IntVector values ) {return Numeric.cumsum( values );} + public static long[] cumsum( io.deephaven.vector.IntVector values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(io.deephaven.vector.LongVector) */ public static long[] cumsum( io.deephaven.vector.LongVector values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#cumsum(io.deephaven.vector.ShortVector) */ - public static short[] cumsum( io.deephaven.vector.ShortVector values ) {return Numeric.cumsum( values );} + public static long[] cumsum( io.deephaven.vector.ShortVector values ) {return Numeric.cumsum( values );} /** @see io.deephaven.function.Numeric#diff(int,byte[]) */ public static byte[] diff( int stride, byte... values ) {return Numeric.diff( stride, values );} @@ -3311,40 +3311,40 @@ public class GroovyStaticImports { public static long parseUnsignedLong( java.lang.String s, int radix ) {return Parse.parseUnsignedLong( s, radix );} /** @see io.deephaven.function.Numeric#percentile(double,byte[]) */ - public static double percentile( double percentile, byte... values ) {return Numeric.percentile( percentile, values );} + public static byte percentile( double percentile, byte... values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,double[]) */ public static double percentile( double percentile, double... values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,float[]) */ - public static double percentile( double percentile, float... values ) {return Numeric.percentile( percentile, values );} + public static float percentile( double percentile, float... values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,int[]) */ - public static double percentile( double percentile, int... values ) {return Numeric.percentile( percentile, values );} + public static int percentile( double percentile, int... values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,long[]) */ - public static double percentile( double percentile, long... values ) {return Numeric.percentile( percentile, values );} + public static long percentile( double percentile, long... values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,short[]) */ - public static double percentile( double percentile, short... values ) {return Numeric.percentile( percentile, values );} + public static short percentile( double percentile, short... values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,io.deephaven.vector.ByteVector) */ - public static double percentile( double percentile, io.deephaven.vector.ByteVector values ) {return Numeric.percentile( percentile, values );} + public static byte percentile( double percentile, io.deephaven.vector.ByteVector values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,io.deephaven.vector.DoubleVector) */ public static double percentile( double percentile, io.deephaven.vector.DoubleVector values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,io.deephaven.vector.FloatVector) */ - public static double percentile( double percentile, io.deephaven.vector.FloatVector values ) {return Numeric.percentile( percentile, values );} + public static float percentile( double percentile, io.deephaven.vector.FloatVector values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,io.deephaven.vector.IntVector) */ - public static double percentile( double percentile, io.deephaven.vector.IntVector values ) {return Numeric.percentile( percentile, values );} + public static int percentile( double percentile, io.deephaven.vector.IntVector values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,io.deephaven.vector.LongVector) */ - public static double percentile( double percentile, io.deephaven.vector.LongVector values ) {return Numeric.percentile( percentile, values );} + public static long percentile( double percentile, io.deephaven.vector.LongVector values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#percentile(double,io.deephaven.vector.ShortVector) */ - public static double percentile( double percentile, io.deephaven.vector.ShortVector values ) {return Numeric.percentile( percentile, values );} + public static short percentile( double percentile, io.deephaven.vector.ShortVector values ) {return Numeric.percentile( percentile, values );} /** @see io.deephaven.function.Numeric#pow(byte,byte) */ public static double pow( byte a, byte b ) {return Numeric.pow( a, b );} @@ -3455,40 +3455,40 @@ public class GroovyStaticImports { public static double pow( short a, short b ) {return Numeric.pow( a, b );} /** @see io.deephaven.function.Numeric#product(byte[]) */ - public static byte product( byte... values ) {return Numeric.product( values );} + public static long product( byte... values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(double[]) */ public static double product( double... values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(float[]) */ - public static float product( float... values ) {return Numeric.product( values );} + public static double product( float... values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(int[]) */ - public static int product( int... values ) {return Numeric.product( values );} + public static long product( int... values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(long[]) */ public static long product( long... values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(short[]) */ - public static short product( short... values ) {return Numeric.product( values );} + public static long product( short... values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(io.deephaven.vector.ByteVector) */ - public static byte product( io.deephaven.vector.ByteVector values ) {return Numeric.product( values );} + public static long product( io.deephaven.vector.ByteVector values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(io.deephaven.vector.DoubleVector) */ public static double product( io.deephaven.vector.DoubleVector values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(io.deephaven.vector.FloatVector) */ - public static float product( io.deephaven.vector.FloatVector values ) {return Numeric.product( values );} + public static double product( io.deephaven.vector.FloatVector values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(io.deephaven.vector.IntVector) */ - public static int product( io.deephaven.vector.IntVector values ) {return Numeric.product( values );} + public static long product( io.deephaven.vector.IntVector values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(io.deephaven.vector.LongVector) */ public static long product( io.deephaven.vector.LongVector values ) {return Numeric.product( values );} /** @see io.deephaven.function.Numeric#product(io.deephaven.vector.ShortVector) */ - public static short product( io.deephaven.vector.ShortVector values ) {return Numeric.product( values );} + public static long product( io.deephaven.vector.ShortVector values ) {return Numeric.product( values );} /** @see io.deephaven.function.Random#random() */ public static double random( ) {return Random.random( );} @@ -4256,40 +4256,40 @@ public class GroovyStaticImports { public static double ste( io.deephaven.vector.ShortVector values ) {return Numeric.ste( values );} /** @see io.deephaven.function.Numeric#sum(byte[]) */ - public static byte sum( byte... values ) {return Numeric.sum( values );} + public static long sum( byte... values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(double[]) */ public static double sum( double... values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(float[]) */ - public static float sum( float... values ) {return Numeric.sum( values );} + public static double sum( float... values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(int[]) */ - public static int sum( int... values ) {return Numeric.sum( values );} + public static long sum( int... values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(long[]) */ public static long sum( long... values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(short[]) */ - public static short sum( short... values ) {return Numeric.sum( values );} + public static long sum( short... values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(io.deephaven.vector.ByteVector) */ - public static byte sum( io.deephaven.vector.ByteVector values ) {return Numeric.sum( values );} + public static long sum( io.deephaven.vector.ByteVector values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(io.deephaven.vector.DoubleVector) */ public static double sum( io.deephaven.vector.DoubleVector values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(io.deephaven.vector.FloatVector) */ - public static float sum( io.deephaven.vector.FloatVector values ) {return Numeric.sum( values );} + public static double sum( io.deephaven.vector.FloatVector values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(io.deephaven.vector.IntVector) */ - public static int sum( io.deephaven.vector.IntVector values ) {return Numeric.sum( values );} + public static long sum( io.deephaven.vector.IntVector values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(io.deephaven.vector.LongVector) */ public static long sum( io.deephaven.vector.LongVector values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#sum(io.deephaven.vector.ShortVector) */ - public static short sum( io.deephaven.vector.ShortVector values ) {return Numeric.sum( values );} + public static long sum( io.deephaven.vector.ShortVector values ) {return Numeric.sum( values );} /** @see io.deephaven.function.Numeric#tan(byte) */ public static double tan( byte value ) {return Numeric.tan( value );} @@ -5795,7 +5795,7 @@ public class GroovyStaticImports { public static double wste( io.deephaven.vector.ShortVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wste( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],byte[]) */ - public static double wsum( byte[] values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],double[]) */ public static double wsum( byte[] values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -5804,16 +5804,16 @@ public class GroovyStaticImports { public static double wsum( byte[] values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],int[]) */ - public static double wsum( byte[] values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],long[]) */ - public static double wsum( byte[] values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],short[]) */ - public static double wsum( byte[] values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],io.deephaven.vector.ByteVector) */ - public static double wsum( byte[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],io.deephaven.vector.DoubleVector) */ public static double wsum( byte[] values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -5822,13 +5822,13 @@ public class GroovyStaticImports { public static double wsum( byte[] values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],io.deephaven.vector.IntVector) */ - public static double wsum( byte[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],io.deephaven.vector.LongVector) */ - public static double wsum( byte[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(byte[],io.deephaven.vector.ShortVector) */ - public static double wsum( byte[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( byte[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(double[],byte[]) */ public static double wsum( double[] values, byte[] weights ) {return Numeric.wsum( values, weights );} @@ -5903,7 +5903,7 @@ public class GroovyStaticImports { public static double wsum( float[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],byte[]) */ - public static double wsum( int[] values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],double[]) */ public static double wsum( int[] values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -5912,16 +5912,16 @@ public class GroovyStaticImports { public static double wsum( int[] values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],int[]) */ - public static double wsum( int[] values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],long[]) */ - public static double wsum( int[] values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],short[]) */ - public static double wsum( int[] values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],io.deephaven.vector.ByteVector) */ - public static double wsum( int[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],io.deephaven.vector.DoubleVector) */ public static double wsum( int[] values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -5930,16 +5930,16 @@ public class GroovyStaticImports { public static double wsum( int[] values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],io.deephaven.vector.IntVector) */ - public static double wsum( int[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],io.deephaven.vector.LongVector) */ - public static double wsum( int[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(int[],io.deephaven.vector.ShortVector) */ - public static double wsum( int[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( int[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],byte[]) */ - public static double wsum( long[] values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],double[]) */ public static double wsum( long[] values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -5948,16 +5948,16 @@ public class GroovyStaticImports { public static double wsum( long[] values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],int[]) */ - public static double wsum( long[] values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],long[]) */ - public static double wsum( long[] values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],short[]) */ - public static double wsum( long[] values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],io.deephaven.vector.ByteVector) */ - public static double wsum( long[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],io.deephaven.vector.DoubleVector) */ public static double wsum( long[] values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -5966,16 +5966,16 @@ public class GroovyStaticImports { public static double wsum( long[] values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],io.deephaven.vector.IntVector) */ - public static double wsum( long[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],io.deephaven.vector.LongVector) */ - public static double wsum( long[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(long[],io.deephaven.vector.ShortVector) */ - public static double wsum( long[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( long[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],byte[]) */ - public static double wsum( short[] values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],double[]) */ public static double wsum( short[] values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -5984,16 +5984,16 @@ public class GroovyStaticImports { public static double wsum( short[] values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],int[]) */ - public static double wsum( short[] values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],long[]) */ - public static double wsum( short[] values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],short[]) */ - public static double wsum( short[] values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],io.deephaven.vector.ByteVector) */ - public static double wsum( short[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],io.deephaven.vector.DoubleVector) */ public static double wsum( short[] values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -6002,16 +6002,16 @@ public class GroovyStaticImports { public static double wsum( short[] values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],io.deephaven.vector.IntVector) */ - public static double wsum( short[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],io.deephaven.vector.LongVector) */ - public static double wsum( short[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(short[],io.deephaven.vector.ShortVector) */ - public static double wsum( short[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( short[] values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,byte[]) */ - public static double wsum( io.deephaven.vector.ByteVector values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,double[]) */ public static double wsum( io.deephaven.vector.ByteVector values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -6020,16 +6020,16 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.ByteVector values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,int[]) */ - public static double wsum( io.deephaven.vector.ByteVector values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,long[]) */ - public static double wsum( io.deephaven.vector.ByteVector values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,short[]) */ - public static double wsum( io.deephaven.vector.ByteVector values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,io.deephaven.vector.ByteVector) */ - public static double wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,io.deephaven.vector.DoubleVector) */ public static double wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -6038,13 +6038,13 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,io.deephaven.vector.IntVector) */ - public static double wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,io.deephaven.vector.LongVector) */ - public static double wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ByteVector,io.deephaven.vector.ShortVector) */ - public static double wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ByteVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.DoubleVector,byte[]) */ public static double wsum( io.deephaven.vector.DoubleVector values, byte[] weights ) {return Numeric.wsum( values, weights );} @@ -6119,7 +6119,7 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.FloatVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,byte[]) */ - public static double wsum( io.deephaven.vector.IntVector values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,double[]) */ public static double wsum( io.deephaven.vector.IntVector values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -6128,16 +6128,16 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.IntVector values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,int[]) */ - public static double wsum( io.deephaven.vector.IntVector values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,long[]) */ - public static double wsum( io.deephaven.vector.IntVector values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,short[]) */ - public static double wsum( io.deephaven.vector.IntVector values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,io.deephaven.vector.ByteVector) */ - public static double wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,io.deephaven.vector.DoubleVector) */ public static double wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -6146,16 +6146,16 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,io.deephaven.vector.IntVector) */ - public static double wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,io.deephaven.vector.LongVector) */ - public static double wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.IntVector,io.deephaven.vector.ShortVector) */ - public static double wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.IntVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,byte[]) */ - public static double wsum( io.deephaven.vector.LongVector values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,double[]) */ public static double wsum( io.deephaven.vector.LongVector values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -6164,16 +6164,16 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.LongVector values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,int[]) */ - public static double wsum( io.deephaven.vector.LongVector values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,long[]) */ - public static double wsum( io.deephaven.vector.LongVector values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,short[]) */ - public static double wsum( io.deephaven.vector.LongVector values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,io.deephaven.vector.ByteVector) */ - public static double wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,io.deephaven.vector.DoubleVector) */ public static double wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -6182,16 +6182,16 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,io.deephaven.vector.IntVector) */ - public static double wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,io.deephaven.vector.LongVector) */ - public static double wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.LongVector,io.deephaven.vector.ShortVector) */ - public static double wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.LongVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,byte[]) */ - public static double wsum( io.deephaven.vector.ShortVector values, byte[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, byte[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,double[]) */ public static double wsum( io.deephaven.vector.ShortVector values, double[] weights ) {return Numeric.wsum( values, weights );} @@ -6200,16 +6200,16 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.ShortVector values, float[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,int[]) */ - public static double wsum( io.deephaven.vector.ShortVector values, int[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, int[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,long[]) */ - public static double wsum( io.deephaven.vector.ShortVector values, long[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, long[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,short[]) */ - public static double wsum( io.deephaven.vector.ShortVector values, short[] weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, short[] weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,io.deephaven.vector.ByteVector) */ - public static double wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.ByteVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,io.deephaven.vector.DoubleVector) */ public static double wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.DoubleVector weights ) {return Numeric.wsum( values, weights );} @@ -6218,13 +6218,13 @@ public class GroovyStaticImports { public static double wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.FloatVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,io.deephaven.vector.IntVector) */ - public static double wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.IntVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,io.deephaven.vector.LongVector) */ - public static double wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.LongVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wsum(io.deephaven.vector.ShortVector,io.deephaven.vector.ShortVector) */ - public static double wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} + public static long wsum( io.deephaven.vector.ShortVector values, io.deephaven.vector.ShortVector weights ) {return Numeric.wsum( values, weights );} /** @see io.deephaven.function.Numeric#wtstat(byte[],byte[]) */ public static double wtstat( byte[] values, byte[] weights ) {return Numeric.wtstat( values, weights );} diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java index dacb162696b..e43c2997cf0 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java @@ -321,7 +321,7 @@ final void assertWithCumProd(@NotNull final Object expected, @NotNull final Obje } else if (expected instanceof long[]) { assertArrayEquals(Numeric.cumprod((long[]) expected), (long[]) actual); } else if (expected instanceof float[]) { - assertArrayEquals(Numeric.cumprod((float[]) expected), (float[]) actual, .001f); + assertArrayEquals(Numeric.cumprod((float[]) expected), (double[]) actual, .001f); } else if (expected instanceof double[]) { assertArrayEquals(Numeric.cumprod((double[]) expected), (double[]) actual, .001d); } else { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java index 0722ef97603..488ff43ec68 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java @@ -361,7 +361,7 @@ final void assertWithCumSum(@NotNull final Object expected, @NotNull final Objec } else if (expected instanceof long[]) { assertArrayEquals(Numeric.cumsum((long[]) expected), (long[]) actual); } else if (expected instanceof float[]) { - assertArrayEquals(Numeric.cumsum((float[]) expected), (float[]) actual, .001f); + assertArrayEquals(Numeric.cumsum((float[]) expected), (double[]) actual, .001f); } else if (expected instanceof double[]) { assertArrayEquals(Numeric.cumsum((double[]) expected), (double[]) actual, .001d); } else if (expected instanceof Boolean[]) { From 2dbb2f996ff55278b4dde33043462db7b5724ad0 Mon Sep 17 00:00:00 2001 From: Ryan Caudy Date: Mon, 6 May 2024 17:06:58 -0400 Subject: [PATCH 2/7] Make DataIndex accumulation and transformation parallelizable (#5457) * Create a tool for marking SelectColumns that are known to be stateless as such * Mark FunctionalColumns used in MergedDataIndex and TransformedDataIndex as stateless in order to enable parallelism --- .../impl/dataindex/TransformedDataIndex.java | 5 +- .../table/impl/select/SelectColumn.java | 10 ++ .../impl/select/StatelessSelectColumn.java | 132 ++++++++++++++++++ .../sources/regioned/MergedDataIndex.java | 13 +- 4 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/select/StatelessSelectColumn.java diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/dataindex/TransformedDataIndex.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/dataindex/TransformedDataIndex.java index 38e01019c8b..a48ab67b977 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/dataindex/TransformedDataIndex.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/dataindex/TransformedDataIndex.java @@ -17,6 +17,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.select.FunctionalColumn; import io.deephaven.engine.table.impl.select.FunctionalColumnLong; +import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.util.SafeCloseable; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -153,10 +154,10 @@ private Table maybeIntersectAndInvert(@NotNull final Table indexTable) { final Function mutator = getMutator(transformer.intersectRowSet().orElse(null), transformer.invertRowSet().orElse(null)); final Table mutated = indexTable - .update(List.of(new FunctionalColumn<>( + .update(List.of(SelectColumn.ofStateless(new FunctionalColumn<>( parentIndex.rowSetColumnName(), RowSet.class, parentIndex.rowSetColumnName(), RowSet.class, - mutator))); + mutator)))); if (transformer.intersectRowSet().isPresent()) { return mutated.where(Filter.isNotNull(ColumnName.of(parentIndex.rowSetColumnName()))); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index fe05a0675ca..9cb52213e3b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -57,6 +57,16 @@ static Collection copyFrom(Collection selectColumns) return selectColumns.stream().map(SelectColumn::copy).collect(Collectors.toList()); } + /** + * Produce a {@link #isStateless() stateless} SelectColumn from {@code selectable}. + * + * @param selectable The {@link Selectable} to adapt and mark as stateless + * @return The resulting SelectColumn + */ + static SelectColumn ofStateless(@NotNull final Selectable selectable) { + return new StatelessSelectColumn(of(selectable)); + } + /** * Convenient static final instance of a zero length Array of SelectColumns for use in toArray calls. */ diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/StatelessSelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/StatelessSelectColumn.java new file mode 100644 index 00000000000..97ca5052ce4 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/StatelessSelectColumn.java @@ -0,0 +1,132 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.select; + +import io.deephaven.api.ColumnName; +import io.deephaven.api.expression.Expression; +import io.deephaven.engine.rowset.TrackingRowSet; +import io.deephaven.engine.table.ColumnDefinition; +import io.deephaven.engine.table.ColumnSource; +import io.deephaven.engine.table.WritableColumnSource; +import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import org.jetbrains.annotations.NotNull; + +import java.util.List; +import java.util.Map; + +/** + * {@link SelectColumn} implementation that wraps another {@link SelectColumn} and makes it report to be + * {@link #isStateless() stateless}. + */ +class StatelessSelectColumn implements SelectColumn { + + private final SelectColumn inner; + + StatelessSelectColumn(@NotNull final SelectColumn inner) { + this.inner = inner; + } + + @Override + public List initInputs( + @NotNull final TrackingRowSet rowSet, + @NotNull final Map> columnsOfInterest) { + return inner.initInputs(rowSet, columnsOfInterest); + } + + @Override + public List initDef(@NotNull final Map> columnDefinitionMap) { + return inner.initDef(columnDefinitionMap); + } + + @Override + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + return inner.initDef(columnDefinitionMap, compilationRequestProcessor); + } + + @Override + public Class getReturnedType() { + return inner.getReturnedType(); + } + + @Override + public Class getReturnedComponentType() { + return inner.getReturnedComponentType(); + } + + @Override + public List getColumns() { + return inner.getColumns(); + } + + @Override + public List getColumnArrays() { + return inner.getColumnArrays(); + } + + @Override + @NotNull + public ColumnSource getDataView() { + return inner.getDataView(); + } + + @Override + @NotNull + public ColumnSource getLazyView() { + return inner.getLazyView(); + } + + @Override + public String getName() { + return inner.getName(); + } + + @Override + public MatchPair getMatchPair() { + return inner.getMatchPair(); + } + + @Override + public WritableColumnSource newDestInstance(final long size) { + return inner.newDestInstance(size); + } + + @Override + public WritableColumnSource newFlatDestInstance(final long size) { + return inner.newFlatDestInstance(size); + } + + @Override + public boolean isRetain() { + return inner.isRetain(); + } + + @Override + public void validateSafeForRefresh(@NotNull final BaseTable sourceTable) { + inner.validateSafeForRefresh(sourceTable); + } + + @Override + public boolean isStateless() { + return true; + } + + @Override + public SelectColumn copy() { + return new StatelessSelectColumn(inner.copy()); + } + + @Override + public ColumnName newColumn() { + return inner.newColumn(); + } + + @Override + public Expression expression() { + return inner.expression(); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MergedDataIndex.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MergedDataIndex.java index d37ca9d5d49..597daae5f65 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MergedDataIndex.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/sources/regioned/MergedDataIndex.java @@ -21,6 +21,7 @@ import io.deephaven.engine.table.impl.locations.TableLocation; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.FunctionalColumn; +import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.InternalUseOnly; import io.deephaven.vector.ObjectVector; @@ -138,11 +139,11 @@ private Table buildTable() { // pages during the accumulation phase. final String[] keyColumnNamesArray = keyColumnNames.toArray(String[]::new); final Table locationDataIndexes = locationTable - .update(List.of(new FunctionalColumn<>( + .update(List.of(SelectColumn.ofStateless(new FunctionalColumn<>( columnSourceManager.locationColumnName(), TableLocation.class, LOCATION_DATA_INDEX_TABLE_COLUMN_NAME, Table.class, (final long locationRowKey, final TableLocation location) -> loadIndexTableAndShiftRowSets( - locationRowKey, location, keyColumnNamesArray)))) + locationRowKey, location, keyColumnNamesArray))))) .dropColumns(columnSourceManager.locationColumnName()); // Merge all the location index tables into a single table @@ -153,10 +154,10 @@ private Table buildTable() { // Combine the row sets from each group into a single row set final Table combined = groupedByKeyColumns - .update(List.of(new FunctionalColumn<>( + .update(List.of(SelectColumn.ofStateless(new FunctionalColumn<>( ROW_SET_COLUMN_NAME, ObjectVector.class, ROW_SET_COLUMN_NAME, RowSet.class, - this::mergeRowSets))); + this::mergeRowSets)))); Assert.assertion(combined.isFlat(), "combined.isFlat()"); Assert.eq(groupedByKeyColumns.size(), "groupedByKeyColumns.size()", combined.size(), "combined.size()"); @@ -180,11 +181,11 @@ private static Table loadIndexTableAndShiftRowSets( String.join(", ", keyColumnNames), location)); } final Table indexTable = dataIndex.table(); - return indexTable.coalesce().update(List.of(new FunctionalColumn<>( + return indexTable.coalesce().update(List.of(SelectColumn.ofStateless(new FunctionalColumn<>( dataIndex.rowSetColumnName(), RowSet.class, ROW_SET_COLUMN_NAME, RowSet.class, (final RowSet rowSet) -> rowSet - .shift(RegionedColumnSource.getFirstRowKey(Math.toIntExact(locationRowKey)))))); + .shift(RegionedColumnSource.getFirstRowKey(Math.toIntExact(locationRowKey))))))); } private RowSet mergeRowSets( From cd444b63b68b31aa0bd32e0bfda48e4645561b4f Mon Sep 17 00:00:00 2001 From: Chip Kent <5250374+chipkent@users.noreply.github.com> Date: Tue, 7 May 2024 15:22:11 -0600 Subject: [PATCH 3/7] Make Numeric.sum return null when there are no inputs or all null inputs (#5461) * Make sum of nulls or no values return null * Another unit test. --- engine/function/src/templates/Numeric.ftl | 14 ++++++++++++++ engine/function/src/templates/TestNumeric.ftl | 15 ++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/engine/function/src/templates/Numeric.ftl b/engine/function/src/templates/Numeric.ftl index 9853bd4ebb8..bb7c8ca0346 100644 --- a/engine/function/src/templates/Numeric.ftl +++ b/engine/function/src/templates/Numeric.ftl @@ -1488,6 +1488,7 @@ public class Numeric { } double sum = 0; + long nullCount = 0; try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { @@ -1499,10 +1500,16 @@ public class Numeric { if (!isNull(c)) { sum += c; + } else { + nullCount++; } } } + if (nullCount == values.size()) { + return NULL_DOUBLE; + } + return sum; } <#else> @@ -1512,6 +1519,7 @@ public class Numeric { } long sum = 0; + long nullCount = 0; try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { @@ -1519,10 +1527,16 @@ public class Numeric { if (!isNull(c)) { sum += c; + } else { + nullCount++; } } } + if (nullCount == values.size()) { + return NULL_LONG; + } + return sum; } diff --git a/engine/function/src/templates/TestNumeric.ftl b/engine/function/src/templates/TestNumeric.ftl index 41855aa6710..56fcb65d2d4 100644 --- a/engine/function/src/templates/TestNumeric.ftl +++ b/engine/function/src/templates/TestNumeric.ftl @@ -567,36 +567,41 @@ public class TestNumeric extends BaseArrayTestCase { public void test${pt.boxed}Sum1() { assertTrue(Math.abs(15 - sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, 5, 6}))) == 0.0); - assertTrue(Math.abs(0 - sum(new ${pt.vectorDirect}())) == 0.0); - assertTrue(Math.abs(0 - sum(new ${pt.vectorDirect}(${pt.null}))) == 0.0); assertTrue(Math.abs(20 - sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{5, ${pt.null}, 15}))) == 0.0); <#if pt.valueType.isFloat > assertEquals(NULL_DOUBLE, sum((${pt.vector}) null)); + assertEquals(NULL_DOUBLE, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{}))); + assertEquals(NULL_DOUBLE, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, ${pt.null}}))); assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, 6}))); assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY}))); assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, 6}))); assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY}))); assertEquals(Double.NaN, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY}))); + assertEquals(Double.NaN, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{4, Float.NaN, 6}))); <#else> assertEquals(NULL_LONG, sum((${pt.vector}) null)); + assertEquals(NULL_LONG, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{}))); + assertEquals(NULL_LONG, sum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, ${pt.null}}))); - } public void test${pt.boxed}Sum2() { assertTrue(Math.abs(15 - sum(new ${pt.primitive}[]{4, 5, 6})) == 0.0); - assertTrue(Math.abs(0 - sum(new ${pt.primitive}[]{})) == 0.0); - assertTrue(Math.abs(0 - sum(new ${pt.primitive}[]{${pt.null}})) == 0.0); assertTrue(Math.abs(20 - sum(new ${pt.primitive}[]{5, ${pt.null}, 15})) == 0.0); <#if pt.valueType.isFloat > assertEquals(NULL_DOUBLE, sum((${pt.primitive}[]) null)); + assertEquals(NULL_DOUBLE, sum(new ${pt.primitive}[]{})); + assertEquals(NULL_DOUBLE, sum(new ${pt.primitive}[]{${pt.null}, ${pt.null}})); assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, 6})); assertEquals(Double.POSITIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY})); assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, 6})); assertEquals(Double.NEGATIVE_INFINITY, sum(new ${pt.primitive}[]{4, Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY})); assertEquals(Double.NaN, sum(new ${pt.primitive}[]{4, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY})); + assertEquals(Double.NaN, sum(new ${pt.primitive}[]{4, Float.NaN, 6})); <#else> assertEquals(NULL_LONG, sum((${pt.primitive}[]) null)); + assertEquals(NULL_LONG, sum(new ${pt.primitive}[]{})); + assertEquals(NULL_LONG, sum(new ${pt.primitive}[]{${pt.null}, ${pt.null}})); } From dcf77c1554dd8395ddabe9ef4821ca0db71ae24e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 23:33:29 -0500 Subject: [PATCH 4/7] [create-pull-request] automated change (#5463) Co-authored-by: deephaven-internal <66694643+deephaven-internal@users.noreply.github.com> --- web/client-ui/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/client-ui/Dockerfile b/web/client-ui/Dockerfile index 1784990dfa9..b2e355a874f 100644 --- a/web/client-ui/Dockerfile +++ b/web/client-ui/Dockerfile @@ -2,10 +2,10 @@ FROM deephaven/node:local-build WORKDIR /usr/src/app # Most of the time, these versions are the same, except in cases where a patch only affects one of the packages -ARG WEB_VERSION=0.76.0 -ARG GRID_VERSION=0.76.0 -ARG CHART_VERSION=0.76.0 -ARG WIDGET_VERSION=0.76.0 +ARG WEB_VERSION=0.77.0 +ARG GRID_VERSION=0.77.0 +ARG CHART_VERSION=0.77.0 +ARG WIDGET_VERSION=0.77.0 # Pull in the published code-studio package from npmjs and extract is RUN set -eux; \ From f2dc4fe15e2b3b35e8481034c8fae7a1e72f79e9 Mon Sep 17 00:00:00 2001 From: Chip Kent <5250374+chipkent@users.noreply.github.com> Date: Wed, 8 May 2024 10:17:51 -0600 Subject: [PATCH 5/7] Fixed problems in cumsum and cumprod where multiple leading null values give incorrect answers. (#5464) Fixed bug in cumsum and cumprod when there are multiple initial null values. Resolves #5462 --- engine/function/src/templates/Numeric.ftl | 20 +++++++++---------- engine/function/src/templates/TestNumeric.ftl | 6 ++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/engine/function/src/templates/Numeric.ftl b/engine/function/src/templates/Numeric.ftl index bb7c8ca0346..30cfc23084e 100644 --- a/engine/function/src/templates/Numeric.ftl +++ b/engine/function/src/templates/Numeric.ftl @@ -1934,10 +1934,10 @@ public class Numeric { if (isNaN(v) || isNaN(result[i - 1])) { Arrays.fill(result, i, n, Double.NaN); return result; - } else if (isNull(result[i - 1])) { - result[i] = v; } else if (isNull(v)) { result[i] = result[i - 1]; + } else if (isNull(result[i - 1])) { + result[i] = v; } else { result[i] = result[i - 1] + v; } @@ -1969,10 +1969,10 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} v = vi.${pt.iteratorNext}(); - if (isNull(result[i - 1])) { - result[i] = v; - } else if (isNull(v)) { + if (isNull(v)) { result[i] = result[i - 1]; + } else if (isNull(result[i - 1])) { + result[i] = v; } else { result[i] = result[i - 1] + v; } @@ -2055,10 +2055,10 @@ public class Numeric { if (isNaN(v) || isNaN(result[i - 1])) { Arrays.fill(result, i, n, Double.NaN); return result; - } else if (isNull(result[i - 1])) { - result[i] = v; } else if (isNull(v)) { result[i] = result[i - 1]; + } else if (isNull(result[i - 1])) { + result[i] = v; } else { result[i] = result[i - 1] * v; } @@ -2090,10 +2090,10 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} v = vi.${pt.iteratorNext}(); - if (isNull(result[i - 1])) { - result[i] = v; - } else if (isNull(v)) { + if (isNull(v)) { result[i] = result[i - 1]; + } else if (isNull(result[i - 1])) { + result[i] = v; } else { result[i] = result[i - 1] * v; } diff --git a/engine/function/src/templates/TestNumeric.ftl b/engine/function/src/templates/TestNumeric.ftl index 56fcb65d2d4..9bf99f52c0b 100644 --- a/engine/function/src/templates/TestNumeric.ftl +++ b/engine/function/src/templates/TestNumeric.ftl @@ -805,6 +805,7 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(new double[]{1, 3, 6, 10, 15}, cumsum(new ${pt.primitive}[]{1, 2, 3, 4, 5})); assertEquals(new double[]{1, 3, 6, 6, 11}, cumsum(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); assertEquals(new double[]{NULL_DOUBLE, 2, 5, 9, 14}, cumsum(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); + assertEquals(new double[]{NULL_DOUBLE, NULL_DOUBLE, 2, 5, 9, 14}, cumsum(new ${pt.primitive}[]{${pt.null}, ${pt.null}, 2, 3, 4, 5})); assertEquals(new double[0], cumsum(new ${pt.primitive}[0])); assertEquals(new double[0], cumsum(new ${pt.boxed}[0])); assertEquals(null, cumsum((${pt.primitive}[]) null)); @@ -812,6 +813,7 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(new double[]{1, 3, 6, 10, 15}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); assertEquals(new double[]{1, 3, 6, 6, 11}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); assertEquals(new double[]{NULL_DOUBLE, 2, 5, 9, 14}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); + assertEquals(new double[]{NULL_DOUBLE, NULL_DOUBLE, 2, 5, 9, 14}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, ${pt.null}, 2, 3, 4, 5}))); assertEquals(new double[0], cumsum(new ${pt.vectorDirect}())); assertEquals(null, cumsum((${pt.vector}) null)); @@ -832,6 +834,7 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(new long[]{1, 3, 6, 10, 15}, cumsum(new ${pt.primitive}[]{1, 2, 3, 4, 5})); assertEquals(new long[]{1, 3, 6, 6, 11}, cumsum(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); assertEquals(new long[]{NULL_LONG, 2, 5, 9, 14}, cumsum(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); + assertEquals(new long[]{NULL_LONG, NULL_LONG, 2, 5, 9, 14}, cumsum(new ${pt.primitive}[]{${pt.null}, ${pt.null}, 2, 3, 4, 5})); assertEquals(new long[0], cumsum(new ${pt.primitive}[0])); assertEquals(new long[0], cumsum(new ${pt.boxed}[0])); assertEquals(null, cumsum((${pt.primitive}[]) null)); @@ -839,6 +842,7 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(new long[]{1, 3, 6, 10, 15}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); assertEquals(new long[]{1, 3, 6, 6, 11}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); assertEquals(new long[]{NULL_LONG, 2, 5, 9, 14}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); + assertEquals(new long[]{NULL_LONG, NULL_LONG, 2, 5, 9, 14}, cumsum(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, ${pt.null}, 2, 3, 4, 5}))); assertEquals(new long[0], cumsum(new ${pt.vectorDirect}())); assertEquals(null, cumsum((${pt.vector}) null)); @@ -852,6 +856,7 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(new double[]{1, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{1, 2, 3, 4, 5})); assertEquals(new double[]{1, 2, 6, 6, 30}, cumprod(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5})); assertEquals(new double[]{NULL_DOUBLE, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5})); + assertEquals(new double[]{NULL_DOUBLE, NULL_DOUBLE, 2, 6, 24, 120}, cumprod(new ${pt.primitive}[]{${pt.null}, ${pt.null}, 2, 3, 4, 5})); assertEquals(new double[0], cumprod(new ${pt.primitive}[0])); assertEquals(new double[0], cumprod(new ${pt.boxed}[0])); assertEquals(null, cumprod((${pt.primitive}[]) null)); @@ -860,6 +865,7 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(new double[]{1, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, 4, 5}))); assertEquals(new double[]{1, 2, 6, 6, 30}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, 2, 3, ${pt.null}, 5}))); assertEquals(new double[]{NULL_DOUBLE, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, 2, 3, 4, 5}))); + assertEquals(new double[]{NULL_DOUBLE, NULL_DOUBLE, 2, 6, 24, 120}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{${pt.null}, ${pt.null}, 2, 3, 4, 5}))); assertEquals(new double[0], cumprod(new ${pt.vectorDirect}())); assertEquals(null, cumprod((${pt.vector}) null)); assertEquals(new double[]{1, Double.NaN, Double.NaN, Double.NaN, Double.NaN}, cumprod(new ${pt.vectorDirect}(new ${pt.primitive}[]{1, Float.NaN, 3, 4, 5}))); From 25e0cb1ff86be7d7814085f8e096e9db4661f4bd Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 8 May 2024 09:54:17 -0700 Subject: [PATCH 6/7] Widen returned types for `UpdateBy` floating point operations. (#5371) * Initial commit of changes for widening update_by returned types (Float -> Double) * Fixed NULL_FLOAT related bugs, adjusted tests to use correct datatypes. * Added all NULL tests to UpdateBy cumulative operations. --- .../updateby/prod/DoubleCumProdOperator.java | 1 + .../updateby/prod/FloatCumProdOperator.java | 9 +- .../rollingsum/DoubleRollingSumOperator.java | 7 +- .../rollingsum/FloatRollingSumOperator.java | 33 ++--- .../updateby/sum/DoubleCumSumOperator.java | 9 +- .../updateby/sum/FloatCumSumOperator.java | 15 ++- .../table/impl/updateby/BaseUpdateByTest.java | 48 ++++++-- .../table/impl/updateby/TestCumMinMax.java | 25 ++++ .../table/impl/updateby/TestCumProd.java | 17 +++ .../table/impl/updateby/TestCumSum.java | 114 ++++-------------- .../table/impl/updateby/TestRollingSum.java | 66 ++++++++-- 11 files changed, 197 insertions(+), 147 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/DoubleCumProdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/DoubleCumProdOperator.java index 56cd5ba585b..32b76c10ba9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/DoubleCumProdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/DoubleCumProdOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.impl.updateby.internal.BaseDoubleUpdateByOperator; import org.jetbrains.annotations.NotNull; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_DOUBLE; public class DoubleCumProdOperator extends BaseDoubleUpdateByOperator { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/FloatCumProdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/FloatCumProdOperator.java index af36f5c9539..5072c54f293 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/FloatCumProdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/prod/FloatCumProdOperator.java @@ -9,16 +9,17 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; -import io.deephaven.engine.table.impl.updateby.internal.BaseFloatUpdateByOperator; +import io.deephaven.engine.table.impl.updateby.internal.BaseDoubleUpdateByOperator; import org.jetbrains.annotations.NotNull; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_FLOAT; -public class FloatCumProdOperator extends BaseFloatUpdateByOperator { +public class FloatCumProdOperator extends BaseDoubleUpdateByOperator { // region extra-fields // endregion extra-fields - protected class Context extends BaseFloatUpdateByOperator.Context { + protected class Context extends BaseDoubleUpdateByOperator.Context { public FloatChunk floatValueChunk; protected Context(final int chunkSize) { @@ -37,7 +38,7 @@ public void push(int pos, int count) { final float val = floatValueChunk.get(pos); if (val != NULL_FLOAT) { - curVal = curVal == NULL_FLOAT ? val : curVal * val; + curVal = curVal == NULL_DOUBLE ? val : curVal * val; } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/DoubleRollingSumOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/DoubleRollingSumOperator.java index 5959e8e3fe4..36f2c5beb7a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/DoubleRollingSumOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/DoubleRollingSumOperator.java @@ -18,6 +18,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_DOUBLE; public class DoubleRollingSumOperator extends BaseDoubleUpdateByOperator { @@ -60,11 +61,13 @@ public void push(int pos, int count) { aggSum.ensureRemaining(count); for (int ii = 0; ii < count; ii++) { - double val = doubleInfluencerValuesChunk.get(pos + ii); - aggSum.addUnsafe(val); + final double val = doubleInfluencerValuesChunk.get(pos + ii); if (val == NULL_DOUBLE) { nullCount++; + aggSum.addUnsafe(NULL_DOUBLE); + } else { + aggSum.addUnsafe(val); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/FloatRollingSumOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/FloatRollingSumOperator.java index 56bbad7f7dc..a5c26d7a4d5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/FloatRollingSumOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingsum/FloatRollingSumOperator.java @@ -3,37 +3,38 @@ // package io.deephaven.engine.table.impl.updateby.rollingsum; -import io.deephaven.base.ringbuffer.AggregatingFloatRingBuffer; +import io.deephaven.base.ringbuffer.AggregatingDoubleRingBuffer; import io.deephaven.base.verify.Assert; import io.deephaven.chunk.Chunk; import io.deephaven.chunk.FloatChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; -import io.deephaven.engine.table.impl.updateby.internal.BaseFloatUpdateByOperator; +import io.deephaven.engine.table.impl.updateby.internal.BaseDoubleUpdateByOperator; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_FLOAT; -public class FloatRollingSumOperator extends BaseFloatUpdateByOperator { +public class FloatRollingSumOperator extends BaseDoubleUpdateByOperator { private static final int BUFFER_INITIAL_SIZE = 64; - protected class Context extends BaseFloatUpdateByOperator.Context { + protected class Context extends BaseDoubleUpdateByOperator.Context { protected FloatChunk floatInfluencerValuesChunk; - protected AggregatingFloatRingBuffer aggSum; + protected AggregatingDoubleRingBuffer aggSum; protected Context(final int chunkSize) { super(chunkSize); - aggSum = new AggregatingFloatRingBuffer(BUFFER_INITIAL_SIZE, + aggSum = new AggregatingDoubleRingBuffer(BUFFER_INITIAL_SIZE, 0, - Float::sum, // tree function + Double::sum, // tree function (a, b) -> { // value function - if (a == NULL_FLOAT && b == NULL_FLOAT) { + if (a == NULL_DOUBLE && b == NULL_DOUBLE) { return 0; // identity val - } else if (a == NULL_FLOAT) { + } else if (a == NULL_DOUBLE) { return b; - } else if (b == NULL_FLOAT) { + } else if (b == NULL_DOUBLE) { return a; } return a + b; @@ -56,11 +57,13 @@ public void push(int pos, int count) { aggSum.ensureRemaining(count); for (int ii = 0; ii < count; ii++) { - float val = floatInfluencerValuesChunk.get(pos + ii); - aggSum.addUnsafe(val); + final float val = floatInfluencerValuesChunk.get(pos + ii); if (val == NULL_FLOAT) { nullCount++; + aggSum.addUnsafe(NULL_DOUBLE); + } else { + aggSum.addUnsafe(val); } } } @@ -70,9 +73,9 @@ public void pop(int count) { Assert.geq(aggSum.size(), "aggSum.size()", count); for (int ii = 0; ii < count; ii++) { - float val = aggSum.removeUnsafe(); + double val = aggSum.removeUnsafe(); - if (val == NULL_FLOAT) { + if (val == NULL_DOUBLE) { nullCount--; } } @@ -81,7 +84,7 @@ public void pop(int count) { @Override public void writeToOutputChunk(int outIdx) { if (aggSum.size() == nullCount) { - outputValues.set(outIdx, NULL_FLOAT); + outputValues.set(outIdx, NULL_DOUBLE); } else { outputValues.set(outIdx, aggSum.evaluate()); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/DoubleCumSumOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/DoubleCumSumOperator.java index 9c06cc12136..dbd5aa1fa62 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/DoubleCumSumOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/DoubleCumSumOperator.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.impl.updateby.internal.BaseDoubleUpdateByOperator; import org.jetbrains.annotations.NotNull; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_DOUBLE; public class DoubleCumSumOperator extends BaseDoubleUpdateByOperator { @@ -37,12 +38,10 @@ public void push(int pos, int count) { Assert.eq(count, "push count", 1); // read the value from the values chunk - final double currentVal = doubleValueChunk.get(pos); + final double val = doubleValueChunk.get(pos); - if (curVal == NULL_DOUBLE) { - curVal = currentVal; - } else if (currentVal != NULL_DOUBLE) { - curVal += currentVal; + if (val != NULL_DOUBLE) { + curVal = curVal == NULL_DOUBLE ? val : curVal + val; } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/FloatCumSumOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/FloatCumSumOperator.java index 08f7ffce1f3..883306a6502 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/FloatCumSumOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/sum/FloatCumSumOperator.java @@ -9,14 +9,15 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; -import io.deephaven.engine.table.impl.updateby.internal.BaseFloatUpdateByOperator; +import io.deephaven.engine.table.impl.updateby.internal.BaseDoubleUpdateByOperator; import org.jetbrains.annotations.NotNull; +import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_FLOAT; -public class FloatCumSumOperator extends BaseFloatUpdateByOperator { +public class FloatCumSumOperator extends BaseDoubleUpdateByOperator { - protected class Context extends BaseFloatUpdateByOperator.Context { + protected class Context extends BaseDoubleUpdateByOperator.Context { public FloatChunk floatValueChunk; protected Context(final int chunkSize) { @@ -33,12 +34,10 @@ public void push(int pos, int count) { Assert.eq(count, "push count", 1); // read the value from the values chunk - final float currentVal = floatValueChunk.get(pos); + final float val = floatValueChunk.get(pos); - if (curVal == NULL_FLOAT) { - curVal = currentVal; - } else if (currentVal != NULL_FLOAT) { - curVal += currentVal; + if (val != NULL_FLOAT) { + curVal = curVal == NULL_DOUBLE ? val : curVal + val; } } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java index 8f8054f8925..dc34435515a 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java @@ -42,14 +42,42 @@ static CreateResult createTestTable(int tableSize, boolean includeSym, boolean i CollectionUtil.ZERO_LENGTH_STRING_ARRAY, new TestDataGenerator[0]); } + static CreateResult createTestTable( + int tableSize, + boolean includeSym, + boolean includeGroups, + boolean isRefreshing, + int seed, + String[] extraNames, + TestDataGenerator[] extraGenerators) { + return createTestTable(tableSize, includeSym, includeGroups, isRefreshing, seed, extraNames, extraGenerators, + 0.1); + } + @SuppressWarnings({"rawtypes"}) - static CreateResult createTestTable(int tableSize, + static CreateResult createTestTableAllNull( + int tableSize, boolean includeSym, boolean includeGroups, boolean isRefreshing, int seed, String[] extraNames, TestDataGenerator[] extraGenerators) { + + return createTestTable(tableSize, includeSym, includeGroups, isRefreshing, seed, extraNames, extraGenerators, + 1.0); + } + + @SuppressWarnings({"rawtypes"}) + static CreateResult createTestTable( + int tableSize, + boolean includeSym, + boolean includeGroups, + boolean isRefreshing, + int seed, + String[] extraNames, + TestDataGenerator[] extraGenerators, + double nullFraction) { if (includeGroups && !includeSym) { throw new IllegalArgumentException(); } @@ -68,15 +96,15 @@ static CreateResult createTestTable(int tableSize, colsList.addAll(Arrays.asList("byteCol", "shortCol", "intCol", "longCol", "floatCol", "doubleCol", "boolCol", "bigIntCol", "bigDecimalCol")); - generators.addAll(Arrays.asList(new ByteGenerator((byte) -127, (byte) 127, .1), - new ShortGenerator((short) -6000, (short) 65535, .1), - new IntGenerator(10, 100, .1), - new LongGenerator(10, 100, .1), - new FloatGenerator(10.1F, 20.1F, .1), - new DoubleGenerator(10.1, 20.1, .1), - new BooleanGenerator(.5, .1), - new BigIntegerGenerator(new BigInteger("-10"), new BigInteger("10"), .1), - new BigDecimalGenerator(new BigInteger("1"), new BigInteger("2"), 5, .1))); + generators.addAll(Arrays.asList(new ByteGenerator((byte) -127, (byte) 127, nullFraction), + new ShortGenerator((short) -6000, (short) 65535, nullFraction), + new IntGenerator(10, 100, nullFraction), + new LongGenerator(10, 100, nullFraction), + new FloatGenerator(10.1F, 20.1F, nullFraction), + new DoubleGenerator(10.1, 20.1, nullFraction), + new BooleanGenerator(.5, nullFraction), + new BigIntegerGenerator(new BigInteger("-10"), new BigInteger("10"), nullFraction), + new BigDecimalGenerator(new BigInteger("1"), new BigInteger("2"), 5, nullFraction))); final Random random = new Random(seed); final ColumnInfo[] columnInfos = initColumnInfos(colsList.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY), diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumMinMax.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumMinMax.java index f6a59ca2167..684d08f6ead 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumMinMax.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumMinMax.java @@ -4,6 +4,7 @@ package io.deephaven.engine.table.impl.updateby; import io.deephaven.api.updateby.UpdateByOperation; +import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.PartitionedTable; import io.deephaven.engine.table.Table; @@ -12,6 +13,7 @@ import io.deephaven.engine.testutil.EvalNugget; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.testutil.TstUtils; +import io.deephaven.engine.testutil.generator.TestDataGenerator; import io.deephaven.function.Numeric; import io.deephaven.test.types.OutOfBandTest; import org.jetbrains.annotations.NotNull; @@ -52,6 +54,29 @@ public void testStaticZeroKey() { } } + @Test + public void testStaticZeroKeyAllNulls() { + final QueryTable t = createTestTableAllNull(100000, false, false, false, 0x31313131, + CollectionUtil.ZERO_LENGTH_STRING_ARRAY, new TestDataGenerator[0]).t; + + final Table result = t.updateBy(List.of( + UpdateByOperation.CumMin("byteColMin=byteCol", "shortColMin=shortCol", "intColMin=intCol", + "longColMin=longCol", "floatColMin=floatCol", "doubleColMin=doubleCol", + "bigIntColMin=bigIntCol", "bigDecimalColMin=bigDecimalCol"), + UpdateByOperation.CumMax("byteColMax=byteCol", "shortColMax=shortCol", "intColMax=intCol", + "longColMax=longCol", "floatColMax=floatCol", "doubleColMax=doubleCol", + "bigIntColMax=bigIntCol", "bigDecimalColMax=bigDecimalCol"))); + for (String col : t.getDefinition().getColumnNamesArray()) { + if ("boolCol".equals(col)) { + continue; + } + assertWithCumMin(DataAccessHelpers.getColumn(t, col).getDirect(), + DataAccessHelpers.getColumn(result, col + "Min").getDirect()); + assertWithCumMax(DataAccessHelpers.getColumn(t, col).getDirect(), + DataAccessHelpers.getColumn(result, col + "Max").getDirect()); + } + } + // endregion // region Bucketed Tests diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java index e43c2997cf0..0f75fe5b85e 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumProd.java @@ -4,6 +4,7 @@ package io.deephaven.engine.table.impl.updateby; import io.deephaven.api.updateby.UpdateByControl; +import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.PartitionedTable; import io.deephaven.engine.table.Table; @@ -14,6 +15,7 @@ import io.deephaven.engine.testutil.GenerateTableUpdates; import io.deephaven.engine.testutil.EvalNugget; import io.deephaven.engine.testutil.TstUtils; +import io.deephaven.engine.testutil.generator.TestDataGenerator; import io.deephaven.function.Numeric; import io.deephaven.test.types.OutOfBandTest; import org.jetbrains.annotations.NotNull; @@ -53,6 +55,21 @@ public void testStaticZeroKey() { } } + @Test + public void testStaticZeroKeyAllNulls() { + final QueryTable t = createTestTableAllNull(100000, false, false, false, 0x31313131, + CollectionUtil.ZERO_LENGTH_STRING_ARRAY, new TestDataGenerator[0]).t; + final Table result = t.updateBy(UpdateByOperation.CumProd()); + for (String col : t.getDefinition().getColumnNamesArray()) { + if ("boolCol".equals(col)) { + continue; + } + assertWithCumProd(DataAccessHelpers.getColumn(t, col).getDirect(), + DataAccessHelpers.getColumn(result, col).getDirect(), + DataAccessHelpers.getColumn(result, col).getType()); + } + } + // endregion // region Bucketed Tests diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java index 488ff43ec68..9ef95d01e31 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestCumSum.java @@ -5,6 +5,7 @@ import io.deephaven.api.updateby.UpdateByControl; import io.deephaven.api.updateby.UpdateByOperation; +import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.table.PartitionedTable; import io.deephaven.engine.table.Table; @@ -56,6 +57,21 @@ public void testStaticZeroKey() { } } + @Test + public void testStaticZeroKeyAllNulls() { + final QueryTable t = createTestTableAllNull(100000, false, false, false, 0x31313131, + CollectionUtil.ZERO_LENGTH_STRING_ARRAY, new TestDataGenerator[0]).t; + + t.setRefreshing(false); + + final Table summed = t.updateBy(UpdateByOperation.CumSum()); + for (String col : t.getDefinition().getColumnNamesArray()) { + assertWithCumSum(DataAccessHelpers.getColumn(t, col).getDirect(), + DataAccessHelpers.getColumn(summed, col).getDirect(), + DataAccessHelpers.getColumn(summed, col).getType()); + } + } + // endregion // region Bucketed Tests @@ -211,91 +227,7 @@ protected Table e() { */ // endregion - private long[] cumsum(byte[] values) { - if (values == null) { - return null; - } - - if (values.length == 0) { - return new long[0]; - } - - long[] result = new long[values.length]; - result[0] = isNull(values[0]) ? NULL_LONG : values[0]; - - for (int i = 1; i < values.length; i++) { - final boolean curValNull = isNull(values[i]); - if (isNull(result[i - 1])) { - result[i] = curValNull ? NULL_LONG : values[i]; - } else { - if (curValNull) { - result[i] = result[i - 1]; - } else { - result[i] = result[i - 1] + values[i]; - } - } - } - - return result; - } - - private long[] cumsum(short[] values) { - if (values == null) { - return null; - } - - if (values.length == 0) { - return new long[0]; - } - - long[] result = new long[values.length]; - result[0] = isNull(values[0]) ? NULL_LONG : values[0]; - - for (int i = 1; i < values.length; i++) { - final boolean curValNull = isNull(values[i]); - if (isNull(result[i - 1])) { - result[i] = curValNull ? NULL_LONG : values[i]; - } else { - if (curValNull) { - result[i] = result[i - 1]; - } else { - result[i] = result[i - 1] + values[i]; - } - } - } - - return result; - } - - private long[] cumsum(int[] values) { - if (values == null) { - return null; - } - - if (values.length == 0) { - return new long[0]; - } - - long[] result = new long[values.length]; - result[0] = isNull(values[0]) ? NULL_LONG : values[0]; - - for (int i = 1; i < values.length; i++) { - final boolean curValNull = isNull(values[i]); - if (isNull(result[i - 1])) { - result[i] = curValNull ? NULL_LONG : values[i]; - } else { - if (curValNull) { - result[i] = result[i - 1]; - } else { - result[i] = result[i - 1] + values[i]; - } - } - } - - return result; - } - - private long[] cumsum(Boolean[] values) { + private long[] boolean_cumsum(Boolean[] values) { if (values == null) { return null; } @@ -323,7 +255,7 @@ private long[] cumsum(Boolean[] values) { return result; } - public static Object[] cumSum(Object[] values, final boolean isBD) { + public static Object[] big_cumSum(Object[] values, final boolean isBD) { if (values == null) { return null; } @@ -353,11 +285,11 @@ public static Object[] cumSum(Object[] values, final boolean isBD) { final void assertWithCumSum(@NotNull final Object expected, @NotNull final Object actual, Class type) { if (expected instanceof byte[]) { - assertArrayEquals(cumsum((byte[]) expected), (long[]) actual); + assertArrayEquals(Numeric.cumsum((byte[]) expected), (long[]) actual); } else if (expected instanceof short[]) { - assertArrayEquals(cumsum((short[]) expected), (long[]) actual); + assertArrayEquals(Numeric.cumsum((short[]) expected), (long[]) actual); } else if (expected instanceof int[]) { - assertArrayEquals(cumsum((int[]) expected), (long[]) actual); + assertArrayEquals(Numeric.cumsum((int[]) expected), (long[]) actual); } else if (expected instanceof long[]) { assertArrayEquals(Numeric.cumsum((long[]) expected), (long[]) actual); } else if (expected instanceof float[]) { @@ -365,9 +297,9 @@ final void assertWithCumSum(@NotNull final Object expected, @NotNull final Objec } else if (expected instanceof double[]) { assertArrayEquals(Numeric.cumsum((double[]) expected), (double[]) actual, .001d); } else if (expected instanceof Boolean[]) { - assertArrayEquals(cumsum((Boolean[]) expected), (long[]) actual); + assertArrayEquals(boolean_cumsum((Boolean[]) expected), (long[]) actual); } else { - assertArrayEquals(cumSum((Object[]) expected, type == BigDecimal.class), (Object[]) actual); + assertArrayEquals(big_cumSum((Object[]) expected, type == BigDecimal.class), (Object[]) actual); } } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java index a5397d9c45d..c92fe6f46d8 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingSum.java @@ -61,6 +61,24 @@ public EnumSet diffItems() { // region Static Zero Key Tests + @Test + public void testStaticZeroKeyWithAllNullWindows() { + final QueryTable t = createTestTable(10000, false, false, false, 0x31313131).t; + t.setRefreshing(false); + + // With a window size of 1 and 10% null generation, guaranteed to cover the all NULL case. + final int prevTicks = 1; + final int postTicks = 0; + + final Table summed = t.updateBy(UpdateByOperation.RollingSum(prevTicks, postTicks)); + + for (String col : t.getDefinition().getColumnNamesArray()) { + assertWithRollingSumTicks(DataAccessHelpers.getColumn(t, col).getDirect(), + DataAccessHelpers.getColumn(summed, col).getDirect(), + DataAccessHelpers.getColumn(summed, col).getType(), prevTicks, postTicks); + } + } + @Test public void testStaticZeroKeyRev() { final QueryTable t = createTestTable(10000, false, false, false, 0x31313131).t; @@ -313,6 +331,14 @@ public void testNullOnBucketChange() { assertTableEquals(expected, r); } + @Test + public void testStaticBucketedAllNull() { + // With a window size of 1 and 10% null generation, guaranteed to cover the all NULL case. + final int prevTicks = 1; + final int postTicks = 0; + doTestStaticBucketed(false, prevTicks, postTicks); + } + @Test public void testStaticBucketedRev() { final int prevTicks = 100; @@ -445,6 +471,14 @@ private void doTestStaticBucketedTimed(boolean grouped, Duration prevTime, Durat // region Live Tests + @Test + public void testZeroKeyAppendOnlyAllNull() { + // With a window size of 1 and 10% null generation, guaranteed to cover the all NULL case. + final int prevTicks = 1; + final int postTicks = 0; + doTestAppendOnly(false, prevTicks, postTicks); + } + @Test public void testZeroKeyAppendOnlyRev() { final int prevTicks = 100; @@ -480,6 +514,14 @@ public void testZeroKeyAppendOnlyFwdRev() { doTestAppendOnly(false, prevTicks, postTicks); } + @Test + public void testBucketedAppendOnlyAllNull() { + // With a window size of 1 and 10% null generation, guaranteed to cover the all NULL case. + final int prevTicks = 1; + final int postTicks = 0; + doTestAppendOnly(true, prevTicks, postTicks); + } + @Test public void testBucketedAppendOnlyRev() { final int prevTicks = 100; @@ -1193,20 +1235,20 @@ private long[] rollingSum(long[] values, int prevTicks, int postTicks) { return result; } - private float[] rollingSum(float[] values, int prevTicks, int postTicks) { + private double[] rollingSum(float[] values, int prevTicks, int postTicks) { if (values == null) { return null; } if (values.length == 0) { - return new float[0]; + return new double[0]; } - float[] result = new float[values.length]; + double[] result = new double[values.length]; for (int i = 0; i < values.length; i++) { - result[i] = NULL_FLOAT; + result[i] = NULL_DOUBLE; // set the head and the tail final int head = Math.max(0, i - prevTicks + 1); @@ -1215,7 +1257,7 @@ private float[] rollingSum(float[] values, int prevTicks, int postTicks) { // compute everything in this window for (int computeIdx = head; computeIdx <= tail; computeIdx++) { if (!isNull(values[computeIdx])) { - if (result[i] == NULL_FLOAT) { + if (result[i] == NULL_DOUBLE) { result[i] = values[computeIdx]; } else { result[i] += values[computeIdx]; @@ -1534,22 +1576,22 @@ private long[] rollingSumTime(long[] values, long[] timestamps, long prevNanos, return result; } - private float[] rollingSumTime(float[] values, long[] timestamps, long prevNanos, long postNanos) { + private double[] rollingSumTime(float[] values, long[] timestamps, long prevNanos, long postNanos) { if (values == null) { return null; } if (values.length == 0) { - return new float[0]; + return new double[0]; } - float[] result = new float[values.length]; + double[] result = new double[values.length]; int head = 0; int tail = 0; for (int i = 0; i < values.length; i++) { - result[i] = NULL_FLOAT; + result[i] = NULL_DOUBLE; // check the current timestamp. skip if NULL if (timestamps[i] == NULL_LONG) { @@ -1572,7 +1614,7 @@ private float[] rollingSumTime(float[] values, long[] timestamps, long prevNanos // compute everything in this window for (int computeIdx = head; computeIdx < tail; computeIdx++) { if (!isNull(values[computeIdx])) { - if (result[i] == NULL_FLOAT) { + if (result[i] == NULL_DOUBLE) { result[i] = values[computeIdx]; } else { result[i] += values[computeIdx]; @@ -1756,7 +1798,7 @@ final void assertWithRollingSumTicks(@NotNull final Object expected, @NotNull fi } else if (expected instanceof long[]) { assertArrayEquals(rollingSum((long[]) expected, prevTicks, postTicks), (long[]) actual); } else if (expected instanceof float[]) { - assertArrayEquals(rollingSum((float[]) expected, prevTicks, postTicks), (float[]) actual, deltaF); + assertArrayEquals(rollingSum((float[]) expected, prevTicks, postTicks), (double[]) actual, deltaF); } else if (expected instanceof double[]) { assertArrayEquals(rollingSum((double[]) expected, prevTicks, postTicks), (double[]) actual, deltaD); } else if (expected instanceof Boolean[]) { @@ -1785,7 +1827,7 @@ final void assertWithRollingSumTime(@NotNull final Object expected, @NotNull fin } else if (expected instanceof long[]) { assertArrayEquals(rollingSumTime((long[]) expected, timestamps, prevTime, postTime), (long[]) actual); } else if (expected instanceof float[]) { - assertArrayEquals(rollingSumTime((float[]) expected, timestamps, prevTime, postTime), (float[]) actual, + assertArrayEquals(rollingSumTime((float[]) expected, timestamps, prevTime, postTime), (double[]) actual, deltaF); } else if (expected instanceof double[]) { assertArrayEquals(rollingSumTime((double[]) expected, timestamps, prevTime, postTime), (double[]) actual, From 2dbbf32af6677c1ef2a5007e92b9a68d6cf4c05f Mon Sep 17 00:00:00 2001 From: Shivam Malhotra Date: Wed, 8 May 2024 16:41:35 -0500 Subject: [PATCH 7/7] Improved threading capabilities of S3+parquet (#5451) --- .../java/io/deephaven/base/FileUtils.java | 1 - .../util/channel/CachedChannelProvider.java | 10 +- .../SeekableChannelsProviderLoader.java | 15 +- .../channel/CachedChannelProviderTest.java | 26 ++- .../deephaven/util/thread/ThreadHelpers.java | 26 +++ .../OperationInitializationThreadPool.java | 16 +- .../table/impl/QueryTableAggregationTest.java | 2 +- .../parquet/base/ParquetFileReader.java | 86 +++------- .../parquet/table/ParquetInstructions.java | 22 ++- .../parquet/table/ParquetSchemaReader.java | 25 --- .../deephaven/parquet/table/ParquetTools.java | 21 +-- .../DeephavenNestedPartitionLayout.java | 13 +- .../layout/ParquetFlatPartitionedLayout.java | 31 ++-- .../ParquetKeyValuePartitionedLayout.java | 42 ++--- .../layout/ParquetMetadataFileLayout.java | 20 ++- .../table/layout/ParquetSingleFileLayout.java | 39 ----- .../table/location/ParquetTableLocation.java | 2 + .../location/ParquetTableLocationKey.java | 64 +++----- .../table/ParquetTableReadWriteTest.java | 87 ++++++++-- .../parquet/table/TestParquetTools.java | 2 +- .../extensions/s3/S3AsyncClientFactory.java | 151 ++++++++++++++++++ .../extensions/s3/S3SeekableByteChannel.java | 6 +- .../s3/S3SeekableChannelProvider.java | 109 +++++++++---- .../s3/S3SeekableChannelTestBase.java | 6 +- 24 files changed, 496 insertions(+), 326 deletions(-) create mode 100644 Util/src/main/java/io/deephaven/util/thread/ThreadHelpers.java delete mode 100644 extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetSingleFileLayout.java create mode 100644 extensions/s3/src/main/java/io/deephaven/extensions/s3/S3AsyncClientFactory.java diff --git a/Base/src/main/java/io/deephaven/base/FileUtils.java b/Base/src/main/java/io/deephaven/base/FileUtils.java index c0196aba58a..da11eef1289 100644 --- a/Base/src/main/java/io/deephaven/base/FileUtils.java +++ b/Base/src/main/java/io/deephaven/base/FileUtils.java @@ -3,7 +3,6 @@ // package io.deephaven.base; -import io.deephaven.base.verify.Assert; import io.deephaven.base.verify.Require; import org.jetbrains.annotations.Nullable; diff --git a/Util/channel/src/main/java/io/deephaven/util/channel/CachedChannelProvider.java b/Util/channel/src/main/java/io/deephaven/util/channel/CachedChannelProvider.java index 2927ce5b706..bb4f5eecab4 100644 --- a/Util/channel/src/main/java/io/deephaven/util/channel/CachedChannelProvider.java +++ b/Util/channel/src/main/java/io/deephaven/util/channel/CachedChannelProvider.java @@ -58,7 +58,15 @@ enum ChannelType { private final RAPriQueue releasePriority = new RAPriQueue<>(8, PerPathPool.RAPQ_ADAPTER, PerPathPool.class); - public CachedChannelProvider(@NotNull final SeekableChannelsProvider wrappedProvider, + public static CachedChannelProvider create(@NotNull final SeekableChannelsProvider wrappedProvider, + final int maximumPooledCount) { + if (wrappedProvider instanceof CachedChannelProvider) { + throw new IllegalArgumentException("Cannot wrap a CachedChannelProvider in another CachedChannelProvider"); + } + return new CachedChannelProvider(wrappedProvider, maximumPooledCount); + } + + private CachedChannelProvider(@NotNull final SeekableChannelsProvider wrappedProvider, final int maximumPooledCount) { this.wrappedProvider = wrappedProvider; this.maximumPooledCount = Require.gtZero(maximumPooledCount, "maximumPooledCount"); diff --git a/Util/channel/src/main/java/io/deephaven/util/channel/SeekableChannelsProviderLoader.java b/Util/channel/src/main/java/io/deephaven/util/channel/SeekableChannelsProviderLoader.java index 343bf3f7683..98ab4d8c583 100644 --- a/Util/channel/src/main/java/io/deephaven/util/channel/SeekableChannelsProviderLoader.java +++ b/Util/channel/src/main/java/io/deephaven/util/channel/SeekableChannelsProviderLoader.java @@ -37,18 +37,19 @@ private SeekableChannelsProviderLoader() { } /** - * Create a new {@link SeekableChannelsProvider} based on given URI and object using the plugins loaded by the - * {@link ServiceLoader}. For example, for a "S3" URI, we will create a {@link SeekableChannelsProvider} which can - * read files from S3. + * Create a new {@link SeekableChannelsProvider} compatible for reading from and writing to the given URI, using the + * plugins loaded by the {@link ServiceLoader}. For example, for a "S3" URI, we will create a + * {@link SeekableChannelsProvider} which can read files from S3. * * @param uri The URI - * @param object An optional object to pass to the {@link SeekableChannelsProviderPlugin} implementations. + * @param specialInstructions An optional object to pass special instructions to the provider. * @return A {@link SeekableChannelsProvider} for the given URI. */ - public SeekableChannelsProvider fromServiceLoader(@NotNull final URI uri, @Nullable final Object object) { + public SeekableChannelsProvider fromServiceLoader(@NotNull final URI uri, + @Nullable final Object specialInstructions) { for (final SeekableChannelsProviderPlugin plugin : providers) { - if (plugin.isCompatible(uri, object)) { - return plugin.createProvider(uri, object); + if (plugin.isCompatible(uri, specialInstructions)) { + return plugin.createProvider(uri, specialInstructions); } } throw new UnsupportedOperationException("No plugin found for uri: " + uri); diff --git a/Util/channel/src/test/java/io/deephaven/util/channel/CachedChannelProviderTest.java b/Util/channel/src/test/java/io/deephaven/util/channel/CachedChannelProviderTest.java index 4eed02d427e..1bd3339df60 100644 --- a/Util/channel/src/test/java/io/deephaven/util/channel/CachedChannelProviderTest.java +++ b/Util/channel/src/test/java/io/deephaven/util/channel/CachedChannelProviderTest.java @@ -20,6 +20,7 @@ import java.util.function.Supplier; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -32,7 +33,7 @@ public class CachedChannelProviderTest { @Test public void testSimpleRead() throws IOException { final SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); - final CachedChannelProvider cachedChannelProvider = new CachedChannelProvider(wrappedProvider, 100); + final CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 100); for (int ii = 0; ii < 100; ++ii) { final SeekableByteChannel[] sameFile = new SeekableByteChannel[10]; for (int jj = 0; jj < sameFile.length; ++jj) { @@ -55,7 +56,7 @@ public void testSimpleRead() throws IOException { @Test public void testSimpleReadWrite() throws IOException { SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); - CachedChannelProvider cachedChannelProvider = new CachedChannelProvider(wrappedProvider, 100); + CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 100); for (int i = 0; i < 1000; i++) { SeekableByteChannel rc = ((i / 100) % 2 == 0 ? cachedChannelProvider.getReadChannel(wrappedProvider.makeContext(), "r" + i) @@ -69,7 +70,7 @@ public void testSimpleReadWrite() throws IOException { @Test public void testSimpleWrite() throws IOException { SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); - CachedChannelProvider cachedChannelProvider = new CachedChannelProvider(wrappedProvider, 100); + CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 100); for (int i = 0; i < 1000; i++) { SeekableByteChannel rc = cachedChannelProvider.getWriteChannel("w" + i, false); // Call write to hit the assertions inside the mock channel @@ -86,7 +87,7 @@ public void testSimpleWrite() throws IOException { @Test public void testSimpleAppend() throws IOException { SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); - CachedChannelProvider cachedChannelProvider = new CachedChannelProvider(wrappedProvider, 100); + CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 100); for (int i = 0; i < 1000; i++) { SeekableByteChannel rc = cachedChannelProvider.getWriteChannel("a" + i, true); rc.close(); @@ -100,7 +101,7 @@ public void testSimpleAppend() throws IOException { @Test public void testCloseOrder() throws IOException { SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); - CachedChannelProvider cachedChannelProvider = new CachedChannelProvider(wrappedProvider, 100); + CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 100); for (int i = 0; i < 20; i++) { List channels = new ArrayList<>(); for (int j = 0; j < 50; j++) { @@ -121,7 +122,7 @@ public void testCloseOrder() throws IOException { @Test public void testReuse() throws IOException { final SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); - final CachedChannelProvider cachedChannelProvider = new CachedChannelProvider(wrappedProvider, 50); + final CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 50); final SeekableByteChannel[] someResult = new SeekableByteChannel[50]; final ByteBuffer buffer = ByteBuffer.allocate(1); for (int ci = 0; ci < someResult.length; ++ci) { @@ -149,7 +150,7 @@ public void testReuse() throws IOException { @Test public void testReuse10() throws IOException { final SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); - final CachedChannelProvider cachedChannelProvider = new CachedChannelProvider(wrappedProvider, 100); + final CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 100); final SeekableByteChannel[] someResult = new SeekableByteChannel[100]; for (int pi = 0; pi < 10; ++pi) { for (int ci = 0; ci < 10; ++ci) { @@ -173,6 +174,17 @@ public void testReuse10() throws IOException { assertEquals(0, closed.size()); } + @Test + void testRewrapCachedChannelProvider() { + final SeekableChannelsProvider wrappedProvider = new TestChannelProvider(); + final CachedChannelProvider cachedChannelProvider = CachedChannelProvider.create(wrappedProvider, 100); + try { + CachedChannelProvider.create(cachedChannelProvider, 100); + fail("Expected IllegalArgumentException on rewrapping CachedChannelProvider"); + } catch (final IllegalArgumentException expected) { + } + } + private class TestChannelProvider implements SeekableChannelsProvider { diff --git a/Util/src/main/java/io/deephaven/util/thread/ThreadHelpers.java b/Util/src/main/java/io/deephaven/util/thread/ThreadHelpers.java new file mode 100644 index 00000000000..e9e2f9af791 --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/thread/ThreadHelpers.java @@ -0,0 +1,26 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.util.thread; + +import io.deephaven.configuration.Configuration; + +public class ThreadHelpers { + /** + * Get the number of threads to use for a given configuration key, defaulting to the number of available processors + * if the configuration key is set to a non-positive value, or the configuration key is not set and the provided + * default is non-positive. + * + * @param configKey The configuration key to look up + * @param defaultValue The default value to use if the configuration key is not set + * @return The number of threads to use + */ + public static int getOrComputeThreadCountProperty(final String configKey, final int defaultValue) { + final int numThreads = Configuration.getInstance().getIntegerWithDefault(configKey, defaultValue); + if (numThreads <= 0) { + return Runtime.getRuntime().availableProcessors(); + } else { + return numThreads; + } + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/OperationInitializationThreadPool.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/OperationInitializationThreadPool.java index 43498b1029c..6ac54c0f980 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/OperationInitializationThreadPool.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/OperationInitializationThreadPool.java @@ -4,7 +4,6 @@ package io.deephaven.engine.table.impl; import io.deephaven.chunk.util.pools.MultiChunkPool; -import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.updategraph.OperationInitializer; import io.deephaven.util.thread.NamingThreadFactory; @@ -17,6 +16,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static io.deephaven.util.thread.ThreadHelpers.getOrComputeThreadCountProperty; + /** * Implementation of OperationInitializer that delegates to a pool of threads. */ @@ -25,17 +26,8 @@ public class OperationInitializationThreadPool implements OperationInitializer { /** * The number of threads that will be used for parallel initialization in this process */ - public static final int NUM_THREADS; - - static { - final int numThreads = - Configuration.getInstance().getIntegerWithDefault("OperationInitializationThreadPool.threads", -1); - if (numThreads <= 0) { - NUM_THREADS = Runtime.getRuntime().availableProcessors(); - } else { - NUM_THREADS = numThreads; - } - } + private static final int NUM_THREADS = + getOrComputeThreadCountProperty("OperationInitializationThreadPool.threads", -1); private final ThreadLocal isInitializationThread = ThreadLocal.withInitial(() -> false); private final ThreadPoolExecutor executorService; diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java index 0456fb78dcc..edfa267f894 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java @@ -3890,7 +3890,7 @@ public void testMultiPartitionSymbolTableBy() throws IOException { t4.updateView("Date=`2021-07-21`", "Num=400")).moveColumnsUp("Date", "Num"); final Table loaded = ParquetTools.readPartitionedTableInferSchema( - new ParquetKeyValuePartitionedLayout(testRootFile, 2, ParquetInstructions.EMPTY), + new ParquetKeyValuePartitionedLayout(testRootFile.toURI(), 2, ParquetInstructions.EMPTY), ParquetInstructions.EMPTY); // verify the sources are identical diff --git a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java index 3b423e9cff4..1cca18b8b4b 100644 --- a/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java +++ b/extensions/parquet/base/src/main/java/io/deephaven/parquet/base/ParquetFileReader.java @@ -3,21 +3,19 @@ // package io.deephaven.parquet.base; -import io.deephaven.UncheckedDeephavenException; import io.deephaven.util.channel.CachedChannelProvider; import io.deephaven.util.channel.SeekableChannelContext; import io.deephaven.util.channel.SeekableChannelsProvider; -import io.deephaven.util.channel.SeekableChannelsProviderLoader; import org.apache.parquet.format.*; import org.apache.parquet.format.ColumnOrder; import org.apache.parquet.format.Type; import org.apache.parquet.schema.*; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.net.URI; import java.nio.channels.SeekableByteChannel; import java.util.*; @@ -44,94 +42,50 @@ public class ParquetFileReader { /** * Make a {@link ParquetFileReader} for the supplied {@link File}. Wraps {@link IOException} as - * {@link UncheckedDeephavenException}. + * {@link UncheckedIOException}. * * @param parquetFile The parquet file or the parquet metadata file - * @param specialInstructions Optional read instructions to pass to {@link SeekableChannelsProvider} while creating - * channels + * @param channelsProvider The {@link SeekableChannelsProvider} to use for reading the file * @return The new {@link ParquetFileReader} */ public static ParquetFileReader create( @NotNull final File parquetFile, - @Nullable final Object specialInstructions) { + @NotNull final SeekableChannelsProvider channelsProvider) { try { - return createChecked(parquetFile, specialInstructions); - } catch (IOException e) { - throw new UncheckedDeephavenException("Failed to create Parquet file reader: " + parquetFile, e); + return new ParquetFileReader(convertToURI(parquetFile, false), channelsProvider); + } catch (final IOException e) { + throw new UncheckedIOException("Failed to create Parquet file reader: " + parquetFile, e); } } /** * Make a {@link ParquetFileReader} for the supplied {@link URI}. Wraps {@link IOException} as - * {@link UncheckedDeephavenException}. + * {@link UncheckedIOException}. * * @param parquetFileURI The URI for the parquet file or the parquet metadata file - * @param specialInstructions Optional read instructions to pass to {@link SeekableChannelsProvider} while creating - * channels + * @param channelsProvider The {@link SeekableChannelsProvider} to use for reading the file * @return The new {@link ParquetFileReader} */ public static ParquetFileReader create( @NotNull final URI parquetFileURI, - @Nullable final Object specialInstructions) { + @NotNull final SeekableChannelsProvider channelsProvider) { try { - return createChecked(parquetFileURI, specialInstructions); - } catch (IOException e) { - throw new UncheckedDeephavenException("Failed to create Parquet file reader: " + parquetFileURI, e); + return new ParquetFileReader(parquetFileURI, channelsProvider); + } catch (final IOException e) { + throw new UncheckedIOException("Failed to create Parquet file reader: " + parquetFileURI, e); } } - /** - * Make a {@link ParquetFileReader} for the supplied {@link File}. - * - * @param parquetFile The parquet file or the parquet metadata file - * @param specialInstructions Optional read instructions to pass to {@link SeekableChannelsProvider} while creating - * channels - * @return The new {@link ParquetFileReader} - * @throws IOException if an IO exception occurs - */ - public static ParquetFileReader createChecked( - @NotNull final File parquetFile, - @Nullable final Object specialInstructions) throws IOException { - return createChecked(convertToURI(parquetFile, false), specialInstructions); - } - - /** - * Make a {@link ParquetFileReader} for the supplied {@link URI}. - * - * @param parquetFileURI The URI for the parquet file or the parquet metadata file - * @param specialInstructions Optional read instructions to pass to {@link SeekableChannelsProvider} while creating - * channels - * @return The new {@link ParquetFileReader} - * @throws IOException if an IO exception occurs - */ - public static ParquetFileReader createChecked( - @NotNull final URI parquetFileURI, - @Nullable final Object specialInstructions) throws IOException { - final SeekableChannelsProvider provider = SeekableChannelsProviderLoader.getInstance().fromServiceLoader( - parquetFileURI, specialInstructions); - return new ParquetFileReader(parquetFileURI, new CachedChannelProvider(provider, 1 << 7)); - } - - /** - * Create a new ParquetFileReader for the provided source. - * - * @param source The source path or URI for the parquet file or the parquet metadata file - * @param channelsProvider The {@link SeekableChannelsProvider} to use for reading the file - */ - public ParquetFileReader(final String source, final SeekableChannelsProvider channelsProvider) - throws IOException { - this(convertToURI(source, false), channelsProvider); - } - /** * Create a new ParquetFileReader for the provided source. * * @param parquetFileURI The URI for the parquet file or the parquet metadata file - * @param channelsProvider The {@link SeekableChannelsProvider} to use for reading the file + * @param provider The {@link SeekableChannelsProvider} to use for reading the file */ - public ParquetFileReader(final URI parquetFileURI, final SeekableChannelsProvider channelsProvider) - throws IOException { - this.channelsProvider = channelsProvider; + private ParquetFileReader( + @NotNull final URI parquetFileURI, + @NotNull final SeekableChannelsProvider provider) throws IOException { + this.channelsProvider = CachedChannelProvider.create(provider, 1 << 7); if (!parquetFileURI.getRawPath().endsWith(".parquet") && FILE_URI_SCHEME.equals(parquetFileURI.getScheme())) { // Construct a new file URI for the parent directory rootURI = convertToURI(new File(parquetFileURI).getParentFile(), true); @@ -270,7 +224,7 @@ private Set calculateColumnsWithDictionaryUsedOnEveryDataPage() { /** * Create a {@link RowGroupReader} object for provided row group number - * + * * @param version The "version" string from deephaven specific parquet metadata, or null if it's not present. */ public RowGroupReader getRowGroup(final int groupNumber, final String version) { @@ -506,7 +460,7 @@ private static LogicalTypeAnnotation getLogicalTypeAnnotation(final ConvertedTyp /** * Helper method to determine if a logical type is adjusted to UTC. - * + * * @param logicalType the logical type to check * @return true if the logical type is a timestamp adjusted to UTC, false otherwise */ diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java index f67e11bd650..e7e773e1bfe 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java @@ -678,13 +678,13 @@ public static class Builder { private TableDefinition tableDefinition; private Collection> indexColumns; - public Builder() {} - /** - * Creates a new {@link ParquetInstructions} object by only copying the column name to instructions mapping and - * parquet column name to instructions mapping from the given {@link ParquetInstructions} object. For copying - * all properties, use something like {@link ParquetInstructions#withTableDefinition}. + * For each additional field added, make sure to update the copy constructor builder + * {@link #Builder(ParquetInstructions)} */ + + public Builder() {} + public Builder(final ParquetInstructions parquetInstructions) { if (parquetInstructions == EMPTY) { return; @@ -692,6 +692,18 @@ public Builder(final ParquetInstructions parquetInstructions) { final ReadOnly readOnlyParquetInstructions = (ReadOnly) parquetInstructions; columnNameToInstructions = readOnlyParquetInstructions.copyColumnNameToInstructions(); parquetColumnNameToInstructions = readOnlyParquetInstructions.copyParquetColumnNameToInstructions(); + compressionCodecName = readOnlyParquetInstructions.getCompressionCodecName(); + maximumDictionaryKeys = readOnlyParquetInstructions.getMaximumDictionaryKeys(); + maximumDictionarySize = readOnlyParquetInstructions.getMaximumDictionarySize(); + isLegacyParquet = readOnlyParquetInstructions.isLegacyParquet(); + targetPageSize = readOnlyParquetInstructions.getTargetPageSize(); + isRefreshing = readOnlyParquetInstructions.isRefreshing(); + specialInstructions = readOnlyParquetInstructions.getSpecialInstructions(); + generateMetadataFiles = readOnlyParquetInstructions.generateMetadataFiles(); + baseNameForPartitionedParquetData = readOnlyParquetInstructions.baseNameForPartitionedParquetData(); + fileLayout = readOnlyParquetInstructions.getFileLayout().orElse(null); + tableDefinition = readOnlyParquetInstructions.getTableDefinition().orElse(null); + indexColumns = readOnlyParquetInstructions.getIndexColumns().orElse(null); } private void newColumnNameToInstructionsMap() { diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java index a8e599b04e0..466b7ee1096 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetSchemaReader.java @@ -34,7 +34,6 @@ import org.apache.parquet.schema.PrimitiveType; import org.jetbrains.annotations.NotNull; -import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.time.Instant; @@ -95,30 +94,6 @@ void reset() { } } - /** - * Obtain schema information from a parquet file - * - * @param filePath Location for input parquet file - * @param readInstructions Parquet read instructions specifying transformations like column mappings and codecs. - * Note a new read instructions based on this one may be returned by this method to provide necessary - * transformations, eg, replacing unsupported characters like ' ' (space) in column names. - * @param consumer A ColumnDefinitionConsumer whose accept method would be called for each column in the file - * @return Parquet read instructions, either the ones supplied or a new object based on the supplied with necessary - * transformations added. - */ - public static ParquetInstructions readParquetSchema( - @NotNull final String filePath, - @NotNull final ParquetInstructions readInstructions, - @NotNull final ColumnDefinitionConsumer consumer, - @NotNull final BiFunction, String> legalizeColumnNameFunc) throws IOException { - final ParquetFileReader parquetFileReader = - ParquetFileReader.createChecked(new File(filePath), readInstructions.getSpecialInstructions()); - final ParquetMetadata parquetMetadata = - new ParquetMetadataConverter().fromParquetMetadata(parquetFileReader.fileMetaData); - return readParquetSchema(parquetFileReader.getSchema(), parquetMetadata.getFileMetaData().getKeyValueMetaData(), - readInstructions, consumer, legalizeColumnNameFunc); - } - public static Optional parseMetadata(@NotNull final Map keyValueMetadata) { final String tableInfoRaw = keyValueMetadata.get(METADATA_KEY); if (tableInfoRaw == null) { diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java index 9b56ae15bb8..a5adb0915c5 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetTools.java @@ -41,7 +41,6 @@ import io.deephaven.parquet.table.layout.ParquetFlatPartitionedLayout; import io.deephaven.parquet.table.layout.ParquetKeyValuePartitionedLayout; import io.deephaven.parquet.table.layout.ParquetMetadataFileLayout; -import io.deephaven.parquet.table.layout.ParquetSingleFileLayout; import io.deephaven.parquet.table.location.ParquetTableLocationFactory; import io.deephaven.parquet.table.location.ParquetTableLocationKey; import io.deephaven.parquet.table.ParquetInstructions.ParquetFileLayout; @@ -103,7 +102,6 @@ private ParquetTools() {} * * @param source The path or URI of file or directory to examine * @return table - * @see ParquetSingleFileLayout * @see ParquetMetadataFileLayout * @see ParquetKeyValuePartitionedLayout * @see ParquetFlatPartitionedLayout @@ -127,7 +125,6 @@ public static Table readTable(@NotNull final String source) { * @param source The path or URI of file or directory to examine * @param readInstructions Instructions for customizations while reading * @return table - * @see ParquetSingleFileLayout * @see ParquetMetadataFileLayout * @see ParquetKeyValuePartitionedLayout * @see ParquetFlatPartitionedLayout @@ -174,7 +171,6 @@ public static Table readTable( * * @param sourceFile The file or directory to examine * @return table - * @see ParquetSingleFileLayout * @see ParquetMetadataFileLayout * @see ParquetKeyValuePartitionedLayout * @see ParquetFlatPartitionedLayout @@ -204,7 +200,6 @@ public static Table readTable(@NotNull final File sourceFile) { * @param sourceFile The file or directory to examine * @param readInstructions Instructions for customizations while reading * @return table - * @see ParquetSingleFileLayout * @see ParquetMetadataFileLayout * @see ParquetKeyValuePartitionedLayout * @see ParquetFlatPartitionedLayout @@ -1378,8 +1373,8 @@ public static Table readPartitionedTableInferSchema( } private static Pair infer( - KnownLocationKeyFinder inferenceKeys, - ParquetInstructions readInstructions) { + final KnownLocationKeyFinder inferenceKeys, + final ParquetInstructions readInstructions) { // TODO(deephaven-core#877): Support schema merge when discovering multiple parquet files final ParquetTableLocationKey lastKey = inferenceKeys.getLastKey().orElse(null); if (lastKey == null) { @@ -1661,14 +1656,13 @@ private static Table readSingleFileTable( @NotNull final URI parquetFileURI, @NotNull final ParquetInstructions readInstructions) { verifyFileLayout(readInstructions, ParquetFileLayout.SINGLE_FILE); + final ParquetTableLocationKey locationKey = + new ParquetTableLocationKey(parquetFileURI, 0, null, readInstructions); if (readInstructions.getTableDefinition().isPresent()) { - return readTable(new ParquetTableLocationKey(parquetFileURI, 0, null, readInstructions), - readInstructions); + return readTable(locationKey, readInstructions); } // Infer the table definition - final TableLocationKeyFinder singleFileLayout = - new ParquetSingleFileLayout(parquetFileURI, readInstructions); - final KnownLocationKeyFinder inferenceKeys = toKnownKeys(singleFileLayout); + final KnownLocationKeyFinder inferenceKeys = new KnownLocationKeyFinder<>(locationKey); final Pair inference = infer(inferenceKeys, readInstructions); final TableDefinition inferredTableDefinition = inference.getFirst(); final ParquetInstructions inferredInstructions = inference.getSecond(); @@ -1722,8 +1716,9 @@ public static Table readParquetSchemaAndTable( @NotNull final File source, @NotNull final ParquetInstructions readInstructionsIn, @Nullable final MutableObject mutableInstructionsOut) { + final URI sourceURI = convertToURI(source, false); final ParquetTableLocationKey tableLocationKey = - new ParquetTableLocationKey(source, 0, null, readInstructionsIn); + new ParquetTableLocationKey(sourceURI, 0, null, readInstructionsIn); final Pair>, ParquetInstructions> schemaInfo = ParquetSchemaReader.convertSchema( tableLocationKey.getFileReader().getSchema(), tableLocationKey.getMetadata().getFileMetaData().getKeyValueMetaData(), diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/DeephavenNestedPartitionLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/DeephavenNestedPartitionLayout.java index 056312818c7..830e3e719cf 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/DeephavenNestedPartitionLayout.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/DeephavenNestedPartitionLayout.java @@ -9,6 +9,8 @@ import io.deephaven.parquet.table.ParquetInstructions; import io.deephaven.parquet.table.location.ParquetTableLocationKey; import io.deephaven.util.annotations.VisibleForTesting; +import io.deephaven.util.channel.SeekableChannelsProvider; +import io.deephaven.util.channel.SeekableChannelsProviderLoader; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -24,6 +26,10 @@ import java.util.function.Consumer; import java.util.function.Predicate; +import java.net.URI; + +import static io.deephaven.base.FileUtils.convertToURI; + /** * {@link TableLocationKeyFinder} that will traverse a directory hierarchy laid out in Deephaven's "nested-partitioned" * format, e.g. @@ -47,13 +53,16 @@ public static DeephavenNestedPartitionLayout forParquet @NotNull final String columnPartitionKey, @Nullable final Predicate internalPartitionValueFilter, @NotNull final ParquetInstructions readInstructions) { + final SeekableChannelsProvider channelsProvider = + SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(tableRootDirectory, true), + readInstructions.getSpecialInstructions()); return new DeephavenNestedPartitionLayout<>(tableRootDirectory, tableName, columnPartitionKey, internalPartitionValueFilter) { @Override protected ParquetTableLocationKey makeKey(@NotNull Path tableLeafDirectory, @NotNull Map> partitions) { - return new ParquetTableLocationKey(tableLeafDirectory.resolve(PARQUET_FILE_NAME).toFile(), 0, - partitions, readInstructions); + final URI fileURI = convertToURI(tableLeafDirectory.resolve(PARQUET_FILE_NAME), false); + return new ParquetTableLocationKey(fileURI, 0, partitions, readInstructions, channelsProvider); } }; } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java index 31914804dc1..71f0068383b 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetFlatPartitionedLayout.java @@ -3,7 +3,6 @@ // package io.deephaven.parquet.table.layout; -import io.deephaven.base.FileUtils; import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.locations.impl.TableLocationKeyFinder; import io.deephaven.parquet.base.ParquetUtils; @@ -30,23 +29,10 @@ */ public final class ParquetFlatPartitionedLayout implements TableLocationKeyFinder { - private static ParquetTableLocationKey locationKey(@NotNull final URI uri, - @NotNull final ParquetInstructions readInstructions) { - return new ParquetTableLocationKey(uri, 0, null, readInstructions); - } - private final URI tableRootDirectory; private final Map cache; private final ParquetInstructions readInstructions; - - /** - * @param tableRootDirectory The directory to search for .parquet files. - * @param readInstructions the instructions for customizations while reading - */ - public ParquetFlatPartitionedLayout(@NotNull final File tableRootDirectory, - @NotNull final ParquetInstructions readInstructions) { - this(FileUtils.convertToURI(tableRootDirectory, true), readInstructions); - } + private final SeekableChannelsProvider channelsProvider; /** * @param tableRootDirectoryURI The directory URI to search for .parquet files. @@ -57,6 +43,8 @@ public ParquetFlatPartitionedLayout(@NotNull final URI tableRootDirectoryURI, this.tableRootDirectory = tableRootDirectoryURI; this.cache = Collections.synchronizedMap(new HashMap<>()); this.readInstructions = readInstructions; + this.channelsProvider = SeekableChannelsProviderLoader.getInstance().fromServiceLoader(tableRootDirectory, + readInstructions.getSpecialInstructions()); } public String toString() { @@ -74,19 +62,14 @@ public void findKeys(@NotNull final Consumer locationKe } else { uriFilter = uri -> uri.getPath().endsWith(ParquetUtils.PARQUET_FILE_EXTENSION); } - try (final SeekableChannelsProvider provider = SeekableChannelsProviderLoader.getInstance().fromServiceLoader( - tableRootDirectory, readInstructions.getSpecialInstructions()); - final Stream stream = provider.list(tableRootDirectory)) { + try (final Stream stream = channelsProvider.list(tableRootDirectory)) { stream.filter(uriFilter).forEach(uri -> { cache.compute(uri, (key, existingLocationKey) -> { if (existingLocationKey != null) { locationKeyObserver.accept(existingLocationKey); return existingLocationKey; } - final ParquetTableLocationKey newLocationKey = locationKey(uri, readInstructions); - if (!newLocationKey.verifyFileReader()) { - return null; - } + final ParquetTableLocationKey newLocationKey = locationKey(uri); locationKeyObserver.accept(newLocationKey); return newLocationKey; }); @@ -95,4 +78,8 @@ public void findKeys(@NotNull final Consumer locationKe throw new TableDataException("Error finding parquet locations under " + tableRootDirectory, e); } } + + private ParquetTableLocationKey locationKey(@NotNull final URI uri) { + return new ParquetTableLocationKey(uri, 0, null, readInstructions, channelsProvider); + } } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java index 4836a176066..53a057c0c95 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetKeyValuePartitionedLayout.java @@ -19,7 +19,6 @@ import io.deephaven.util.channel.SeekableChannelsProviderLoader; import org.jetbrains.annotations.NotNull; -import java.io.File; import java.io.IOException; import java.net.URI; import java.nio.file.Path; @@ -48,42 +47,50 @@ public class ParquetKeyValuePartitionedLayout extends URIStreamKeyValuePartitionLayout implements TableLocationKeyFinder { - private final ParquetInstructions readInstructions; + private final SeekableChannelsProvider channelsProvider; public ParquetKeyValuePartitionedLayout( - @NotNull final File tableRootDirectory, + @NotNull final URI tableRootDirectory, @NotNull final TableDefinition tableDefinition, @NotNull final ParquetInstructions readInstructions) { - this(convertToURI(tableRootDirectory, true), tableDefinition, readInstructions); + this(tableRootDirectory, tableDefinition, readInstructions, + SeekableChannelsProviderLoader.getInstance().fromServiceLoader(tableRootDirectory, + readInstructions.getSpecialInstructions())); } - public ParquetKeyValuePartitionedLayout( + private ParquetKeyValuePartitionedLayout( @NotNull final URI tableRootDirectory, @NotNull final TableDefinition tableDefinition, - @NotNull final ParquetInstructions readInstructions) { + @NotNull final ParquetInstructions readInstructions, + @NotNull final SeekableChannelsProvider channelsProvider) { super(tableRootDirectory, () -> new LocationTableBuilderDefinition(tableDefinition), - (uri, partitions) -> new ParquetTableLocationKey(uri, 0, partitions, readInstructions), + (uri, partitions) -> new ParquetTableLocationKey(uri, 0, partitions, readInstructions, + channelsProvider), Math.toIntExact(tableDefinition.getColumnStream().filter(ColumnDefinition::isPartitioning).count())); - this.readInstructions = readInstructions; + this.channelsProvider = channelsProvider; } public ParquetKeyValuePartitionedLayout( - @NotNull final File tableRootDirectory, + @NotNull final URI tableRootDirectory, final int maxPartitioningLevels, @NotNull final ParquetInstructions readInstructions) { - this(convertToURI(tableRootDirectory, true), maxPartitioningLevels, readInstructions); + this(tableRootDirectory, maxPartitioningLevels, readInstructions, + SeekableChannelsProviderLoader.getInstance().fromServiceLoader(tableRootDirectory, + readInstructions.getSpecialInstructions())); } - public ParquetKeyValuePartitionedLayout( + private ParquetKeyValuePartitionedLayout( @NotNull final URI tableRootDirectory, final int maxPartitioningLevels, - @NotNull final ParquetInstructions readInstructions) { + @NotNull final ParquetInstructions readInstructions, + @NotNull final SeekableChannelsProvider channelsProvider) { super(tableRootDirectory, () -> new LocationTableBuilderCsv(tableRootDirectory), - (uri, partitions) -> new ParquetTableLocationKey(uri, 0, partitions, readInstructions), + (uri, partitions) -> new ParquetTableLocationKey(uri, 0, partitions, readInstructions, + channelsProvider), maxPartitioningLevels); - this.readInstructions = readInstructions; + this.channelsProvider = channelsProvider; } @Override @@ -95,11 +102,8 @@ public final void findKeys(@NotNull final Consumer loca } else { uriFilter = uri -> uri.getPath().endsWith(ParquetUtils.PARQUET_FILE_EXTENSION); } - try (final SeekableChannelsProvider provider = SeekableChannelsProviderLoader.getInstance().fromServiceLoader( - tableRootDirectory, readInstructions.getSpecialInstructions()); - final Stream uriStream = provider.walk(tableRootDirectory)) { - final Stream filteredStream = uriStream.filter(uriFilter); - findKeys(filteredStream, locationKeyObserver); + try (final Stream filteredUriStream = channelsProvider.walk(tableRootDirectory).filter(uriFilter)) { + findKeys(filteredUriStream, locationKeyObserver); } catch (final IOException e) { throw new TableDataException("Error finding parquet locations under " + tableRootDirectory, e); } diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java index 82eee8e2353..bd0588770b5 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetMetadataFileLayout.java @@ -16,6 +16,8 @@ import io.deephaven.parquet.table.location.ParquetTableLocationKey; import io.deephaven.parquet.table.ParquetInstructions; import io.deephaven.parquet.base.ParquetFileReader; +import io.deephaven.util.channel.SeekableChannelsProvider; +import io.deephaven.util.channel.SeekableChannelsProviderLoader; import org.apache.commons.io.FilenameUtils; import org.apache.parquet.format.converter.ParquetMetadataConverter; import io.deephaven.util.type.TypeUtils; @@ -28,6 +30,7 @@ import java.io.File; import java.io.IOException; +import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -39,6 +42,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static io.deephaven.base.FileUtils.convertToURI; import static io.deephaven.parquet.base.ParquetUtils.COMMON_METADATA_FILE_NAME; import static io.deephaven.parquet.base.ParquetUtils.METADATA_FILE_NAME; import static io.deephaven.parquet.base.ParquetUtils.METADATA_KEY; @@ -69,6 +73,7 @@ public class ParquetMetadataFileLayout implements TableLocationKeyFinder keys; + private final SeekableChannelsProvider channelsProvider; public ParquetMetadataFileLayout(@NotNull final File directory) { this(directory, ParquetInstructions.EMPTY); @@ -96,12 +101,13 @@ public ParquetMetadataFileLayout( } this.metadataFile = metadataFile; this.commonMetadataFile = commonMetadataFile; + channelsProvider = + SeekableChannelsProviderLoader.getInstance().fromServiceLoader(convertToURI(metadataFile, false), + inputInstructions.getSpecialInstructions()); if (!metadataFile.exists()) { throw new TableDataException(String.format("Parquet metadata file %s does not exist", metadataFile)); } - final ParquetFileReader metadataFileReader = - ParquetFileReader.create(metadataFile, inputInstructions.getSpecialInstructions()); - + final ParquetFileReader metadataFileReader = ParquetFileReader.create(metadataFile, channelsProvider); final ParquetMetadataConverter converter = new ParquetMetadataConverter(); final ParquetMetadata metadataFileMetadata = convertMetadata(metadataFile, metadataFileReader, converter); final Pair>, ParquetInstructions> leafSchemaInfo = ParquetSchemaReader.convertSchema( @@ -111,7 +117,7 @@ public ParquetMetadataFileLayout( if (commonMetadataFile != null && commonMetadataFile.exists()) { final ParquetFileReader commonMetadataFileReader = - ParquetFileReader.create(commonMetadataFile, inputInstructions.getSpecialInstructions()); + ParquetFileReader.create(commonMetadataFile, channelsProvider); final Pair>, ParquetInstructions> fullSchemaInfo = ParquetSchemaReader.convertSchema( commonMetadataFileReader.getSchema(), @@ -202,9 +208,9 @@ public ParquetMetadataFileLayout( partitions.put(partitionKey, partitionValue); } } - final File partitionFile = new File(directory, relativePathString); - final ParquetTableLocationKey tlk = new ParquetTableLocationKey(partitionFile, - partitionOrder.getAndIncrement(), partitions, inputInstructions); + final URI partitionFileURI = convertToURI(new File(directory, relativePathString), false); + final ParquetTableLocationKey tlk = new ParquetTableLocationKey(partitionFileURI, + partitionOrder.getAndIncrement(), partitions, inputInstructions, channelsProvider); tlk.setFileReader(metadataFileReader); tlk.setMetadata(getParquetMetadataForFile(relativePathString, metadataFileMetadata)); tlk.setRowGroupIndices(rowGroupIndices); diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetSingleFileLayout.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetSingleFileLayout.java deleted file mode 100644 index cc328eaba63..00000000000 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/layout/ParquetSingleFileLayout.java +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.parquet.table.layout; - -import io.deephaven.engine.table.impl.locations.impl.TableLocationKeyFinder; -import io.deephaven.parquet.table.ParquetInstructions; -import io.deephaven.parquet.table.location.ParquetTableLocationKey; -import org.jetbrains.annotations.NotNull; - -import java.net.URI; -import java.util.function.Consumer; - -/** - * Parquet {@link TableLocationKeyFinder location finder} that will discover a single file. - */ -public final class ParquetSingleFileLayout implements TableLocationKeyFinder { - private final URI parquetFileUri; - private final ParquetInstructions readInstructions; - - /** - * @param parquetFileUri URI of single parquet file to find - * @param readInstructions the instructions for customizations while reading - */ - public ParquetSingleFileLayout(@NotNull final URI parquetFileUri, - @NotNull final ParquetInstructions readInstructions) { - this.parquetFileUri = parquetFileUri; - this.readInstructions = readInstructions; - } - - public String toString() { - return ParquetSingleFileLayout.class.getSimpleName() + '[' + parquetFileUri + ']'; - } - - @Override - public void findKeys(@NotNull final Consumer locationKeyObserver) { - locationKeyObserver.accept(new ParquetTableLocationKey(parquetFileUri, 0, null, readInstructions)); - } -} diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java index 7d77ba5e240..da0efbefbe8 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocation.java @@ -82,6 +82,8 @@ public ParquetTableLocation(@NotNull final TableKey tableKey, final ParquetMetadata parquetMetadata; // noinspection SynchronizationOnLocalVariableOrMethodParameter synchronized (tableLocationKey) { + // Following methods are internally synchronized, we synchronize them together here to minimize lock/unlock + // calls parquetFileReader = tableLocationKey.getFileReader(); parquetMetadata = tableLocationKey.getMetadata(); rowGroupIndices = tableLocationKey.getRowGroupIndices(); diff --git a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java index a33cd48f609..cafff0a3210 100644 --- a/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java +++ b/extensions/parquet/table/src/main/java/io/deephaven/parquet/table/location/ParquetTableLocationKey.java @@ -8,6 +8,8 @@ import io.deephaven.engine.table.impl.locations.TableDataException; import io.deephaven.engine.table.impl.locations.TableLocationKey; import io.deephaven.parquet.base.ParquetFileReader; +import io.deephaven.util.channel.SeekableChannelsProvider; +import io.deephaven.util.channel.SeekableChannelsProviderLoader; import org.apache.commons.io.FilenameUtils; import org.apache.parquet.format.converter.ParquetMetadataConverter; import org.apache.parquet.format.RowGroup; @@ -15,7 +17,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.File; import java.io.IOException; import java.net.URI; import java.util.List; @@ -35,23 +36,27 @@ public class ParquetTableLocationKey extends URITableLocationKey { private ParquetFileReader fileReader; private ParquetMetadata metadata; private int[] rowGroupIndices; - private final ParquetInstructions readInstructions; + private SeekableChannelsProvider channelsProvider; /** - * Construct a new ParquetTableLocationKey for the supplied {@code file} and {@code partitions}. + * Construct a new ParquetTableLocationKey for the supplied {@code parquetFileUri} and {@code partitions}. + *

+ * This constructor will create a new {@link SeekableChannelsProvider} for reading the file. If you have multiple + * location keys that should share a provider, use the other constructor and set the provider manually. * - * @param file The parquet file that backs the keyed location. Will be adjusted to an absolute path. + * @param parquetFileUri The parquet file that backs the keyed location. Will be adjusted to an absolute path. * @param order Explicit ordering index, taking precedence over other fields * @param partitions The table partitions enclosing the table location keyed by {@code this}. Note that if this * parameter is {@code null}, the location will be a member of no partitions. An ordered copy of the map will * be made, so the calling code is free to mutate the map after this call * @param readInstructions the instructions for customizations while reading */ - public ParquetTableLocationKey(@NotNull final File file, final int order, + public ParquetTableLocationKey(@NotNull final URI parquetFileUri, final int order, @Nullable final Map> partitions, @NotNull final ParquetInstructions readInstructions) { - super(validateParquetFile(file), order, partitions); - this.readInstructions = readInstructions; + this(parquetFileUri, order, partitions, readInstructions, + SeekableChannelsProviderLoader.getInstance().fromServiceLoader(parquetFileUri, + readInstructions.getSpecialInstructions())); } /** @@ -63,21 +68,20 @@ public ParquetTableLocationKey(@NotNull final File file, final int order, * parameter is {@code null}, the location will be a member of no partitions. An ordered copy of the map will * be made, so the calling code is free to mutate the map after this call * @param readInstructions the instructions for customizations while reading + * @param channelsProvider the provider for reading the file */ public ParquetTableLocationKey(@NotNull final URI parquetFileUri, final int order, @Nullable final Map> partitions, - @NotNull final ParquetInstructions readInstructions) { + @NotNull final ParquetInstructions readInstructions, + @NotNull final SeekableChannelsProvider channelsProvider) { super(validateParquetFile(parquetFileUri), order, partitions); - this.readInstructions = readInstructions; - } - - private static URI validateParquetFile(@NotNull final File file) { - return validateParquetFile(convertToURI(file, false)); + this.channelsProvider = channelsProvider; } private static URI validateParquetFile(@NotNull final URI parquetFileUri) { if (!parquetFileUri.getRawPath().endsWith(PARQUET_FILE_EXTENSION)) { - throw new IllegalArgumentException("Parquet file must end in " + PARQUET_FILE_EXTENSION); + throw new IllegalArgumentException("Parquet file must end in " + PARQUET_FILE_EXTENSION + ", found: " + + parquetFileUri.getRawPath()); } return parquetFileUri; } @@ -87,36 +91,6 @@ public String getImplementationName() { return IMPLEMENTATION_NAME; } - /** - * Returns {@code true} if a previous {@link ParquetFileReader} has been created, or if one was successfully created - * on-demand. - * - *

- * When {@code false}, this may mean that the file: - *

    - *
  1. does not exist, or is otherwise inaccessible
    2. - *
  2. is in the process of being written, and is not yet a valid parquet file
    3. - *
  3. is _not_ a parquet file
    4. - *
  4. is a corrupt parquet file
    5. - *
- * - * Callers wishing to handle these cases more explicit may call - * {@link ParquetFileReader#createChecked(URI, Object)}. - * - * @return true if the file reader exists or was successfully created - */ - public synchronized boolean verifyFileReader() { - if (fileReader != null) { - return true; - } - try { - fileReader = ParquetFileReader.createChecked(uri, readInstructions.getSpecialInstructions()); - } catch (IOException e) { - return false; - } - return true; - } - /** * Get a previously-{@link #setFileReader(ParquetFileReader) set} or on-demand created {@link ParquetFileReader} for * this location key's {@code file}. @@ -127,7 +101,7 @@ public synchronized ParquetFileReader getFileReader() { if (fileReader != null) { return fileReader; } - return fileReader = ParquetFileReader.create(uri, readInstructions.getSpecialInstructions()); + return fileReader = ParquetFileReader.create(uri, channelsProvider); } /** diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java index 44ed9586a61..1fc4cf47f16 100644 --- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java +++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/ParquetTableReadWriteTest.java @@ -746,6 +746,13 @@ public void flatPartitionedParquetWithMetadataTest() throws IOException { final Table fromDiskWithMetadataWithoutData = readTable(metadataFile); assertEquals(source.size(), fromDiskWithMetadataWithoutData.size()); + // If we call select now, this should fail because the data files are empty + try { + fromDiskWithMetadataWithoutData.select(); + fail("Expected exception when reading table with empty data files"); + } catch (final RuntimeException expected) { + } + // Now write with flat partitioned parquet files to different directories with metadata file parentDir.delete(); final File updatedSecondDataFile = new File(rootFile, "testDir/data2.parquet"); @@ -786,6 +793,56 @@ public void flatPartitionedParquetWithBigDecimalMetadataTest() throws IOExceptio assertTableEquals(expected, fromDiskWithMetadata); } + @Test + public void keyValuePartitionedWithMetadataTest() throws IOException { + final TableDefinition definition = TableDefinition.of( + ColumnDefinition.ofInt("PC1").withPartitioning(), + ColumnDefinition.ofInt("PC2").withPartitioning(), + ColumnDefinition.ofLong("I")); + final Table source = ((QueryTable) TableTools.emptyTable(1_000_000) + .updateView("PC1 = (int)(ii%3)", + "PC2 = (int)(ii%2)", + "I = ii")) + .withDefinitionUnsafe(definition); + + final File parentDir = new File(rootFile, "keyValuePartitionedWithMetadataTest"); + final ParquetInstructions writeInstructions = ParquetInstructions.builder() + .setGenerateMetadataFiles(true) + .setBaseNameForPartitionedParquetData("data") + .build(); + writeKeyValuePartitionedTable(source, parentDir.getAbsolutePath(), writeInstructions); + + final Table fromDisk = readTable(parentDir); + assertTableEquals(source.sort("PC1", "PC2"), fromDisk.sort("PC1", "PC2")); + + final File metadataFile = new File(parentDir, "_metadata"); + final Table fromDiskWithMetadata = readTable(metadataFile); + assertTableEquals(source.sort("PC1", "PC2"), fromDiskWithMetadata.sort("PC1", "PC2")); + + final File firstDataFile = + new File(parentDir, "PC1=0" + File.separator + "PC2=0" + File.separator + "data.parquet"); + final File secondDataFile = + new File(parentDir, "PC1=0" + File.separator + "PC2=1" + File.separator + "data.parquet"); + assertTrue(firstDataFile.exists()); + assertTrue(secondDataFile.exists()); + + // Now replace the underlying data files with empty files and read the size from metadata file verifying that + // we can read the size without touching the data + firstDataFile.delete(); + firstDataFile.createNewFile(); + secondDataFile.delete(); + secondDataFile.createNewFile(); + final Table fromDiskWithMetadataWithoutData = readTable(metadataFile); + assertEquals(source.size(), fromDiskWithMetadataWithoutData.size()); + + // If we call select now, this should fail because the data files are empty + try { + fromDiskWithMetadataWithoutData.select(); + fail("Expected exception when reading table with empty data files"); + } catch (final RuntimeException expected) { + } + } + @Test public void writeKeyValuePartitionedDataWithIntegerPartitionsTest() { final TableDefinition definition = TableDefinition.of( @@ -1404,7 +1461,8 @@ public void testArrayColumns() { writeReadTableTest(arrayTable, dest, writeInstructions); // Make sure the column didn't use dictionary encoding - ParquetMetadata metadata = new ParquetTableLocationKey(dest, 0, null, ParquetInstructions.EMPTY).getMetadata(); + ParquetMetadata metadata = + new ParquetTableLocationKey(dest.toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); String firstColumnMetadata = metadata.getBlocks().get(0).getColumns().get(0).toString(); assertTrue(firstColumnMetadata.contains("someStringArrayColumn") && !firstColumnMetadata.contains("RLE_DICTIONARY")); @@ -1413,7 +1471,7 @@ public void testArrayColumns() { writeReadTableTest(vectorTable, dest, writeInstructions); // Make sure the column didn't use dictionary encoding - metadata = new ParquetTableLocationKey(dest, 0, null, ParquetInstructions.EMPTY).getMetadata(); + metadata = new ParquetTableLocationKey(dest.toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); firstColumnMetadata = metadata.getBlocks().get(0).getColumns().get(0).toString(); assertTrue(firstColumnMetadata.contains("someStringArrayColumn") && !firstColumnMetadata.contains("RLE_DICTIONARY")); @@ -1699,7 +1757,7 @@ public void testReadOldParquetData() { String path = ParquetTableReadWriteTest.class.getResource("/ReferenceParquetData.parquet").getFile(); readParquetFileFromGitLFS(new File(path)).select(); final ParquetMetadata metadata = - new ParquetTableLocationKey(new File(path), 0, null, ParquetInstructions.EMPTY).getMetadata(); + new ParquetTableLocationKey(new File(path).toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); assertTrue(metadata.getFileMetaData().getKeyValueMetaData().get("deephaven").contains("\"version\":\"0.4.0\"")); path = ParquetTableReadWriteTest.class.getResource("/ReferenceParquetVectorData.parquet").getFile(); @@ -2083,7 +2141,7 @@ private void indexedColumnsBasicWriteTestsImpl(TestParquetTableWriter writer) { // Verify that the key-value metadata in the file has the correct name ParquetTableLocationKey tableLocationKey = - new ParquetTableLocationKey(destFile, 0, null, ParquetInstructions.EMPTY); + new ParquetTableLocationKey(destFile.toURI(), 0, null, ParquetInstructions.EMPTY); String metadataString = tableLocationKey.getMetadata().getFileMetaData().toString(); assertTrue(metadataString.contains(vvvIndexFilePath)); @@ -2116,7 +2174,7 @@ public void legacyGroupingFileReadTest() { // Verify that the key-value metadata in the file has the correct legacy grouping file name final ParquetTableLocationKey tableLocationKey = - new ParquetTableLocationKey(destFile, 0, null, ParquetInstructions.EMPTY); + new ParquetTableLocationKey(destFile.toURI(), 0, null, ParquetInstructions.EMPTY); final String metadataString = tableLocationKey.getMetadata().getFileMetaData().toString(); String groupingFileName = ParquetTools.legacyGroupingFileName(destFile, groupingColName); assertTrue(metadataString.contains(groupingFileName)); @@ -2281,10 +2339,10 @@ public void writeMultiTableIndexTest() { // Verify that the key-value metadata in the file has the correct name ParquetTableLocationKey tableLocationKey = - new ParquetTableLocationKey(firstDestFile, 0, null, ParquetInstructions.EMPTY); + new ParquetTableLocationKey(firstDestFile.toURI(), 0, null, ParquetInstructions.EMPTY); String metadataString = tableLocationKey.getMetadata().getFileMetaData().toString(); assertTrue(metadataString.contains(firstIndexFilePath)); - tableLocationKey = new ParquetTableLocationKey(secondDestFile, 0, null, ParquetInstructions.EMPTY); + tableLocationKey = new ParquetTableLocationKey(secondDestFile.toURI(), 0, null, ParquetInstructions.EMPTY); metadataString = tableLocationKey.getMetadata().getFileMetaData().toString(); assertTrue(metadataString.contains(secondIndexFilePath)); @@ -2334,7 +2392,7 @@ private void indexOverwritingTestsImpl(TestParquetTableWriter writer) { checkSingleTable(anotherTableToSave, destFile); ParquetTableLocationKey tableLocationKey = - new ParquetTableLocationKey(destFile, 0, null, ParquetInstructions.EMPTY); + new ParquetTableLocationKey(destFile.toURI(), 0, null, ParquetInstructions.EMPTY); String metadataString = tableLocationKey.getMetadata().getFileMetaData().toString(); assertTrue(metadataString.contains(xxxIndexFilePath) && !metadataString.contains(vvvIndexFilePath)); @@ -2350,7 +2408,7 @@ private void indexOverwritingTestsImpl(TestParquetTableWriter writer) { Map.of("vvv", new String[] {vvvIndexFilePath}, "xxx", new String[] {xxxIndexFilePath})); - tableLocationKey = new ParquetTableLocationKey(destFile, 0, null, ParquetInstructions.EMPTY); + tableLocationKey = new ParquetTableLocationKey(destFile.toURI(), 0, null, ParquetInstructions.EMPTY); metadataString = tableLocationKey.getMetadata().getFileMetaData().toString(); assertTrue(metadataString.contains(xxxIndexFilePath) && !metadataString.contains(vvvIndexFilePath) && !metadataString.contains(backupXXXIndexFileName)); @@ -2442,7 +2500,7 @@ public void dictionaryEncodingTest() { // Verify that string columns are properly dictionary encoded final ParquetMetadata metadata = - new ParquetTableLocationKey(dest, 0, null, ParquetInstructions.EMPTY).getMetadata(); + new ParquetTableLocationKey(dest.toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); final String firstColumnMetadata = metadata.getBlocks().get(0).getColumns().get(0).toString(); assertTrue(firstColumnMetadata.contains("shortStringColumn") && firstColumnMetadata.contains("RLE_DICTIONARY")); final String secondColumnMetadata = metadata.getBlocks().get(0).getColumns().get(1).toString(); @@ -2515,7 +2573,8 @@ private static ColumnChunkMetaData overflowingStringsTestHelper(final Collection writeTable(stringTable, dest, writeInstructions); checkSingleTable(stringTable, dest); - ParquetMetadata metadata = new ParquetTableLocationKey(dest, 0, null, ParquetInstructions.EMPTY).getMetadata(); + ParquetMetadata metadata = + new ParquetTableLocationKey(dest.toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); return metadata.getBlocks().get(0).getColumns().get(0); } @@ -2539,7 +2598,7 @@ public void overflowingCodecsTest() { checkSingleTable(table, dest); final ParquetMetadata metadata = - new ParquetTableLocationKey(dest, 0, null, ParquetInstructions.EMPTY).getMetadata(); + new ParquetTableLocationKey(dest.toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); final String metadataStr = metadata.getFileMetaData().getKeyValueMetaData().get("deephaven"); assertTrue( metadataStr.contains("VariableWidthByteArrayColumn") && metadataStr.contains("SimpleByteArrayCodec")); @@ -2600,7 +2659,7 @@ public void readWriteDateTimeTest() { // Verify that the types are correct in the schema final ParquetMetadata metadata = - new ParquetTableLocationKey(dest, 0, null, ParquetInstructions.EMPTY).getMetadata(); + new ParquetTableLocationKey(dest.toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); final ColumnChunkMetaData dateColMetadata = metadata.getBlocks().get(0).getColumns().get(0); assertTrue(dateColMetadata.toString().contains("someDateColumn")); assertEquals(PrimitiveType.PrimitiveTypeName.INT32, dateColMetadata.getPrimitiveType().getPrimitiveTypeName()); @@ -3061,7 +3120,7 @@ public void readSingleColumn() { private void assertTableStatistics(Table inputTable, File dest) { // Verify that the columns have the correct statistics. final ParquetMetadata metadata = - new ParquetTableLocationKey(dest, 0, null, ParquetInstructions.EMPTY).getMetadata(); + new ParquetTableLocationKey(dest.toURI(), 0, null, ParquetInstructions.EMPTY).getMetadata(); final String[] colNames = inputTable.getDefinition().getColumnNamesArray(); for (int colIdx = 0; colIdx < inputTable.numColumns(); ++colIdx) { diff --git a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/TestParquetTools.java b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/TestParquetTools.java index db959e8483e..644fda8bbe7 100644 --- a/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/TestParquetTools.java +++ b/extensions/parquet/table/src/test/java/io/deephaven/parquet/table/TestParquetTools.java @@ -353,7 +353,7 @@ public void testPartitionedRead() { final TableDefinition partitionedDefinition = TableDefinition.of(allColumns); final Table result = ParquetTools.readPartitionedTableInferSchema( - new ParquetKeyValuePartitionedLayout(testRootFile, 2, ParquetInstructions.EMPTY), + new ParquetKeyValuePartitionedLayout(testRootFile.toURI(), 2, ParquetInstructions.EMPTY), ParquetInstructions.EMPTY); TestCase.assertEquals(partitionedDefinition, result.getDefinition()); final Table expected = TableTools.merge( diff --git a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3AsyncClientFactory.java b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3AsyncClientFactory.java new file mode 100644 index 00000000000..2b2071fde08 --- /dev/null +++ b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3AsyncClientFactory.java @@ -0,0 +1,151 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.extensions.s3; + +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import org.jetbrains.annotations.NotNull; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.retry.RetryMode; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.utils.ThreadFactoryBuilder; + +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +import static io.deephaven.util.thread.ThreadHelpers.getOrComputeThreadCountProperty; + +class S3AsyncClientFactory { + + private static final int NUM_FUTURE_COMPLETION_THREADS = + getOrComputeThreadCountProperty("S3.numFutureCompletionThreads", -1); + private static final int NUM_SCHEDULED_EXECUTOR_THREADS = + getOrComputeThreadCountProperty("S3.numScheduledExecutorThreads", 5); + + private static final Logger log = LoggerFactory.getLogger(S3AsyncClientFactory.class); + private static final Map httpClientCache = new ConcurrentHashMap<>(); + + private static volatile Executor futureCompletionExecutor; + private static volatile ScheduledExecutorService scheduledExecutor; + + static S3AsyncClient getAsyncClient(@NotNull final S3Instructions instructions) { + final S3AsyncClientBuilder builder = S3AsyncClient.builder() + .asyncConfiguration( + b -> b.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, + ensureAsyncFutureCompletionExecutor())) + .httpClient(getOrBuildHttpClient(instructions)) + .overrideConfiguration(ClientOverrideConfiguration.builder() + // If we find that the STANDARD retry policy does not work well in all situations, we might + // try experimenting with ADAPTIVE retry policy, potentially with fast fail. + // .retryPolicy(RetryPolicy.builder(RetryMode.ADAPTIVE).fastFailRateLimiting(true).build()) + .retryPolicy(RetryMode.STANDARD) + .apiCallAttemptTimeout(instructions.readTimeout().dividedBy(3)) + .apiCallTimeout(instructions.readTimeout()) + // Adding a metrics publisher may be useful for debugging, but it's very verbose. + // .addMetricPublisher(LoggingMetricPublisher.create(Level.INFO, Format.PRETTY)) + .scheduledExecutorService(ensureScheduledExecutor()) + .build()) + .region(Region.of(instructions.regionName())) + .credentialsProvider(instructions.awsV2CredentialsProvider()); + instructions.endpointOverride().ifPresent(builder::endpointOverride); + final S3AsyncClient ret = builder.build(); + if (log.isDebugEnabled()) { + log.debug().append("Building S3AsyncClient with instructions: ").append(instructions).endl(); + } + return ret; + } + + private static class HttpClientConfig { + private final int maxConcurrentRequests; + private final Duration connectionTimeout; + + HttpClientConfig(final int maxConcurrentRequests, final Duration connectionTimeout) { + this.maxConcurrentRequests = maxConcurrentRequests; + this.connectionTimeout = connectionTimeout; + } + + int maxConcurrentRequests() { + return maxConcurrentRequests; + } + + Duration connectionTimeout() { + return connectionTimeout; + } + + @Override + public int hashCode() { + int result = maxConcurrentRequests; + result = 31 * result + connectionTimeout.hashCode(); + return result; + } + + @Override + public boolean equals(final Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + final HttpClientConfig that = (HttpClientConfig) other; + return maxConcurrentRequests == that.maxConcurrentRequests + && connectionTimeout.equals(that.connectionTimeout); + } + } + + private static SdkAsyncHttpClient getOrBuildHttpClient(@NotNull final S3Instructions instructions) { + final HttpClientConfig config = new HttpClientConfig(instructions.maxConcurrentRequests(), + instructions.connectionTimeout()); + return httpClientCache.computeIfAbsent(config, key -> AwsCrtAsyncHttpClient.builder() + .maxConcurrency(config.maxConcurrentRequests()) + .connectionTimeout(config.connectionTimeout()) + .build()); + } + + /** + * The following executor will be used to complete the futures returned by the async client. This is a shared + * executor across all clients with fixed number of threads. This pattern is inspired by the default executor used + * by the SDK + * ({@code software.amazon.awssdk.core.client.builder.SdkDefaultClientBuilder#resolveAsyncFutureCompletionExecutor}) + * + */ + private static Executor ensureAsyncFutureCompletionExecutor() { + if (futureCompletionExecutor == null) { + synchronized (S3AsyncClientFactory.class) { + if (futureCompletionExecutor == null) { + futureCompletionExecutor = Executors.newFixedThreadPool(NUM_FUTURE_COMPLETION_THREADS, + new ThreadFactoryBuilder().threadNamePrefix("s3-async-future-completion").build()); + } + } + } + return futureCompletionExecutor; + } + + /** + * The following executor will be used to schedule tasks for the async client, such as timeouts and retries. This is + * a shared executor across all clients with fixed number of threads. This pattern is inspired by the default + * executor used by the SDK + * ({@code software.amazon.awssdk.core.client.builder.SdkDefaultClientBuilder#resolveScheduledExecutorService}). + */ + private static ScheduledExecutorService ensureScheduledExecutor() { + if (scheduledExecutor == null) { + synchronized (S3AsyncClientFactory.class) { + if (scheduledExecutor == null) { + scheduledExecutor = Executors.newScheduledThreadPool(NUM_SCHEDULED_EXECUTOR_THREADS, + new ThreadFactoryBuilder().threadNamePrefix("s3-scheduled-executor").build()); + } + } + } + return scheduledExecutor; + } +} diff --git a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableByteChannel.java b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableByteChannel.java index 3579961b3a3..6f0d6ffa057 100644 --- a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableByteChannel.java +++ b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableByteChannel.java @@ -41,8 +41,12 @@ final class S3SeekableByteChannel implements SeekableByteChannel, CachedChannelP private long size; S3SeekableByteChannel(final S3Uri uri) { + this(uri, UNINITIALIZED_SIZE); + } + + S3SeekableByteChannel(final S3Uri uri, final long size) { this.uri = Objects.requireNonNull(uri); - this.size = UNINITIALIZED_SIZE; + this.size = size; this.position = INIT_POSITION; } diff --git a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java index 5d773ea484b..414ae9fd715 100644 --- a/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java +++ b/extensions/s3/src/main/java/io/deephaven/extensions/s3/S3SeekableChannelProvider.java @@ -5,29 +5,29 @@ import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.verify.Assert; +import io.deephaven.base.verify.Require; +import io.deephaven.hash.KeyedObjectHashMap; +import io.deephaven.hash.KeyedObjectKey; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.util.channel.Channels; import io.deephaven.util.channel.SeekableChannelContext; import io.deephaven.util.channel.SeekableChannelsProvider; import org.jetbrains.annotations.NotNull; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.retry.RetryMode; -import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; import software.amazon.awssdk.services.s3.S3Uri; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import java.io.IOException; import java.io.InputStream; +import java.lang.ref.SoftReference; import java.net.URI; import java.net.URISyntaxException; import java.nio.channels.SeekableByteChannel; import java.nio.file.Path; import java.util.Iterator; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Spliterator; import java.util.Spliterators; @@ -35,6 +35,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -62,36 +63,17 @@ final class S3SeekableChannelProvider implements SeekableChannelsProvider { private final S3AsyncClient s3AsyncClient; private final S3Instructions s3Instructions; + @SuppressWarnings("rawtypes") + private static final AtomicReferenceFieldUpdater FILE_SIZE_CACHE_REF_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(S3SeekableChannelProvider.class, SoftReference.class, + "fileSizeCacheRef"); + + private volatile SoftReference> fileSizeCacheRef; + S3SeekableChannelProvider(@NotNull final S3Instructions s3Instructions) { - // TODO(deephaven-core#5062): Add support for async client recovery and auto-close - // TODO(deephaven-core#5063): Add support for caching clients for re-use - this.s3AsyncClient = buildClient(s3Instructions); + this.s3AsyncClient = S3AsyncClientFactory.getAsyncClient(s3Instructions); this.s3Instructions = s3Instructions; - } - - private static S3AsyncClient buildClient(@NotNull S3Instructions s3Instructions) { - final S3AsyncClientBuilder builder = S3AsyncClient.builder() - .httpClient(AwsCrtAsyncHttpClient.builder() - .maxConcurrency(s3Instructions.maxConcurrentRequests()) - .connectionTimeout(s3Instructions.connectionTimeout()) - .build()) - .overrideConfiguration(ClientOverrideConfiguration.builder() - // If we find that the STANDARD retry policy does not work well in all situations, we might - // try experimenting with ADAPTIVE retry policy, potentially with fast fail. - // .retryPolicy(RetryPolicy.builder(RetryMode.ADAPTIVE).fastFailRateLimiting(true).build()) - .retryPolicy(RetryMode.STANDARD) - .apiCallAttemptTimeout(s3Instructions.readTimeout().dividedBy(3)) - .apiCallTimeout(s3Instructions.readTimeout()) - // Adding a metrics publisher may be useful for debugging, but it's very verbose. - // .addMetricPublisher(LoggingMetricPublisher.create(Level.INFO, Format.PRETTY)) - .build()) - .region(Region.of(s3Instructions.regionName())) - .credentialsProvider(s3Instructions.awsV2CredentialsProvider()); - s3Instructions.endpointOverride().ifPresent(builder::endpointOverride); - if (log.isDebugEnabled()) { - log.debug().append("Building client with instructions: ").append(s3Instructions).endl(); - } - return builder.build(); + this.fileSizeCacheRef = new SoftReference<>(new KeyedObjectHashMap<>(FileSizeInfo.URI_MATCH_KEY)); } @Override @@ -99,11 +81,15 @@ public SeekableByteChannel getReadChannel(@NotNull final SeekableChannelContext @NotNull final URI uri) { final S3Uri s3Uri = s3AsyncClient.utilities().parseUri(uri); // context is unused here, will be set before reading from the channel + final Map fileSizeCache = fileSizeCacheRef.get(); + if (fileSizeCache != null && fileSizeCache.containsKey(uri)) { + return new S3SeekableByteChannel(s3Uri, fileSizeCache.get(uri).size); + } return new S3SeekableByteChannel(s3Uri); } @Override - public InputStream getInputStream(SeekableByteChannel channel) { + public InputStream getInputStream(final SeekableByteChannel channel) { // S3SeekableByteChannel is internally buffered, no need to re-buffer return Channels.newInputStreamNoClose(channel); } @@ -213,14 +199,17 @@ private void fetchNextBatch() throws IOException { if (path.contains(REPEATED_URI_SEPARATOR)) { path = REPEATED_URI_SEPARATOR_PATTERN.matcher(path).replaceAll(URI_SEPARATOR); } + final URI uri; try { - return new URI(S3_URI_SCHEME, directory.getUserInfo(), directory.getHost(), + uri = new URI(S3_URI_SCHEME, directory.getUserInfo(), directory.getHost(), directory.getPort(), path, null, null); } catch (final URISyntaxException e) { throw new UncheckedDeephavenException("Failed to create URI for S3 object with key: " + s3Object.key() + " and bucket " + bucketName + " inside directory " + directory, e); } + updateFileSizeCache(getFileSizeCache(), uri, s3Object.size()); + return uri; }).iterator(); // The following token is null when the last batch is fetched. continuationToken = response.nextContinuationToken(); @@ -230,6 +219,56 @@ private void fetchNextBatch() throws IOException { Spliterator.ORDERED | Spliterator.DISTINCT | Spliterator.NONNULL), false); } + /** + * Get a strong reference to the file size cache, creating it if necessary. + */ + private Map getFileSizeCache() { + SoftReference> cacheRef; + Map cache; + while ((cache = (cacheRef = fileSizeCacheRef).get()) == null) { + if (FILE_SIZE_CACHE_REF_UPDATER.compareAndSet(this, cacheRef, + new SoftReference<>(cache = new KeyedObjectHashMap<>(FileSizeInfo.URI_MATCH_KEY)))) { + return cache; + } + } + return cache; + } + + /** + * Update the given file size cache with the given URI and size. + */ + private static void updateFileSizeCache( + @NotNull final Map fileSizeCache, + @NotNull final URI uri, + final long size) { + fileSizeCache.compute(uri, (key, existingInfo) -> { + if (existingInfo == null) { + return new FileSizeInfo(uri, size); + } else if (existingInfo.size != size) { + throw new IllegalStateException("Existing size " + existingInfo.size + " does not match " + + " the new size " + size + " for key " + key); + } + return existingInfo; + }); + } + + private static final class FileSizeInfo { + private final URI uri; + private final long size; + + FileSizeInfo(@NotNull final URI uri, final long size) { + this.uri = Require.neqNull(uri, "uri"); + this.size = size; + } + + private static final KeyedObjectKey URI_MATCH_KEY = new KeyedObjectKey.Basic<>() { + @Override + public URI getKey(@NotNull final FileSizeInfo value) { + return value.uri; + } + }; + } + @Override public void close() { s3AsyncClient.close(); diff --git a/extensions/s3/src/test/java/io/deephaven/extensions/s3/S3SeekableChannelTestBase.java b/extensions/s3/src/test/java/io/deephaven/extensions/s3/S3SeekableChannelTestBase.java index 74e941966f2..0075568581b 100644 --- a/extensions/s3/src/test/java/io/deephaven/extensions/s3/S3SeekableChannelTestBase.java +++ b/extensions/s3/src/test/java/io/deephaven/extensions/s3/S3SeekableChannelTestBase.java @@ -71,7 +71,7 @@ void readSimpleFiles() final ByteBuffer buffer = ByteBuffer.allocate(1); try ( final SeekableChannelsProvider providerImpl = providerImpl(uri); - final SeekableChannelsProvider provider = new CachedChannelProvider(providerImpl, 32); + final SeekableChannelsProvider provider = CachedChannelProvider.create(providerImpl, 32); final SeekableChannelContext context = provider.makeContext(); final SeekableByteChannel readChannel = provider.getReadChannel(context, uri)) { assertThat(readChannel.read(buffer)).isEqualTo(-1); @@ -81,7 +81,7 @@ void readSimpleFiles() final URI uri = uri("hello/world.txt"); try ( final SeekableChannelsProvider providerImpl = providerImpl(uri); - final SeekableChannelsProvider provider = new CachedChannelProvider(providerImpl, 32); + final SeekableChannelsProvider provider = CachedChannelProvider.create(providerImpl, 32); final SeekableChannelContext context = provider.makeContext(); final SeekableByteChannel readChannel = provider.getReadChannel(context, uri)) { final ByteBuffer bytes = readAll(readChannel, 32); @@ -103,7 +103,7 @@ public int read() { final ByteBuffer buffer = ByteBuffer.allocate(1); try ( final SeekableChannelsProvider providerImpl = providerImpl(uri); - final SeekableChannelsProvider provider = new CachedChannelProvider(providerImpl, 32); + final SeekableChannelsProvider provider = CachedChannelProvider.create(providerImpl, 32); final SeekableChannelContext context = provider.makeContext(); final SeekableByteChannel readChannel = provider.getReadChannel(context, uri)) { for (long p = 0; p < numBytes; ++p) {