diff --git a/engine/function/src/templates/Numeric.ftl b/engine/function/src/templates/Numeric.ftl index 6270b603bea..d9d5404d3ca 100644 --- a/engine/function/src/templates/Numeric.ftl +++ b/engine/function/src/templates/Numeric.ftl @@ -365,6 +365,9 @@ public class Numeric { try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + if (isNaN(c)) { + return Double.NaN; + } if (!isNull(c)) { sum += c; count++; @@ -416,6 +419,12 @@ public class Numeric { try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + if (isNaN(c)) { + return Double.NaN; + } + if (isInf(c)) { + return Double.POSITIVE_INFINITY; + } if (!isNull(c)) { sum += Math.abs(c); count++; @@ -472,11 +481,13 @@ public class Numeric { double sum = 0; double sum2 = 0; - double count = 0; - + long count = 0; try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + if (isNaN(c) || isInf(c)) { + return Double.NaN; + } if (!isNull(c)) { sum += (double)c; sum2 += (double)c * (double)c; @@ -485,19 +496,19 @@ public class Numeric { } } - // Return NaN if poisoned or too few values to compute sample variance. - if (count <= 1 || Double.isNaN(sum) || Double.isNaN(sum2)) { + // Return NaN if overflow or too few values to compute variance. + if (count <= 1 || Double.isInfinite(sum) || Double.isInfinite(sum2)) { return Double.NaN; } // Perform the calculation in a way that minimizes the impact of floating point error. final double eps = Math.ulp(sum2); - final double vs2bar = sum * (sum / count); + final double vs2bar = sum * (sum / (double)count); final double delta = sum2 - vs2bar; final double rel_eps = delta / eps; // Return zero when the sample variance is leq the floating point error. - return Math.abs(rel_eps) > 1.0 ? delta / (count - 1) : 0.0; + return Math.abs(rel_eps) > 1.0 ? delta / ((double)count - 1) : 0.0; } <#list primitiveTypes as pt2> @@ -590,7 +601,12 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); final ${pt2.primitive} w = wi.${pt2.iteratorNext}(); - + if (isNaN(c) || isInf(c)) { + return Double.NaN; + } + if (isNaN(w) || isInf(w)) { + return Double.NaN; + } if (!isNull(c) && !isNull(w)) { sum += w * c; sum2 += w * c * c; @@ -600,8 +616,8 @@ public class Numeric { } } - // Return NaN if poisoned or too few values to compute sample variance. - if (count <= 1 || Double.isNaN(sum) || Double.isNaN(sum2) || Double.isNaN(count) || Double.isNaN(count2)) { + // Return NaN if overflow or too few values to compute variance. + if (count <= 1 || Double.isInfinite(sum) || Double.isInfinite(sum2)) { return Double.NaN; } @@ -1333,6 +1349,12 @@ public class Numeric { while (v0i.hasNext()) { final ${pt.primitive} v0 = v0i.${pt.iteratorNext}(); final ${pt2.primitive} v1 = v1i.${pt2.iteratorNext}(); + if (isNaN(v0) || isInf(v0)) { + return Double.NaN; + } + if (isNaN(v1) || isInf(v1)) { + return Double.NaN; + } if (!isNull(v0) && !isNull(v1)) { sum0 += v0; @@ -1421,6 +1443,12 @@ public class Numeric { while (v0i.hasNext()) { final ${pt.primitive} v0 = v0i.${pt.iteratorNext}(); final ${pt2.primitive} v1 = v1i.${pt2.iteratorNext}(); + if (isNaN(v0) || isInf(v0)) { + return Double.NaN; + } + if (isNaN(v1) || isInf(v1)) { + return Double.NaN; + } if (!isNull(v0) && !isNull(v1)) { sum0 += v0; @@ -1460,6 +1488,11 @@ public class Numeric { try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + <#if pt.valueType.isFloat > + if (isNaN(c)) { + return ${pt.boxed}.NaN; + } + if (!isNull(c)) { sum += c; } @@ -1496,10 +1529,33 @@ public class Numeric { ${pt.primitive} prod = 1; int count = 0; + <#if pt.valueType.isFloat > + long zeroCount = 0; + long infCount = 0; + try ( final ${pt.vectorIterator} vi = values.iterator() ) { while ( vi.hasNext() ) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); + <#if pt.valueType.isFloat > + if (isNaN(c)) { + return ${pt.boxed}.NaN; + } else if (Double.isInfinite(c)) { + if (zeroCount > 0) { + return ${pt.boxed}.NaN; + } + infCount++; + } else if (c == 0) { + if (infCount > 0) { + return ${pt.boxed}.NaN; + } + zeroCount++; + } + <#else> + if (c == 0) { + return 0; + } + if (!isNull(c)) { count++; prod *= c; @@ -1511,7 +1567,11 @@ public class Numeric { return ${pt.null}; } + <#if pt.valueType.isFloat > + return zeroCount > 0 ? 0 : (${pt.primitive}) (prod); + <#else> return (${pt.primitive}) (prod); + } /** @@ -1549,24 +1609,7 @@ public class Numeric { return null; } - if (values.length == 0) { - return new ${pt.primitive}[0]; - } - - ${pt.primitive}[] result = new ${pt.primitive}[values.length]; - result[0] = values[0]; - - for (int i = 1; i < values.length; i++) { - if (isNull(result[i - 1])) { - result[i] = values[i]; - } else if (isNull(values[i])) { - result[i] = result[i - 1]; - } else { - result[i] = (${pt.primitive})Math.min(result[i - 1], values[i]); - } - } - - return result; + return cummin(new ${pt.vectorDirect}(values)); } /** @@ -1630,24 +1673,7 @@ public class Numeric { return null; } - if (values.length == 0) { - return new ${pt.primitive}[0]; - } - - ${pt.primitive}[] result = new ${pt.primitive}[values.length]; - result[0] = values[0]; - - for (int i = 1; i < values.length; i++) { - if (isNull(result[i - 1])) { - result[i] = values[i]; - } else if (isNull(values[i])) { - result[i] = result[i - 1]; - } else { - result[i] = (${pt.primitive})Math.max(result[i - 1], values[i]); - } - } - - return result; + return cummax(new ${pt.vectorDirect}(values)); } /** @@ -1711,24 +1737,7 @@ public class Numeric { return null; } - if (values.length == 0) { - return new ${pt.primitive}[0]; - } - - ${pt.primitive}[] result = new ${pt.primitive}[values.length]; - result[0] = values[0]; - - for (int i = 1; i < values.length; i++) { - if (isNull(result[i - 1])) { - result[i] = values[i]; - } else if (isNull(values[i])) { - result[i] = result[i - 1]; - } else { - result[i] = (${pt.primitive}) (result[i - 1] + values[i]); - } - } - - return result; + return cumsum(new ${pt.vectorDirect}(values)); } /** @@ -1792,24 +1801,7 @@ public class Numeric { return null; } - if (values.length == 0) { - return new ${pt.primitive}[0]; - } - - ${pt.primitive}[] result = new ${pt.primitive}[values.length]; - result[0] = values[0]; - - for (int i = 1; i < values.length; i++) { - if (isNull(result[i - 1])) { - result[i] = values[i]; - } else if (isNull(values[i])) { - result[i] = result[i - 1]; - } else { - result[i] = (${pt.primitive}) (result[i - 1] * values[i]); - } - } - - return result; + return cumprod(new ${pt.vectorDirect}(values)); } /** @@ -2322,7 +2314,13 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); final ${pt2.primitive} w = wi.${pt2.iteratorNext}(); - + if (isNaN(c)) { + return Double.NaN; + } + if (isNaN(w)) { + return Double.NaN; + } + if (!isNull(c) && !isNull(w)) { vsum += c * w; } @@ -2405,7 +2403,12 @@ public class Numeric { while (vi.hasNext()) { final ${pt.primitive} c = vi.${pt.iteratorNext}(); final ${pt2.primitive} w = wi.${pt2.iteratorNext}(); - + if (isNaN(c)) { + return Double.NaN; + } + if (isNaN(w)) { + return Double.NaN; + } if (!isNull(c) && !isNull(w)) { vsum += c * w; wsum += w; diff --git a/engine/function/src/templates/TestNumeric.ftl b/engine/function/src/templates/TestNumeric.ftl index 24226383f07..e8dd31f5fe8 100644 --- a/engine/function/src/templates/TestNumeric.ftl +++ b/engine/function/src/templates/TestNumeric.ftl @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Deephaven Data Labs and Patent Pending + * Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending */ package io.deephaven.function; @@ -8,9 +8,6 @@ import io.deephaven.base.testing.BaseArrayTestCase; import io.deephaven.vector.*; import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import static io.deephaven.util.QueryConstants.*; import static io.deephaven.function.Basic.count; @@ -367,6 +364,118 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(tstat(v), tstat((${pt.primitive})0, (${pt.primitive})40, ${pt.null}, (${pt.primitive})50, (${pt.primitive})60, (${pt.primitive}) -1, (${pt.primitive})0)); } +<#if pt.valueType.isFloat > + public void test${pt.boxed}NaNAndInfHandling() { + double result; + + final ${pt.primitive}[] normal = new ${pt.primitive}[]{1, 2, 3, 4, 5, 6}; + + final ${pt.primitive}[] normalWithNaN = new ${pt.primitive}[]{1, 2, 3, ${pt.boxed}.NaN, 4, 5}; + assertTrue(Double.isNaN(avg(normalWithNaN))); + assertTrue(Double.isNaN(absAvg(normalWithNaN))); + assertTrue(Double.isNaN(var(normalWithNaN))); + assertTrue(Double.isNaN(std(normalWithNaN))); + assertTrue(Double.isNaN(ste(normalWithNaN))); + assertTrue(Double.isNaN(tstat(normalWithNaN))); + + assertTrue(Double.isNaN(cov(normalWithNaN, normal))); + assertTrue(Double.isNaN(cor(normalWithNaN, normal))); + assertTrue(Double.isNaN(wavg(normalWithNaN, normal))); + assertTrue(Double.isNaN(wvar(normalWithNaN, normal))); + assertTrue(Double.isNaN(wstd(normalWithNaN, normal))); + assertTrue(Double.isNaN(wste(normalWithNaN, normal))); + assertTrue(Double.isNaN(wtstat(normalWithNaN, normal))); + + assertTrue(Double.isNaN(cov(normal, normalWithNaN))); + assertTrue(Double.isNaN(cor(normal, normalWithNaN))); + assertTrue(Double.isNaN(wavg(normal, normalWithNaN))); + assertTrue(Double.isNaN(wvar(normal, normalWithNaN))); + assertTrue(Double.isNaN(wstd(normal, normalWithNaN))); + assertTrue(Double.isNaN(wste(normal, normalWithNaN))); + assertTrue(Double.isNaN(wtstat(normal, normalWithNaN))); + + final ${pt.primitive}[] normalWithInf = new ${pt.primitive}[]{1, 2, 3, ${pt.boxed}.POSITIVE_INFINITY, 4, 5}; + result = avg(normalWithInf); + assertTrue(Double.isInfinite(result) && result > 0); // positive infinity + result = absAvg(normalWithInf); + assertTrue(Double.isInfinite(result) && result > 0); // positive infinity + + assertTrue(Double.isNaN(var(normalWithInf))); + assertTrue(Double.isNaN(std(normalWithInf))); + assertTrue(Double.isNaN(ste(normalWithInf))); + assertTrue(Double.isNaN(tstat(normalWithInf))); + + assertTrue(Double.isNaN(cov(normalWithInf, normal))); + assertTrue(Double.isNaN(cor(normalWithInf, normal))); + result = wavg(normalWithInf, normal); + assertTrue(Double.isInfinite(result) && result > 0); // positive infinity + assertTrue(Double.isNaN(wvar(normalWithInf, normal))); + assertTrue(Double.isNaN(wstd(normalWithInf, normal))); + assertTrue(Double.isNaN(wste(normalWithInf, normal))); + assertTrue(Double.isNaN(wtstat(normalWithInf, normal))); + + assertTrue(Double.isNaN(cov(normal, normalWithInf))); + assertTrue(Double.isNaN(cor(normal, normalWithInf))); + assertTrue(Double.isNaN(wavg(normal, normalWithInf))); // is NaN because of inf/inf division + assertTrue(Double.isNaN(wvar(normal, normalWithInf))); + assertTrue(Double.isNaN(wstd(normal, normalWithInf))); + assertTrue(Double.isNaN(wste(normal, normalWithInf))); + assertTrue(Double.isNaN(wtstat(normal, normalWithInf))); + + final ${pt.primitive}[] normalWithNegInf = new ${pt.primitive}[]{1, 2, 3, ${pt.boxed}.NEGATIVE_INFINITY, 4, 5}; + result = avg(normalWithNegInf); + assertTrue(Double.isInfinite(result) && result < 0); // negative infinity + result = absAvg(normalWithNegInf); + assertTrue(Double.isInfinite(result) && result > 0); // positive infinity + + assertTrue(Double.isNaN(var(normalWithNegInf))); + assertTrue(Double.isNaN(std(normalWithNegInf))); + assertTrue(Double.isNaN(ste(normalWithNegInf))); + assertTrue(Double.isNaN(tstat(normalWithNegInf))); + + assertTrue(Double.isNaN(cov(normalWithNegInf, normal))); + assertTrue(Double.isNaN(cor(normalWithNegInf, normal))); + result = wavg(normalWithNegInf, normal); + assertTrue(Double.isInfinite(result) && result < 0); // negative infinity + assertTrue(Double.isNaN(wvar(normalWithNegInf, normal))); + assertTrue(Double.isNaN(wstd(normalWithNegInf, normal))); + assertTrue(Double.isNaN(wste(normalWithNegInf, normal))); + assertTrue(Double.isNaN(wtstat(normalWithNegInf, normal))); + + assertTrue(Double.isNaN(cov(normal, normalWithNegInf))); + assertTrue(Double.isNaN(cor(normal, normalWithNegInf))); + assertTrue(Double.isNaN(wavg(normal, normalWithNegInf))); // is NaN because of -inf/-inf division + assertTrue(Double.isNaN(wvar(normal, normalWithNegInf))); + assertTrue(Double.isNaN(wstd(normal, normalWithNegInf))); + assertTrue(Double.isNaN(wste(normal, normalWithNegInf))); + assertTrue(Double.isNaN(wtstat(normal, normalWithNegInf))); + + <#if pt.primitive == "double" > + // testing normal value overflow. NOTE: this is testing for doubles only, since overflowing a double using + // smaller types is quite difficult + final double LARGE_VALUE = Math.nextDown(Double.MAX_VALUE); + + final double[] overflow = new double[]{1, LARGE_VALUE, LARGE_VALUE}; + assertTrue(Double.isInfinite(avg(overflow))); + + assertTrue(Double.isNaN(var(overflow))); + assertTrue(Double.isNaN(std(overflow))); + assertTrue(Double.isNaN(ste(overflow))); + assertTrue(Double.isNaN(tstat(overflow))); + + final double[] negOverflow = new double[]{1, LARGE_VALUE, -LARGE_VALUE}; + assertTrue(Double.isNaN(var(negOverflow))); + assertTrue(Double.isNaN(std(negOverflow))); + assertTrue(Double.isNaN(ste(negOverflow))); + assertTrue(Double.isNaN(tstat(negOverflow))); + + final double[] negAdditionOverflow = new double[]{1, -LARGE_VALUE, -LARGE_VALUE}; + result = avg(negAdditionOverflow); + assertTrue(Double.isInfinite(result) && result < 0); // negative infinity + + } + + <#list primitiveTypes as pt2> <#if pt2.valueType.isNumber > @@ -512,6 +621,39 @@ public class TestNumeric extends BaseArrayTestCase { assertEquals(${pt.null}, product((${pt.vector}) null)); } +<#if pt.valueType.isFloat > + public void test${pt.boxed}ProductOverflowAndNaN() { + final ${pt.primitive} LARGE_VALUE = Math.nextDown(${pt.boxed}.MAX_VALUE); + + final ${pt.primitive}[] overflow = new ${pt.primitive}[]{1, LARGE_VALUE, LARGE_VALUE}; + final ${pt.primitive} overflowProduct = product(overflow); + assertTrue(${pt.boxed}.isInfinite(overflowProduct) && overflowProduct > 0); + + final ${pt.primitive}[] negOverflow = new ${pt.primitive}[]{1, LARGE_VALUE, -LARGE_VALUE}; + final ${pt.primitive} negOverflowProduct = product(negOverflow); + assertTrue(${pt.boxed}.isInfinite(negOverflowProduct) && negOverflowProduct < 0); + + final ${pt.primitive}[] overflowWithZero = new ${pt.primitive}[]{1, LARGE_VALUE, LARGE_VALUE, 0}; + assertTrue(Math.abs(product(overflowWithZero)) == 0.0); + + final ${pt.primitive}[] normalWithNaN = new ${pt.primitive}[]{1, 2, 3, ${pt.boxed}.NaN, 4, 5}; + assertTrue(${pt.boxed}.isNaN(product(normalWithNaN))); + + final ${pt.primitive}[] posInfAndZero = new ${pt.primitive}[]{1, ${pt.boxed}.POSITIVE_INFINITY, 0}; + assertTrue(${pt.boxed}.isNaN(product(posInfAndZero))); + + final ${pt.primitive}[] negInfAndZero = new ${pt.primitive}[]{1, ${pt.boxed}.NEGATIVE_INFINITY, 0}; + assertTrue(${pt.boxed}.isNaN(product(negInfAndZero))); + + final ${pt.primitive}[] zeroAndPosInf = new ${pt.primitive}[]{1, 0, ${pt.boxed}.POSITIVE_INFINITY}; + assertTrue(${pt.boxed}.isNaN(product(zeroAndPosInf))); + + final ${pt.primitive}[] zeroAndNegInf = new ${pt.primitive}[]{1, 0, ${pt.boxed}.NEGATIVE_INFINITY}; + assertTrue(${pt.boxed}.isNaN(product(zeroAndNegInf))); + + } + + // public void test${pt.boxed}ProdObjectVector() { // assertEquals(new ${pt.primitive}[]{-30, 120}, product(new ObjectVectorDirect<>(new ${pt.primitive}[][]{{5, 4}, {-3, 5}, {2, 6}}))); // assertEquals(new ${pt.primitive}[]{-30, ${pt.null}}, product(new ObjectVectorDirect<>(new ${pt.primitive}[][]{{5, ${pt.null}}, {-3, 5}, {2, 6}}))); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java index 711ba70ab0a..19b4ef31f6f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/DoubleChunkedVarOperator.java @@ -97,7 +97,12 @@ private boolean addChunk(DoubleChunk values, long destination, if (forceNanResult || nonNullCount <= 1) { resultColumn.set(destination, Double.NaN); } else { - final double variance = (newSum2 - newSum * newSum / nonNullCount) / (nonNullCount - 1); + // If the sum or sumSquared has reached +/-Infinity, we are stuck with NaN forever. + if (Double.isInfinite(newSum) || Double.isInfinite(newSum2)) { + resultColumn.set(destination, Double.NaN); + return true; + } + final double variance = computeVariance(nonNullCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } return true; @@ -109,6 +114,17 @@ private boolean addChunk(DoubleChunk values, long destination, } } + private static double computeVariance(long nonNullCount, double newSum, double newSum2) { + // Perform the calculation in a way that minimizes the impact of FP error. + final double eps = Math.ulp(newSum2); + final double vs2bar = newSum * (newSum / nonNullCount); + final double delta = newSum2 - vs2bar; + final double rel_eps = delta / eps; + + // Return zero when the variance is leq the FP error or when variance becomes negative + final double variance = Math.abs(rel_eps) > 1.0 ? delta / (nonNullCount - 1) : 0.0; + return Math.max(variance, 0.0); + } private boolean removeChunk(DoubleChunk values, long destination, int chunkStart, int chunkSize) { final MutableDouble sum2 = new MutableDouble(); @@ -150,7 +166,15 @@ private boolean removeChunk(DoubleChunk values, long destinati resultColumn.set(destination, Double.NaN); return true; } - final double variance = (newSum2 - newSum * newSum / totalNormalCount) / (totalNormalCount - 1); + + // If the sum has reach +/-Infinity, we are stuck with NaN forever. + if (Double.isInfinite(newSum) || Double.isInfinite(newSum2)) { + resultColumn.set(destination, Double.NaN); + return true; + } + + // Perform the calculation in a way that minimizes the impact of FP error. + final double variance = computeVariance(totalNormalCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java index 5e525b07e2a..641e5e95ec7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FloatChunkedVarOperator.java @@ -92,7 +92,12 @@ private boolean addChunk(FloatChunk values, long destination, if (forceNanResult || nonNullCount <= 1) { resultColumn.set(destination, Double.NaN); } else { - final double variance = (newSum2 - newSum * newSum / nonNullCount) / (nonNullCount - 1); + // If the sum or sumSquared has reached +/-Infinity, we are stuck with NaN forever. + if (Double.isInfinite(newSum) || Double.isInfinite(newSum2)) { + resultColumn.set(destination, Double.NaN); + return true; + } + final double variance = computeVariance(nonNullCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); } return true; @@ -104,6 +109,17 @@ private boolean addChunk(FloatChunk values, long destination, } } + private static double computeVariance(long nonNullCount, double newSum, double newSum2) { + // Perform the calculation in a way that minimizes the impact of FP error. + final double eps = Math.ulp(newSum2); + final double vs2bar = newSum * (newSum / nonNullCount); + final double delta = newSum2 - vs2bar; + final double rel_eps = delta / eps; + + // Return zero when the variance is leq the FP error or when variance becomes negative + final double variance = Math.abs(rel_eps) > 1.0 ? delta / (nonNullCount - 1) : 0.0; + return Math.max(variance, 0.0); + } private boolean removeChunk(FloatChunk values, long destination, int chunkStart, int chunkSize) { final MutableDouble sum2 = new MutableDouble(); @@ -145,7 +161,15 @@ private boolean removeChunk(FloatChunk values, long destinatio resultColumn.set(destination, Double.NaN); return true; } - final double variance = (newSum2 - newSum * newSum / totalNormalCount) / (totalNormalCount - 1); + + // If the sum has reach +/-Infinity, we are stuck with NaN forever. + if (Double.isInfinite(newSum) || Double.isInfinite(newSum2)) { + resultColumn.set(destination, Double.NaN); + return true; + } + + // Perform the calculation in a way that minimizes the impact of FP error. + final double variance = computeVariance(totalNormalCount, newSum, newSum2); resultColumn.set(destination, std ? Math.sqrt(variance) : variance); return true; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java index 91d128411cb..dc354836853 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ByteRollingAvgOperator.java @@ -88,7 +88,11 @@ public void writeToOutputChunk(int outIdx) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = byteWindowValues.size() - nullCount; - outputValues.set(outIdx, curVal / (double)count); + if (count == 0) { + outputValues.set(outIdx, Double.NaN); + } else { + outputValues.set(outIdx, curVal / (double)count); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java index babdae78e29..6f5c7610c74 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/CharRollingAvgOperator.java @@ -82,7 +82,11 @@ public void writeToOutputChunk(int outIdx) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = charWindowValues.size() - nullCount; - outputValues.set(outIdx, curVal / (double)count); + if (count == 0) { + outputValues.set(outIdx, Double.NaN); + } else { + outputValues.set(outIdx, curVal / (double)count); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java index 3748d4cd90c..3c3e405368d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/DoubleRollingAvgOperator.java @@ -84,7 +84,11 @@ public void writeToOutputChunk(int outIdx) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = aggSum.size() - nullCount; - outputValues.set(outIdx, aggSum.evaluate() / (double)count); + if (count == 0) { + outputValues.set(outIdx, Double.NaN); + } else { + outputValues.set(outIdx, aggSum.evaluate() / (double)count); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java index d8e7431c071..220f3df01e9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/FloatRollingAvgOperator.java @@ -79,7 +79,11 @@ public void writeToOutputChunk(int outIdx) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = aggSum.size() - nullCount; - outputValues.set(outIdx, aggSum.evaluate() / (double)count); + if (count == 0) { + outputValues.set(outIdx, Double.NaN); + } else { + outputValues.set(outIdx, aggSum.evaluate() / (double)count); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java index 57b8ef4ff5e..3fc6e88bf80 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/IntRollingAvgOperator.java @@ -87,7 +87,11 @@ public void writeToOutputChunk(int outIdx) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = intWindowValues.size() - nullCount; - outputValues.set(outIdx, curVal / (double)count); + if (count == 0) { + outputValues.set(outIdx, Double.NaN); + } else { + outputValues.set(outIdx, curVal / (double)count); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java index 7fae62c0f5d..3fcc9ff5006 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/LongRollingAvgOperator.java @@ -87,7 +87,11 @@ public void writeToOutputChunk(int outIdx) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = longWindowValues.size() - nullCount; - outputValues.set(outIdx, curVal / (double)count); + if (count == 0) { + outputValues.set(outIdx, Double.NaN); + } else { + outputValues.set(outIdx, curVal / (double)count); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java index 7c81f008a00..70920192b0c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingavg/ShortRollingAvgOperator.java @@ -87,7 +87,11 @@ public void writeToOutputChunk(int outIdx) { outputValues.set(outIdx, NULL_DOUBLE); } else { final int count = shortWindowValues.size() - nullCount; - outputValues.set(outIdx, curVal / (double)count); + if (count == 0) { + outputValues.set(outIdx, Double.NaN); + } else { + outputValues.set(outIdx, curVal / (double)count); + } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/DoubleRollingProductOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/DoubleRollingProductOperator.java index 23385c171c0..6d0d3cef925 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/DoubleRollingProductOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/DoubleRollingProductOperator.java @@ -1,6 +1,6 @@ /* * --------------------------------------------------------------------------------------------------------------------- - * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingProductOperator and regenerate + * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit FloatRollingProductOperator and regenerate * --------------------------------------------------------------------------------------------------------------------- */ package io.deephaven.engine.table.impl.updateby.rollingproduct; @@ -30,6 +30,8 @@ protected class Context extends BaseDoubleUpdateByOperator.Context { protected AggregatingDoubleRingBuffer buffer; private int zeroCount; + private int nanCount; + private int infCount; protected Context(final int affectedChunkSize, final int influencerChunkSize) { super(affectedChunkSize); @@ -48,6 +50,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { }, true); zeroCount = 0; + nanCount = 0; + infCount = 0; } @Override @@ -76,6 +80,10 @@ public void push(int pos, int count) { buffer.addUnsafe(val); if (val == 0) { zeroCount++; + } else if (Double.isNaN(val)) { + nanCount++; + } else if (Double.isInfinite(val)) { + infCount++; } } } @@ -90,8 +98,12 @@ public void pop(int count) { if (val == NULL_DOUBLE) { nullCount--; + } else if (Double.isNaN(val)) { + --nanCount; } else if (val == 0) { --zeroCount; + } else if (Double.isInfinite(val)) { + --infCount; } } } @@ -101,7 +113,14 @@ public void writeToOutputChunk(int outIdx) { if (buffer.size() == nullCount) { outputValues.set(outIdx, NULL_DOUBLE); } else { - outputValues.set(outIdx, zeroCount > 0 ? 0.0 : buffer.evaluate()); + if (nanCount > 0 || (infCount > 0 && zeroCount > 0)) { + // Output NaN without evaluating the buffer when the buffer is poisoned with NaNs or when we + // have an Inf * 0 case + outputValues.set(outIdx, Double.NaN); + } else { + // When zeros are present, we can skip evaluating the buffer. + outputValues.set(outIdx, zeroCount > 0 ? 0.0 : buffer.evaluate()); + } } } @@ -109,6 +128,8 @@ public void writeToOutputChunk(int outIdx) { public void reset() { super.reset(); zeroCount = 0; + nanCount = 0; + infCount = 0; buffer.clear(); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/FloatRollingProductOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/FloatRollingProductOperator.java index 44e4d737d14..f7868c2c0ae 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/FloatRollingProductOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/FloatRollingProductOperator.java @@ -1,8 +1,3 @@ -/* - * --------------------------------------------------------------------------------------------------------------------- - * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingProductOperator and regenerate - * --------------------------------------------------------------------------------------------------------------------- - */ package io.deephaven.engine.table.impl.updateby.rollingproduct; import io.deephaven.base.ringbuffer.AggregatingDoubleRingBuffer; @@ -30,6 +25,8 @@ protected class Context extends BaseDoubleUpdateByOperator.Context { protected AggregatingDoubleRingBuffer buffer; private int zeroCount; + private int nanCount; + private int infCount; protected Context(final int affectedChunkSize, final int influencerChunkSize) { super(affectedChunkSize); @@ -48,6 +45,8 @@ protected Context(final int affectedChunkSize, final int influencerChunkSize) { }, true); zeroCount = 0; + nanCount = 0; + infCount = 0; } @Override @@ -76,6 +75,10 @@ public void push(int pos, int count) { buffer.addUnsafe(val); if (val == 0) { zeroCount++; + } else if (Double.isNaN(val)) { + nanCount++; + } else if (Double.isInfinite(val)) { + infCount++; } } } @@ -90,8 +93,12 @@ public void pop(int count) { if (val == NULL_DOUBLE) { nullCount--; + } else if (Double.isNaN(val)) { + --nanCount; } else if (val == 0) { --zeroCount; + } else if (Double.isInfinite(val)) { + --infCount; } } } @@ -101,7 +108,14 @@ public void writeToOutputChunk(int outIdx) { if (buffer.size() == nullCount) { outputValues.set(outIdx, NULL_DOUBLE); } else { - outputValues.set(outIdx, zeroCount > 0 ? 0.0 : buffer.evaluate()); + if (nanCount > 0 || (infCount > 0 && zeroCount > 0)) { + // Output NaN without evaluating the buffer when the buffer is poisoned with NaNs or when we + // have an Inf * 0 case + outputValues.set(outIdx, Double.NaN); + } else { + // When zeros are present, we can skip evaluating the buffer. + outputValues.set(outIdx, zeroCount > 0 ? 0.0 : buffer.evaluate()); + } } } @@ -109,6 +123,8 @@ public void writeToOutputChunk(int outIdx) { public void reset() { super.reset(); zeroCount = 0; + nanCount = 0; + infCount = 0; buffer.clear(); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java index 1a5c3b42770..1707f555603 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ByteRollingStdOperator.java @@ -121,7 +121,10 @@ public void writeToOutputChunk(int outIdx) { final double valueSquareSum = valueSquareBuffer.evaluate(); final double valueSum = valueBuffer.evaluate(); - if (Double.isNaN(valueSquareSum) || Double.isNaN(valueSum)) { + if (Double.isNaN(valueSquareSum) + || Double.isNaN(valueSum) + || Double.isInfinite(valueSquareSum) + || Double.isInfinite(valueSum)) { outputValues.set(outIdx, Double.NaN); return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java index b965e373639..4e2893a8120 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/CharRollingStdOperator.java @@ -115,7 +115,10 @@ public void writeToOutputChunk(int outIdx) { final double valueSquareSum = valueSquareBuffer.evaluate(); final double valueSum = valueBuffer.evaluate(); - if (Double.isNaN(valueSquareSum) || Double.isNaN(valueSum)) { + if (Double.isNaN(valueSquareSum) + || Double.isNaN(valueSum) + || Double.isInfinite(valueSquareSum) + || Double.isInfinite(valueSum)) { outputValues.set(outIdx, Double.NaN); return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java index b9320231ba0..3adbc7e04d3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/DoubleRollingStdOperator.java @@ -120,7 +120,10 @@ public void writeToOutputChunk(int outIdx) { final double valueSquareSum = valueSquareBuffer.evaluate(); final double valueSum = valueBuffer.evaluate(); - if (Double.isNaN(valueSquareSum) || Double.isNaN(valueSum)) { + if (Double.isNaN(valueSquareSum) + || Double.isNaN(valueSum) + || Double.isInfinite(valueSquareSum) + || Double.isInfinite(valueSum)) { outputValues.set(outIdx, Double.NaN); return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java index e1c90523f14..082fa4f46e7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/FloatRollingStdOperator.java @@ -120,7 +120,10 @@ public void writeToOutputChunk(int outIdx) { final double valueSquareSum = valueSquareBuffer.evaluate(); final double valueSum = valueBuffer.evaluate(); - if (Double.isNaN(valueSquareSum) || Double.isNaN(valueSum)) { + if (Double.isNaN(valueSquareSum) + || Double.isNaN(valueSum) + || Double.isInfinite(valueSquareSum) + || Double.isInfinite(valueSum)) { outputValues.set(outIdx, Double.NaN); return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java index d35df8d2423..3aa840cbf17 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/IntRollingStdOperator.java @@ -120,7 +120,10 @@ public void writeToOutputChunk(int outIdx) { final double valueSquareSum = valueSquareBuffer.evaluate(); final double valueSum = valueBuffer.evaluate(); - if (Double.isNaN(valueSquareSum) || Double.isNaN(valueSum)) { + if (Double.isNaN(valueSquareSum) + || Double.isNaN(valueSum) + || Double.isInfinite(valueSquareSum) + || Double.isInfinite(valueSum)) { outputValues.set(outIdx, Double.NaN); return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java index 87cb06edca2..793d441e497 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/LongRollingStdOperator.java @@ -120,7 +120,10 @@ public void writeToOutputChunk(int outIdx) { final double valueSquareSum = valueSquareBuffer.evaluate(); final double valueSum = valueBuffer.evaluate(); - if (Double.isNaN(valueSquareSum) || Double.isNaN(valueSum)) { + if (Double.isNaN(valueSquareSum) + || Double.isNaN(valueSum) + || Double.isInfinite(valueSquareSum) + || Double.isInfinite(valueSum)) { outputValues.set(outIdx, Double.NaN); return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java index e4335953489..84ca25415a8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingstd/ShortRollingStdOperator.java @@ -120,7 +120,10 @@ public void writeToOutputChunk(int outIdx) { final double valueSquareSum = valueSquareBuffer.evaluate(); final double valueSum = valueBuffer.evaluate(); - if (Double.isNaN(valueSquareSum) || Double.isNaN(valueSum)) { + if (Double.isNaN(valueSquareSum) + || Double.isNaN(valueSum) + || Double.isInfinite(valueSquareSum) + || Double.isInfinite(valueSum)) { outputValues.set(outIdx, Double.NaN); return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java b/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java index aa1f18a75c1..cdc01bb862a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/TableDiff.java @@ -59,8 +59,8 @@ static Pair diffInternal(Table actualResult, Table expectedResult, } } - final Map actualNameToColumnSource = actualResult.getColumnSourceMap(); - final Map expectedNameToColumnSource = expectedResult.getColumnSourceMap(); + final Map> actualNameToColumnSource = actualResult.getColumnSourceMap(); + final Map> expectedNameToColumnSource = expectedResult.getColumnSourceMap(); final String[] actualColumnNames = actualResult.getDefinition().getColumnNames().toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); final String[] expectedColumnNames = @@ -78,8 +78,8 @@ static Pair diffInternal(Table actualResult, Table expectedResult, final Set columnNamesForDiff = new LinkedHashSet<>(); for (int ci = 0; ci < expectedColumnNames.length; ci++) { final String expectedColumnName = expectedColumnNames[ci]; - final ColumnSource expectedColumnSource = expectedNameToColumnSource.get(expectedColumnName); - final ColumnSource actualColumnSource = actualNameToColumnSource.get(expectedColumnName); + final ColumnSource expectedColumnSource = expectedNameToColumnSource.get(expectedColumnName); + final ColumnSource actualColumnSource = actualNameToColumnSource.get(expectedColumnName); if (actualColumnSource == null) { issues.add("Expected column " + expectedColumnName + " not found"); } else { @@ -114,7 +114,7 @@ static Pair diffInternal(Table actualResult, Table expectedResult, try (final SafeCloseableList safeCloseables = new SafeCloseableList(); final SharedContext expectedSharedContext = SharedContext.makeSharedContext(); final SharedContext actualSharedContext = SharedContext.makeSharedContext(); - final WritableBooleanChunk equalValues = WritableBooleanChunk.makeWritableChunk(chunkSize)) { + final WritableBooleanChunk equalValues = WritableBooleanChunk.makeWritableChunk(chunkSize)) { final ColumnDiffContext[] columnContexts = columnNamesForDiff.stream() .map(name -> safeCloseables.add(new ColumnDiffContext(name, expectedNameToColumnSource.get(name), @@ -231,7 +231,7 @@ private ColumnDiffContext(@NotNull final String name, */ private long diffChunk(@NotNull final RowSequence expectedChunkOk, @NotNull final RowSequence actualChunkOk, - @NotNull final WritableBooleanChunk equalValues, + @NotNull final WritableBooleanChunk equalValues, @NotNull final Set itemsToSkip, @NotNull final List issues, long position) { @@ -267,6 +267,13 @@ private long diffChunk(@NotNull final RowSequence expectedChunkOk, } else if (chunkType == ChunkType.Float) { final float expectedValue = expectedValues.asFloatChunk().get(ii); final float actualValue = actualValues.asFloatChunk().get(ii); + if (Float.isNaN(expectedValue) || Float.isNaN(actualValue)) { + final String actualString = Float.isNaN(actualValue) ? "NaN" : Float.toString(actualValue); + final String expectString = Float.isNaN(expectedValue) ? "NaN" : Float.toString(expectedValue); + issues.add("Column " + name + " different from the expected set, first difference at row " + + position + " encountered " + actualString + " expected " + expectString); + return position; + } if (expectedValue == io.deephaven.util.QueryConstants.NULL_FLOAT || actualValue == io.deephaven.util.QueryConstants.NULL_FLOAT) { final String actualString = actualValue == io.deephaven.util.QueryConstants.NULL_FLOAT ? "null" @@ -297,6 +304,14 @@ private long diffChunk(@NotNull final RowSequence expectedChunkOk, } else if (chunkType == ChunkType.Double) { final double expectedValue = expectedValues.asDoubleChunk().get(ii); final double actualValue = actualValues.asDoubleChunk().get(ii); + if (Double.isNaN(expectedValue) || Double.isNaN(actualValue)) { + final String actualString = Double.isNaN(actualValue) ? "NaN" : Double.toString(actualValue); + final String expectString = + Double.isNaN(expectedValue) ? "NaN" : Double.toString(expectedValue); + issues.add("Column " + name + " different from the expected set, first difference at row " + + position + " encountered " + actualString + " expected " + expectString); + return position; + } if (expectedValue == io.deephaven.util.QueryConstants.NULL_DOUBLE || actualValue == io.deephaven.util.QueryConstants.NULL_DOUBLE) { final String actualString = actualValue == io.deephaven.util.QueryConstants.NULL_DOUBLE ? "null" diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java index 9dd2cb2b4cc..fcb3daec101 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/BaseUpdateByTest.java @@ -1,6 +1,8 @@ package io.deephaven.engine.table.impl.updateby; import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.AbstractColumnSource; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.testutil.ColumnInfo; import io.deephaven.engine.testutil.generator.*; @@ -9,10 +11,7 @@ import org.junit.Rule; import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; +import java.util.*; import static io.deephaven.engine.testutil.TstUtils.getTable; import static io.deephaven.engine.testutil.TstUtils.initColumnInfos; @@ -81,11 +80,11 @@ static CreateResult createTestTable(int tableSize, generators.toArray(new TestDataGenerator[0])); final QueryTable t = getTable(tableSize, random, columnInfos); - - // if (!isRefreshing && includeGroups) { - // final ColumnSource groupingSource = t.getColumnSource("Sym"); - // groupingSource.setGroupingProvider(StaticGroupingProvider.buildFrom(groupingSource, t.getRowSet())); - // } + if (!isRefreshing && includeGroups) { + final AbstractColumnSource groupingSource = (AbstractColumnSource) t.getColumnSource("Sym"); + final Map gtr = groupingSource.getValuesMapping(t.getRowSet()); + groupingSource.setGroupToRange(gtr); + } t.setRefreshing(isRefreshing); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java index e3b24089b7a..d01a1749111 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestRollingProduct.java @@ -4,18 +4,17 @@ import io.deephaven.api.updateby.UpdateByControl; import io.deephaven.api.updateby.UpdateByOperation; import io.deephaven.base.verify.Assert; +import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryScope; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.AbstractColumnSource; import io.deephaven.engine.table.impl.DataAccessHelpers; import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.testutil.ControlledUpdateGraph; -import io.deephaven.engine.testutil.EvalNugget; -import io.deephaven.engine.testutil.GenerateTableUpdates; -import io.deephaven.engine.testutil.TstUtils; -import io.deephaven.engine.testutil.generator.CharGenerator; -import io.deephaven.engine.testutil.generator.SortedInstantGenerator; -import io.deephaven.engine.testutil.generator.TestDataGenerator; +import io.deephaven.engine.testutil.*; +import io.deephaven.engine.testutil.generator.*; import io.deephaven.engine.util.TableDiff; import io.deephaven.test.types.OutOfBandTest; import io.deephaven.time.DateTimeUtils; @@ -28,13 +27,12 @@ import java.math.BigInteger; import java.math.MathContext; import java.time.Duration; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; -import java.util.Random; +import java.util.*; import java.util.function.Function; import static io.deephaven.engine.testutil.GenerateTableUpdates.generateAppends; +import static io.deephaven.engine.testutil.TstUtils.getTable; +import static io.deephaven.engine.testutil.TstUtils.initColumnInfos; import static io.deephaven.engine.testutil.testcase.RefreshingTableTestCase.simulateShiftAwareStep; import static io.deephaven.function.Basic.isNull; @@ -102,6 +100,62 @@ private String[] getCastingFormulas(String[] columns) { .toArray(String[]::new); } + /** + * Create a custom test table where the values are small enough they won't overflow Double.MAX_VALUE when + * multiplied. This will allow the results to be verified with the Numeric#product() function. + */ + @SuppressWarnings({"rawtypes"}) + static CreateResult createSmallTestTable(int tableSize, + boolean includeSym, + boolean includeGroups, + boolean isRefreshing, + int seed, + String[] extraNames, + TestDataGenerator[] extraGenerators) { + if (includeGroups && !includeSym) { + throw new IllegalArgumentException(); + } + + final List colsList = new ArrayList<>(); + final List generators = new ArrayList<>(); + if (includeSym) { + colsList.add("Sym"); + generators.add(new SetGenerator<>("a", "b", "c", "d", null)); + } + + if (extraNames.length > 0) { + colsList.addAll(Arrays.asList(extraNames)); + generators.addAll(Arrays.asList(extraGenerators)); + } + + colsList.addAll(Arrays.asList("byteCol", "shortCol", "intCol", "longCol", "floatCol", "doubleCol", "boolCol", + "bigIntCol", "bigDecimalCol")); + generators.addAll(Arrays.asList(new ByteGenerator((byte) -1, (byte) 5, .1), + new ShortGenerator((short) -1, (short) 5, .1), + new IntGenerator(-1, 5, .1), + new LongGenerator(-1, 5, .1), + new FloatGenerator(-1, 5, .1), + new DoubleGenerator(-1, 5, .1), + new BooleanGenerator(.5, .1), + new BigIntegerGenerator(new BigInteger("-1"), new BigInteger("5"), .1), + new BigDecimalGenerator(new BigInteger("1"), new BigInteger("2"), 5, .1))); + + final Random random = new Random(seed); + final ColumnInfo[] columnInfos = initColumnInfos(colsList.toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY), + generators.toArray(new TestDataGenerator[0])); + final QueryTable t = getTable(tableSize, random, columnInfos); + + if (!isRefreshing && includeGroups) { + final AbstractColumnSource groupingSource = (AbstractColumnSource) t.getColumnSource("Sym"); + final Map gtr = groupingSource.getValuesMapping(t.getRowSet()); + groupingSource.setGroupToRange(gtr); + } + + t.setRefreshing(isRefreshing); + + return new CreateResult(t, columnInfos, random); + } + // region Object Helper functions final Function, BigInteger> prodBigInt = bigIntegerObjectVector -> { @@ -396,7 +450,7 @@ public void testStaticZeroKeyTimedFwdRev() { } private void doTestStaticZeroKey(final int prevTicks, final int postTicks) { - final QueryTable t = createTestTable(STATIC_TABLE_SIZE, true, false, false, 0x31313131, + final QueryTable t = createSmallTestTable(STATIC_TABLE_SIZE, true, false, false, 0x31313131, new String[] {"charCol"}, new TestDataGenerator[] {new CharGenerator('A', 'z', 0.1)}).t; @@ -411,7 +465,7 @@ private void doTestStaticZeroKey(final int prevTicks, final int postTicks) { } private void doTestStaticZeroKeyTimed(final Duration prevTime, final Duration postTime) { - final QueryTable t = createTestTable(STATIC_TABLE_SIZE, false, false, false, 0xFFFABBBC, + final QueryTable t = createSmallTestTable(STATIC_TABLE_SIZE, false, false, false, 0xFFFABBBC, new String[] {"ts", "charCol"}, new TestDataGenerator[] {new SortedInstantGenerator( DateTimeUtils.parseInstant("2022-03-09T09:00:00.000 NY"), DateTimeUtils.parseInstant("2022-03-09T16:30:00.000 NY")), diff --git a/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java b/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java index c5fb4853029..da86b6cb406 100644 --- a/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java +++ b/replication/static/src/main/java/io/deephaven/replicators/ReplicateUpdateBy.java @@ -134,13 +134,17 @@ public static void main(String[] args) throws IOException { } } - files = ReplicatePrimitiveCode.charToAllButBoolean( - "engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/CharRollingProductOperator.java"); + files = ReplicatePrimitiveCode.charToIntegers( + "engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/CharRollingProductOperator.java", + exemptions); for (final String f : files) { - if (f.contains("Integer")) { + if (f.contains("Int")) { fixupInteger(f); } } + ReplicatePrimitiveCode.floatToAllFloatingPoints( + "engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingproduct/FloatRollingProductOperator.java"); + files = ReplicatePrimitiveCode.charToAllButBoolean( "engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/CharDeltaOperator.java",