Skip to content

Commit

Permalink
correct precision error in unit test
Browse files Browse the repository at this point in the history
  • Loading branch information
@pengfei99
pengfei99 committed Dec 14, 2023
1 parent deb9c6e commit d6ddca5
Show file tree
Hide file tree
Showing 6 changed files with 71 additions and 20 deletions.
26 changes: 26 additions & 0 deletions vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/TestUtilities.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package fr.insee.vtl.spark.processing.engine;

import fr.insee.vtl.model.Dataset;

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.List;
import java.util.Map;

public class TestUtilities {

public static List<Map<String, Object>> roundDecimalInDataset(Dataset dataset){
List<Map<String, Object>> res = dataset.getDataAsMap();
for (Map<String, Object> map : res) {
for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() instanceof Double || entry.getValue() instanceof Float) {
double value = ((Number) entry.getValue()).doubleValue();
BigDecimal roundedValue = BigDecimal.valueOf(value).setScale(2, RoundingMode.HALF_UP);
map.put(entry.getKey(), roundedValue.doubleValue());
}
}
}
return res;
}

}
10 changes: 7 additions & 3 deletions vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticAvgTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import fr.insee.vtl.engine.VtlScriptEngine;
import fr.insee.vtl.model.Dataset;
import fr.insee.vtl.model.InMemoryDataset;
import fr.insee.vtl.spark.processing.engine.TestUtilities;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -206,10 +207,12 @@ public void testAnAvgWithPartitionOrderByClause() throws ScriptException {
+----+----+----+----+----+-----------------+--------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).contains(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));

assertThat(res).contains(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3.0D, "Me_2", 1.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.5D, "Me_2", 5.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.666666666666667D, "Me_2", 5.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.67D, "Me_2", 5.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5.0D, "Me_2", 5.75D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9.0D, "Me_2", 3.0D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.0D, "Me_2", 3.5D),
Expand Down Expand Up @@ -261,7 +264,8 @@ public void testAnAvgWithPartitionOrderByDPClause() throws ScriptException {
+----+----+----+----+----+-----------------+-----------------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4.67D, "Me_2", 5.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 5.0D, "Me_2", 5.75D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.8D, "Me_2", 5.2D),
Expand Down
12 changes: 8 additions & 4 deletions ...spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdPopTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package fr.insee.vtl.spark.processing.engine.analytic;

import fr.insee.vtl.model.Dataset;
import fr.insee.vtl.spark.processing.engine.TestUtilities;
import org.junit.jupiter.api.Test;

import javax.script.ScriptContext;
Expand Down Expand Up @@ -107,7 +108,8 @@ public void testAnStdPopWithPartitionClause() throws ScriptException {
| A| YY|2003| 5| 7.0| 2.277608394786075|1.8708286933869707|
+----+----+----+----+----+------------------+------------------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.58D, "Me_2", 3.11D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.58D, "Me_2", 3.11D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.58D, "Me_2", 3.11D),
Expand Down Expand Up @@ -160,7 +162,8 @@ public void testAnStdPopWithPartitionOrderByClause() throws ScriptException {
+----+----+----+----+----+------------------+------------------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.5D, "Me_2", 4.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.70D, "Me_2", 3.27D),
Expand Down Expand Up @@ -215,13 +218,14 @@ public void testAnStdPopWithPartitionOrderByDPClause() throws ScriptException {
+----+----+----+----+----+------------------+------------------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.70D, "Me_2", 3.27D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.58D, "Me_2", 3.11D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.14D, "Me_2", 2.99D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.72D, "Me_2", 2.32D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 1.85D, "Me_2", 2.06D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.09D, "Me_2", 2.32D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.1D, "Me_2", 2.32D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.28D, "Me_2", 1.87D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.36D, "Me_2", 2.05D)
);
Expand Down
11 changes: 8 additions & 3 deletions ...park/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdSampTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package fr.insee.vtl.spark.processing.engine.analytic;

import fr.insee.vtl.model.Dataset;
import fr.insee.vtl.spark.processing.engine.TestUtilities;
import org.junit.jupiter.api.Test;

import javax.script.ScriptContext;
Expand Down Expand Up @@ -115,7 +116,8 @@ public void testAnStdSampWithPartitionClause() throws ScriptException {
| A| YY|2003| 5| 7.0|2.6299556396765835|2.160246899469287|
+----+----+----+----+----+------------------+-----------------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.83D, "Me_2", 3.59D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.83D, "Me_2", 3.59D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.83D, "Me_2", 3.59D),
Expand Down Expand Up @@ -231,10 +233,13 @@ public void testAnStdSampWithPartitionOrderByDPClause() throws ScriptException {
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(

List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));

assertThat(res).contains(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.08D, "Me_2", 4.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.83D, "Me_2", 3.59D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.38D, "Me_2", 3.35D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.39D, "Me_2", 3.35D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.92D, "Me_2", 2.59D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.07D, "Me_2", 2.30D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.35D, "Me_2", 2.59D),
Expand Down
16 changes: 12 additions & 4 deletions ...spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarPopTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import fr.insee.vtl.engine.VtlScriptEngine;
import fr.insee.vtl.model.Dataset;
import fr.insee.vtl.model.InMemoryDataset;
import fr.insee.vtl.spark.processing.engine.TestUtilities;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
Expand All @@ -12,6 +13,8 @@
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -158,7 +161,8 @@ public void testAnVarPopWithPartitionClause() throws ScriptException {
| A| YY|2003| 5| 7.0| 5.1875| 3.5|
+----+----+----+----+----+------------------+------------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.50D, "Me_2", 9.69D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.50D, "Me_2", 9.69D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.50D, "Me_2", 9.69D),
Expand Down Expand Up @@ -212,7 +216,10 @@ public void testAnVarPopWithPartitionOrderByClause() throws ScriptException {
* */

assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));


assertThat(res).contains(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.25D, "Me_2", 16.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.89D, "Me_2", 10.67D),
Expand Down Expand Up @@ -268,9 +275,10 @@ public void testAnVarPopWithPartitionOrderByDPClause() throws ScriptException {
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.89D, "Me_2", 10.67D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.49D, "Me_2", 9.69D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.5D, "Me_2", 9.69D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.56D, "Me_2", 8.96D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.96D, "Me_2", 5.36D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 3.44D, "Me_2", 4.24D),
Expand Down
16 changes: 10 additions & 6 deletions ...park/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarSampTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package fr.insee.vtl.spark.processing.engine.analytic;

import fr.insee.vtl.model.Dataset;
import fr.insee.vtl.spark.processing.engine.TestUtilities;
import org.junit.jupiter.api.Test;

import javax.script.ScriptContext;
import javax.script.ScriptException;
import java.util.List;
import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -52,11 +54,12 @@ public void testAnVarSampWithPartitionClause() throws ScriptException {
| A| YY|2003| 5| 7.0| 6.916666666666667| 4.666666666666667|
+----+----+----+----+----+------------------+------------------+
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3.33D, "Me_2", 12.91D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.33D, "Me_2", 12.91D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3.33D, "Me_2", 12.91D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.33D, "Me_2", 12.91D),
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).contains(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3.33D, "Me_2", 12.92D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.33D, "Me_2", 12.92D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3.33D, "Me_2", 12.92D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.33D, "Me_2", 12.92D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 6.92D, "Me_2", 4.67D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6.92D, "Me_2", 4.67D),
Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6.92D, "Me_2", 4.67D),
Expand Down Expand Up @@ -168,7 +171,8 @@ public void testAnVarSampWithPartitionOrderByDPClause() throws ScriptException {
* */
assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly(
List<Map<String, Object>> res = TestUtilities.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"));
assertThat(res).containsExactly(
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4.33D, "Me_2", 16.0D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.33D, "Me_2", 12.92D),
Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.70D, "Me_2", 11.2D),
Expand Down

0 comments on commit d6ddca5

Please sign in to comment.