diff --git a/README.md b/README.md index d6ed84808..96031c1bd 100644 --- a/README.md +++ b/README.md @@ -28,4 +28,4 @@ If you want to contribute, see this [guide](docs/CONTRIBUTING.md). ## Requirements -Open JDK 11.0.4 + is required. +Open JDK 8+ is required. diff --git a/pom.xml b/pom.xml index f0bc4790d..285d2ae38 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ - 11 + 8 UTF-8 0.8.7 @@ -109,7 +109,7 @@ - 11.0.4 + 8 3.3.9 diff --git a/vtl-csv/src/main/java/fr/insee/vtl/csv/CSVDataset.java b/vtl-csv/src/main/java/fr/insee/vtl/csv/CSVDataset.java index 287956c5a..a99211d70 100644 --- a/vtl-csv/src/main/java/fr/insee/vtl/csv/CSVDataset.java +++ b/vtl-csv/src/main/java/fr/insee/vtl/csv/CSVDataset.java @@ -1,6 +1,7 @@ package fr.insee.vtl.csv; import fr.insee.vtl.model.Dataset; +import fr.insee.vtl.model.utils.Java8Helpers; import org.supercsv.cellprocessor.Optional; import org.supercsv.cellprocessor.ParseBool; import org.supercsv.cellprocessor.ParseDouble; @@ -26,8 +27,8 @@ public class CSVDataset implements Dataset { public CSVDataset(DataStructure structure, Reader csv) throws IOException { this.structure = structure; this.csvReader = new CsvMapReader(csv, CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE); - var columns = this.csvReader.getHeader(true); - if (!this.structure.keySet().containsAll(List.of(columns))) { + String[] columns = this.csvReader.getHeader(true); + if (!this.structure.keySet().containsAll(Java8Helpers.listOf(columns))) { throw new RuntimeException("missing columns in CSV"); } } @@ -68,8 +69,8 @@ public List getDataPoints() { if (this.data == null) { this.data = new ArrayList<>(); try { - var header = getNameMapping(); - var processors = getProcessors(); + String[] header = getNameMapping(); + CellProcessor[] processors = getProcessors(); Map datum; while ((datum = this.csvReader.read(header, processors)) != null) { this.data.add(new DataPoint(this.structure, datum)); diff --git a/vtl-engine/pom.xml b/vtl-engine/pom.xml index f95e896c6..363a402f1 100644 --- a/vtl-engine/pom.xml +++ b/vtl-engine/pom.xml @@ -71,6 +71,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/TemporalFunctions.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/TemporalFunctions.java index acb5550da..fc558a8a7 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/TemporalFunctions.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/TemporalFunctions.java @@ -1,14 +1,10 @@ package fr.insee.vtl.engine; -import fr.insee.vtl.engine.exceptions.VtlRuntimeException; -import fr.insee.vtl.model.exceptions.VtlScriptException; import org.threeten.extra.Interval; import org.threeten.extra.PeriodDuration; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; import java.time.*; -import java.time.temporal.*; +import java.time.temporal.ChronoUnit; /** * This comment explains the temporal functionality supported by Trevas, as defined in the VTL 2.0 specification. @@ -188,19 +184,19 @@ public static PeriodDuration period_indicator(Interval timePeriod) { public static Interval timeshift(Interval time, Long n) { OffsetDateTime from = time.getStart().atOffset(ZoneOffset.UTC); OffsetDateTime to = time.getEnd().atOffset(ZoneOffset.UTC); - var dur = PeriodDuration.between(from, to) + PeriodDuration dur = PeriodDuration.between(from, to) .multipliedBy(n.intValue()); return Interval.of(from.plus(dur.getPeriod()).toInstant(), to.plus(dur.getPeriod()).toInstant()); } public static ZonedDateTime at_zone(Instant op, String zone) { - var zid = ZoneId.of(zone); + ZoneId zid = ZoneId.of(zone); return op.atZone(zid); } private static Interval truncate_time(Interval op, ChronoUnit unit, ZoneId zone) { - var start = truncate_time(op.getStart(), unit, zone); + Instant start = truncate_time(op.getStart(), unit, zone); return Interval.of(start, unit.getDuration()); } @@ -213,7 +209,7 @@ public static Interval truncate_time(Interval op, String unit) { } private static Instant truncate_time(Instant op, ChronoUnit unit, ZoneId zone) { - var zonedOp = op.atZone(zone); + ZonedDateTime zonedOp = op.atZone(zone); switch (unit) { case DAYS: return zonedOp.truncatedTo(ChronoUnit.DAYS).toInstant(); @@ -248,7 +244,7 @@ public static ZonedDateTime truncate_time(ZonedDateTime op, String unit) { } public static OffsetDateTime truncate_time(OffsetDateTime op, String unit) { - var zoned = op.toZonedDateTime(); + ZonedDateTime zoned = op.toZonedDateTime(); return truncate_time(zoned.toInstant(), toChronoUnit(unit), zoned.getZone()).atOffset(op.getOffset()); } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlNativeMethods.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlNativeMethods.java index 43a9316c2..cdc544539 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlNativeMethods.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlNativeMethods.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine; import com.github.hervian.reflection.Fun; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.*; import fr.insee.vtl.engine.visitors.expression.functions.ComparisonFunctionsVisitor; import fr.insee.vtl.engine.visitors.expression.functions.DistanceFunctionsVisitor; @@ -16,7 +17,7 @@ public class VtlNativeMethods { - public static final Set NATIVE_METHODS = Set.of( + public static final Set NATIVE_METHODS = Java8Helpers.setOf( // NumericFunctionsVisitor Fun.toMethod(NumericFunctionsVisitor::ceil), Fun.toMethod(NumericFunctionsVisitor::floor), diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngine.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngine.java index 2a061a78d..49407ba41 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngine.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngine.java @@ -2,53 +2,24 @@ import fr.insee.vtl.engine.exceptions.VtlRuntimeException; import fr.insee.vtl.engine.exceptions.VtlSyntaxException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.AssignmentVisitor; -import fr.insee.vtl.model.FunctionProvider; -import fr.insee.vtl.model.Positioned; -import fr.insee.vtl.model.ProcessingEngine; -import fr.insee.vtl.model.ProcessingEngineFactory; -import fr.insee.vtl.model.VtlMethod; +import fr.insee.vtl.model.*; import fr.insee.vtl.model.exceptions.VtlScriptException; import fr.insee.vtl.parser.VtlLexer; import fr.insee.vtl.parser.VtlParser; -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CodePointCharStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.TerminalNode; -import javax.script.AbstractScriptEngine; -import javax.script.Bindings; -import javax.script.ScriptContext; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineFactory; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; -import javax.script.SimpleBindings; +import javax.script.*; import java.io.IOException; import java.io.Reader; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.lang.reflect.TypeVariable; -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Collection; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.ServiceLoader; -import java.util.Set; -import java.util.StringJoiner; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -113,7 +84,7 @@ public static Positioned fromTokens(Token from, Token to) { if (to == null) { to = from; } - var position = new Positioned.Position( + Positioned.Position position = new Positioned.Position( from.getLine() - 1, to.getLine() - 1, from.getCharPositionInLine(), @@ -201,12 +172,10 @@ private String getProcessingEngineName() { */ public ProcessingEngine getProcessingEngine() { String name = getProcessingEngineName(); - Optional factory = ServiceLoader.load(ProcessingEngineFactory.class) - .stream() - .map(ServiceLoader.Provider::get) + Optional factory = Java8Helpers.streamIterator(ServiceLoader.load(ProcessingEngineFactory.class).iterator()) .filter(f -> f.getName().equals(name)) .findFirst(); - return factory.orElseThrow().getProcessingEngine(this); + return factory.orElseThrow(() -> new NoSuchElementException("No value present")).getProcessingEngine(this); } /** @@ -231,7 +200,7 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int if (offendingSymbol instanceof Token) { errors.add(new VtlSyntaxException(msg, fromToken((Token) offendingSymbol))); } else { - var pos = new Positioned.Position(startLine, startLine, startColumn, startColumn + 1); + Positioned.Position pos = new Positioned.Position(startLine, startLine, startColumn, startColumn + 1); errors.add(new VtlScriptException(msg, () -> pos)); } } @@ -248,10 +217,10 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int // Note that we need to call this method to trigger the // error listener. - var start = parser.start(); + VtlParser.StartContext start = parser.start(); if (!errors.isEmpty()) { - var first = errors.removeFirst(); + VtlScriptException first = errors.removeFirst(); for (VtlScriptException suppressed : errors) { first.addSuppressed(suppressed); } @@ -322,14 +291,14 @@ public ScriptEngineFactory getFactory() { } public VtlMethod findMethod(String name, Collection types) throws NoSuchMethodException { - Set customMethods = methodCache == null ? Set.of() + Set customMethods = methodCache == null ? Java8Helpers.setOf() : new HashSet<>(methodCache.values()); Set methods = Stream.concat(NATIVE_METHODS.stream(), customMethods.stream()) .collect(Collectors.toSet()); List candidates = methods.stream() .filter(method -> method.getName().equals(name)) - .filter(method -> matchParameters(method, types.toArray(Class[]::new))) + .filter(method -> matchParameters(method, types.toArray(new Class[0]))) .collect(Collectors.toList()); if (candidates.size() == 1) { return new VtlMethod(candidates.get(0)); @@ -349,7 +318,7 @@ public VtlMethod findGlobalMethod(String name, Collection types) throws N List candidates = methods.stream() .filter(method -> method.getName().equals(name)) - .filter(method -> matchParameters(method, types.toArray(Class[]::new))) + .filter(method -> matchParameters(method, types.toArray(new Class[0]))) .collect(Collectors.toList()); if (candidates.size() == 0) { diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngineFactory.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngineFactory.java index 4abfa10ef..9efde7638 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngineFactory.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngineFactory.java @@ -1,5 +1,7 @@ package fr.insee.vtl.engine; +import fr.insee.vtl.model.utils.Java8Helpers; + import javax.script.ScriptEngine; import javax.script.ScriptEngineFactory; import java.util.List; @@ -37,7 +39,7 @@ public String getEngineVersion() { */ @Override public List getExtensions() { - return List.of("vtl"); + return Java8Helpers.listOf("vtl"); } /** @@ -47,7 +49,7 @@ public List getExtensions() { */ @Override public List getMimeTypes() { - return List.of(); + return Java8Helpers.listOf(); } /** @@ -57,7 +59,7 @@ public List getMimeTypes() { */ @Override public List getNames() { - return List.of(getLanguageName(), getEngineName(), "vtl", "Trevas", "trevas"); + return Java8Helpers.listOf(getLanguageName(), getEngineName(), "vtl", "Trevas", "trevas"); } /** diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/CastExpression.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/CastExpression.java index 2af0d1c38..056f1cd59 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/CastExpression.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/CastExpression.java @@ -22,7 +22,7 @@ public class CastExpression extends ResolvableExpression { public CastExpression(Positioned position, ResolvableExpression expr, String mask, Class target) throws VtlScriptException { super(position); this.target = target; - var source = expr.getType(); + Class source = expr.getType(); if (source.equals(target)) { this.expr = expr; } else { @@ -45,7 +45,7 @@ public CastExpression(Positioned position, ResolvableExpression expr, String mas } public ResolvableExpression castBoolean(ResolvableExpression expr) { - var outputClass = getType(); + Class outputClass = getType(); if (outputClass.equals(String.class)) { return ResolvableExpression.withType(String.class).withPosition(expr).using(context -> { Boolean exprValue = (Boolean) expr.resolve(context); @@ -69,7 +69,7 @@ public ResolvableExpression castBoolean(ResolvableExpression expr) { } private ResolvableExpression castDouble(ResolvableExpression expr) { - var outputClass = getType(); + Class outputClass = getType(); if (outputClass.equals(String.class)) return ResolvableExpression.withType(String.class).withPosition(expr).using(context -> { Double exprValue = (Double) expr.resolve(context); @@ -98,11 +98,11 @@ private ResolvableExpression castDouble(ResolvableExpression expr) { } private ResolvableExpression castInstant(ResolvableExpression expr, String mask) { - var outputClass = getType(); + Class outputClass = getType(); if (outputClass.equals(String.class)) return ResolvableExpression.withType(String.class).withPosition(expr).using(context -> { - var value = expr.resolve(context); + Object value = expr.resolve(context); Instant exprValue; if (value instanceof LocalDate) { exprValue = ((LocalDate) value).atStartOfDay().toInstant(ZoneOffset.UTC); @@ -117,7 +117,7 @@ private ResolvableExpression castInstant(ResolvableExpression expr, String mask) } private ResolvableExpression castLong(ResolvableExpression expr) { - var outputClass = getType(); + Class outputClass = getType(); if (outputClass.equals(String.class)) return ResolvableExpression.withType(String.class).withPosition(expr).using(context -> { Long exprValue = (Long) expr.resolve(context); @@ -140,7 +140,7 @@ private ResolvableExpression castLong(ResolvableExpression expr) { } private ResolvableExpression castString(ResolvableExpression expr, String mask) { - var outputClass = getType(); + Class outputClass = getType(); if (outputClass.equals(Long.class)) { return ResolvableExpression.withType(Long.class).withPosition(expr).using(context -> { String exprValue = (String) expr.resolve(context); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/FunctionExpression.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/FunctionExpression.java index 1a00b3765..9fc6bec34 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/FunctionExpression.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/expressions/FunctionExpression.java @@ -8,10 +8,7 @@ import fr.insee.vtl.model.exceptions.VtlScriptException; import java.lang.reflect.InvocationTargetException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; /** * An expression that calls a method. @@ -29,18 +26,18 @@ public FunctionExpression(VtlMethod method, List parameter this.parameters = Objects.requireNonNull(parameters); this.returnType = this.method.getMethod(position).getReturnType(); - var expectedTypes = Arrays.asList(this.method.getMethod(position).getParameterTypes()); + List> expectedTypes = Arrays.asList(this.method.getMethod(position).getParameterTypes()); if (expectedTypes.size() < parameters.size()) { throw new VtlScriptException("unexpected parameter", parameters.get(expectedTypes.size())); } else if (expectedTypes.size() > parameters.size()) { throw new VtlScriptException("missing parameter", position); } - var exprIt = parameters.iterator(); - var typeIt = expectedTypes.iterator(); + Iterator exprIt = parameters.iterator(); + Iterator> typeIt = expectedTypes.iterator(); while (exprIt.hasNext() && typeIt.hasNext()) { - var expression = exprIt.next(); - var type = typeIt.next(); + ResolvableExpression expression = exprIt.next(); + Class type = typeIt.next(); if (type.equals(Object.class)) { continue; } @@ -57,7 +54,7 @@ public Object resolve(Map context) { try { return method.getMethod(this).invoke(null, evaluatedParameters); } catch (InvocationTargetException ite) { - var cause = ite.getCause(); + Throwable cause = ite.getCause(); if (cause instanceof Exception) { throw new VtlRuntimeException( new VtlScriptException((Exception) ite.getCause(), this)); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/functions/LevenshteinProvider.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/functions/LevenshteinProvider.java index 4b23ee8cd..65281a517 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/functions/LevenshteinProvider.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/functions/LevenshteinProvider.java @@ -1,5 +1,6 @@ package fr.insee.vtl.engine.functions; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.FunctionProvider; import javax.script.ScriptEngine; @@ -10,7 +11,7 @@ public class LevenshteinProvider implements FunctionProvider { @Override public Map getFunctions(ScriptEngine vtlEngine) { - return Map.of(); + return Java8Helpers.mapOf(); } } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/processors/InMemoryProcessingEngine.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/processors/InMemoryProcessingEngine.java index af4214369..68245fa0b 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/processors/InMemoryProcessingEngine.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/processors/InMemoryProcessingEngine.java @@ -1,22 +1,16 @@ package fr.insee.vtl.engine.processors; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.utils.KeyExtractor; import fr.insee.vtl.engine.utils.MapCollector; import fr.insee.vtl.model.*; import javax.script.ScriptEngine; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; -import static fr.insee.vtl.model.Structured.Component; -import static fr.insee.vtl.model.Structured.DataPoint; -import static fr.insee.vtl.model.Structured.DataStructure; +import static fr.insee.vtl.model.Structured.*; /** * The InMemoryProcessingEngine class is an implementation of a VTL engine that performs all operations in memory. @@ -28,7 +22,7 @@ public DatasetExpression executeCalc(DatasetExpression expression, Map roles, Map expressionStrings) { // Copy the structure and mutate based on the expressions. - var newStructure = new DataStructure(expression.getDataStructure()); + Structured.DataStructure newStructure = new DataStructure(expression.getDataStructure()); for (String columnName : expressions.keySet()) { // TODO: refine nullable strategy newStructure.put(columnName, new Dataset.Component( @@ -43,9 +37,9 @@ public DatasetExpression executeCalc(DatasetExpression expression, Map context) { - var dataset = expression.resolve(context); + Dataset dataset = expression.resolve(context); List> result = dataset.getDataPoints().stream().map(dataPoint -> { - var newDataPoint = new DataPoint(newStructure, dataPoint); + DataPoint newDataPoint = new DataPoint(newStructure, dataPoint); for (String columnName : expressions.keySet()) { newDataPoint.set(columnName, expressions.get(columnName).resolve(dataPoint)); } @@ -75,7 +69,7 @@ public Dataset resolve(Map context) { Dataset resolve = expression.resolve(context); List> result = resolve.getDataPoints().stream() .filter(map -> { - var res = filter.resolve(map); + Object res = filter.resolve(map); if (res == null) return false; return (boolean) res; }) @@ -90,7 +84,7 @@ public DatasetExpression executeRename(DatasetExpression expression, Map structure = expression.getDataStructure().values().stream() .map(component -> !fromTo.containsKey(component.getName()) ? component @@ -105,11 +99,11 @@ public DatasetExpression executeRename(DatasetExpression expression, Map context) { - var result = expression.resolve(context).getDataPoints().stream() + List result = expression.resolve(context).getDataPoints().stream() .map(dataPoint -> { - var newDataPoint = new DataPoint(renamedStructure, dataPoint); + Structured.DataPoint newDataPoint = new DataPoint(renamedStructure, dataPoint); for (String fromName : fromTo.keySet()) { - var toName = fromTo.get(fromName); + String toName = fromTo.get(fromName); newDataPoint.set(toName, dataPoint.get(fromName)); } return newDataPoint; @@ -127,18 +121,18 @@ public DataStructure getDataStructure() { @Override public DatasetExpression executeProject(DatasetExpression expression, List columnNames) { - var structure = expression.getDataStructure().values().stream() + List structure = expression.getDataStructure().values().stream() .filter(component -> columnNames.contains(component.getName())) .collect(Collectors.toList()); - var newStructure = new DataStructure(structure); + DataStructure newStructure = new DataStructure(structure); return new DatasetExpression(expression) { @Override public Dataset resolve(Map context) { - var columnNames = getColumnNames(); + List columnNames = getColumnNames(); List> result = expression.resolve(context).getDataPoints().stream() .map(data -> { - var projectedDataPoint = new DataPoint(newStructure); + DataPoint projectedDataPoint = new DataPoint(newStructure); for (String column : columnNames) { projectedDataPoint.set(column, data.get(column)); } @@ -162,7 +156,7 @@ public DatasetExpression executeUnion(List datasets) { public Dataset resolve(Map context) { Stream stream = Stream.empty(); for (DatasetExpression datasetExpression : datasets) { - var dataset = datasetExpression.resolve(context); + Dataset dataset = datasetExpression.resolve(context); stream = Stream.concat(stream, dataset.getDataPoints().stream()); } List data = stream.distinct().collect(Collectors.toList()); @@ -179,7 +173,7 @@ public DataStructure getDataStructure() { @Override public DatasetExpression executeAggr(DatasetExpression expression, List groupBy, Map collectorMap) { // Create a keyExtractor with the columns we group by. - var keyExtractor = new KeyExtractor(groupBy); + KeyExtractor keyExtractor = new KeyExtractor(groupBy); // Compute the new data structure. Map newStructure = new LinkedHashMap<>(); @@ -203,7 +197,7 @@ public DatasetExpression executeAggr(DatasetExpression expression, List @Override public Dataset resolve(Map context) { - List data = expression.resolve(Map.of()).getDataPoints(); + List data = expression.resolve(Java8Helpers.mapOf()).getDataPoints(); MapCollector collector = new MapCollector(structure, collectorMap); List collect = data.stream() .collect(Collectors.groupingBy(keyExtractor, collector)) @@ -253,8 +247,8 @@ public DatasetExpression executeRankAn(DatasetExpression dataset, @Override public DatasetExpression executeLeftJoin(Map datasets, List components) { - var iterator = datasets.values().iterator(); - var leftMost = iterator.next(); + Iterator iterator = datasets.values().iterator(); + DatasetExpression leftMost = iterator.next(); while (iterator.hasNext()) { leftMost = handleLeftJoin(components, leftMost, iterator.next()); } @@ -263,8 +257,8 @@ public DatasetExpression executeLeftJoin(Map datasets @Override public DatasetExpression executeInnerJoin(Map datasets, List components) { - var iterator = datasets.values().iterator(); - var leftMost = iterator.next(); + Iterator iterator = datasets.values().iterator(); + DatasetExpression leftMost = iterator.next(); while (iterator.hasNext()) { leftMost = handleInnerJoin(components, leftMost, iterator.next()); } @@ -273,8 +267,8 @@ public DatasetExpression executeInnerJoin(Map dataset @Override public DatasetExpression executeCrossJoin(Map datasets, List identifiers) { - var iterator = datasets.values().iterator(); - var leftMost = iterator.next(); + Iterator iterator = datasets.values().iterator(); + DatasetExpression leftMost = iterator.next(); while (iterator.hasNext()) { leftMost = handleCrossJoin(identifiers, leftMost, iterator.next()); } @@ -283,8 +277,8 @@ public DatasetExpression executeCrossJoin(Map dataset @Override public DatasetExpression executeFullJoin(Map datasets, List identifiers) { - var iterator = datasets.values().iterator(); - var leftMost = iterator.next(); + Iterator iterator = datasets.values().iterator(); + DatasetExpression leftMost = iterator.next(); while (iterator.hasNext()) { leftMost = handleFullJoin(identifiers, leftMost, iterator.next()); } @@ -343,14 +337,14 @@ private Comparator createPredicate(List identifiers) { } private DatasetExpression handleInnerJoin(List identifiers, DatasetExpression left, DatasetExpression right) { - var structure = createCommonStructure(identifiers, left, right); - var predicate = createPredicate(identifiers); + Structured.DataStructure structure = createCommonStructure(identifiers, left, right); + Comparator predicate = createPredicate(identifiers); return new DatasetExpression(left) { @Override public Dataset resolve(Map context) { - var leftPoints = left.resolve(context).getDataPoints(); - var rightPoints = right.resolve(context).getDataPoints(); + List leftPoints = left.resolve(context).getDataPoints(); + List rightPoints = right.resolve(context).getDataPoints(); List result = new ArrayList<>(); for (DataPoint leftPoint : leftPoints) { List matches = new ArrayList<>(); @@ -363,12 +357,12 @@ public Dataset resolve(Map context) { if (!matches.isEmpty()) { // Create merge datapoint. - var mergedPoint = new DataPoint(structure); + DataPoint mergedPoint = new DataPoint(structure); for (String leftColumn : left.getDataStructure().keySet()) { mergedPoint.set(leftColumn, leftPoint.get(leftColumn)); } for (DataPoint match : matches) { - var matchPoint = new DataPoint(structure, mergedPoint); + DataPoint matchPoint = new DataPoint(structure, mergedPoint); for (String rightColumn : right.getDataStructure().keySet()) { matchPoint.set(rightColumn, match.get(rightColumn)); } @@ -388,21 +382,21 @@ public DataStructure getDataStructure() { private DatasetExpression handleFullJoin(List identifiers, DatasetExpression left, DatasetExpression right) { // Naive implementation, left and right union. Could be optimized. - return executeUnion(List.of( + return executeUnion(Java8Helpers.listOf( handleLeftJoin(identifiers, left, right), handleLeftJoin(identifiers, right, left) )); } private DatasetExpression handleLeftJoin(List identifiers, DatasetExpression left, DatasetExpression right) { - var structure = createCommonStructure(identifiers, left, right); - var predicate = createPredicate(identifiers); + Structured.DataStructure structure = createCommonStructure(identifiers, left, right); + Comparator predicate = createPredicate(identifiers); return new DatasetExpression(left) { @Override public Dataset resolve(Map context) { - var leftPoints = left.resolve(context).getDataPoints(); - var rightPoints = right.resolve(context).getDataPoints(); + List leftPoints = left.resolve(context).getDataPoints(); + List rightPoints = right.resolve(context).getDataPoints(); List result = new ArrayList<>(); for (DataPoint leftPoint : leftPoints) { List matches = new ArrayList<>(); @@ -414,7 +408,7 @@ public Dataset resolve(Map context) { } // Create merge datapoint. - var mergedPoint = new DataPoint(structure); + DataPoint mergedPoint = new DataPoint(structure); for (String leftColumn : left.getDataStructure().keySet()) { mergedPoint.set(leftColumn, leftPoint.get(leftColumn)); } @@ -423,7 +417,7 @@ public Dataset resolve(Map context) { result.add(mergedPoint); } else { for (DataPoint match : matches) { - var matchPoint = new DataPoint(structure, mergedPoint); + DataPoint matchPoint = new DataPoint(structure, mergedPoint); for (String rightColumn : right.getDataStructure().keySet()) { matchPoint.set(rightColumn, match.get(rightColumn)); } @@ -442,17 +436,17 @@ public DataStructure getDataStructure() { } private DatasetExpression handleCrossJoin(List identifiers, DatasetExpression left, DatasetExpression right) { - var structure = createCommonStructure(identifiers, left, right); + DataStructure structure = createCommonStructure(identifiers, left, right); return new DatasetExpression(left) { @Override public Dataset resolve(Map context) { - var leftPoints = left.resolve(context).getDataPoints(); - var rightPoints = right.resolve(context).getDataPoints(); + List leftPoints = left.resolve(context).getDataPoints(); + List rightPoints = right.resolve(context).getDataPoints(); List result = new ArrayList<>(); // Nested-loop implementation for (DataPoint leftPoint : leftPoints) { for (DataPoint rightPoint : rightPoints) { - var mergedPoint = new DataPoint(structure); + DataPoint mergedPoint = new DataPoint(structure); for (String leftColumn : left.getDataStructure().keySet()) { mergedPoint.set(leftColumn, leftPoint.get(leftColumn)); } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/KeyExtractor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/KeyExtractor.java index d960f420b..f3ae8a06c 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/KeyExtractor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/KeyExtractor.java @@ -1,13 +1,9 @@ package fr.insee.vtl.engine.utils; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.function.Function; /** @@ -25,10 +21,10 @@ public KeyExtractor(Collection columns) { @Override public Map apply(Structured.DataPoint objects) { - List> entries = new ArrayList<>(objects.size()); + List> entries = new ArrayList<>(objects.size()); for (String column : columns) { - entries.add(Map.entry(column, objects.get(column))); + entries.add(Java8Helpers.MapEntry.of(column, objects.get(column))); } - return Map.ofEntries(entries.toArray(Map.Entry[]::new)); + return Java8Helpers.mapOfEntries(entries.toArray(new Java8Helpers.MapEntry[0])); } } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/MapCollector.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/MapCollector.java index 38817e78a..cfbb55707 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/MapCollector.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/MapCollector.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.utils; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import java.util.HashMap; import java.util.Map; @@ -90,6 +91,6 @@ public Function finisher() { @Override public Set characteristics() { - return Set.of(Characteristics.UNORDERED); + return Java8Helpers.setOf(Characteristics.UNORDERED); } } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/TypeChecking.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/TypeChecking.java index b2aad09ce..c52b1f698 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/TypeChecking.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/utils/TypeChecking.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.utils; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.Positioned; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.TypedExpression; import fr.insee.vtl.model.exceptions.InvalidTypeException; @@ -37,7 +38,7 @@ private TypeChecking() { */ public static T assertTypeExpression(T expression, Class type, ParseTree tree) { if (isNull(expression)) { - var pos = fromContext(tree); + Positioned pos = fromContext(tree); return (T) ResolvableExpression.withType(type).withPosition(pos).using(c -> null); } if (!isType(expression, type)) { @@ -83,7 +84,7 @@ public static T assertNumberOrTypeExpression(T expre * @return A boolean which is true if the expression can be interpreted as the given type, false otherwise. */ public static boolean isNumberOrSameType(TypedExpression expression, Class type) { - var expressionType = expression.getType(); + Class expressionType = expression.getType(); if (isNumber(expression) && Number.class.isAssignableFrom(type)) return true; return type.isAssignableFrom(expressionType); } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AnalyticsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AnalyticsVisitor.java index c95de8c22..cf082821d 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AnalyticsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AnalyticsVisitor.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.InvalidArgumentException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Analytics; import fr.insee.vtl.model.DatasetExpression; import fr.insee.vtl.model.ProcessingEngine; @@ -86,7 +87,7 @@ private Analytics.Function toFunctionEnum(Token op, ParseTree ctx) { */ private List toPartitionBy(VtlParser.PartitionByClauseContext partition) { if (partition == null) { - return List.of(); + return Java8Helpers.listOf(); } return partition.componentID().stream() .map(ClauseVisitor::getName) @@ -101,7 +102,7 @@ private List toPartitionBy(VtlParser.PartitionByClauseContext partition) */ private Map toOrderBy(VtlParser.OrderByClauseContext orderByCtx) { if (orderByCtx == null) { - return Map.of(); + return Java8Helpers.mapOf(); } Map orderBy = new LinkedHashMap<>(); for (VtlParser.OrderByItemContext item : orderByCtx.orderByItem()) { diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AssignmentVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AssignmentVisitor.java index b4cbfafd4..3d278d54c 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AssignmentVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AssignmentVisitor.java @@ -3,6 +3,7 @@ import fr.insee.vtl.engine.VtlScriptEngine; import fr.insee.vtl.engine.exceptions.InvalidArgumentException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.*; import fr.insee.vtl.model.exceptions.InvalidTypeException; @@ -54,7 +55,7 @@ private Object visitAssignment(VtlParser.ExprContext expr) { @Override public Object visitTemporaryAssignment(VtlParser.TemporaryAssignmentContext ctx) { - var result = visitAssignment(ctx.expr()); + Object result = visitAssignment(ctx.expr()); Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); String variableIdentifier = ctx.varID().getText(); bindings.put(variableIdentifier, result); @@ -63,7 +64,7 @@ public Object visitTemporaryAssignment(VtlParser.TemporaryAssignmentContext ctx) @Override public Object visitPersistAssignment(VtlParser.PersistAssignmentContext ctx) { - var result = visitAssignment(ctx.expr()); + Object result = visitAssignment(ctx.expr()); if (result instanceof Dataset) { result = new PersistentDataset((Dataset) result); Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); @@ -78,17 +79,17 @@ public Object visitPersistAssignment(VtlParser.PersistAssignmentContext ctx) { @Override public Object visitDefDatapointRuleset(VtlParser.DefDatapointRulesetContext ctx) { - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); String rulesetName = ctx.rulesetID().getText(); List signature = ctx.rulesetSignature().signature(); List variables = ctx.rulesetSignature().VARIABLE() != null ? signature.stream() .map(s -> s.varID().getText()) - .collect(Collectors.toList()) : List.of(); + .collect(Collectors.toList()) : Java8Helpers.listOf(); List valuedomains = ctx.rulesetSignature().VALUE_DOMAIN() != null ? signature.stream() .map(s -> s.varID().getText()) - .collect(Collectors.toList()) : List.of(); + .collect(Collectors.toList()) : Java8Helpers.listOf(); Map alias = signature.stream() .filter(s -> null != s.alias()) .collect(Collectors.toMap(k -> k.varID().getText(), v -> v.alias().getText())); @@ -171,7 +172,7 @@ public Object visitDefDatapointRuleset(VtlParser.DefDatapointRulesetContext ctx) // TODO: handle when clause (expr ctx) @Override public Object visitDefHierarchical(VtlParser.DefHierarchicalContext ctx) { - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); String rulesetName = ctx.rulesetID().getText(); // Mix variables and valuedomain. Information useless for now, find use case to do so diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java index 3d2dd0dc9..776ae9695 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/ClauseVisitor.java @@ -6,8 +6,11 @@ import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.*; import fr.insee.vtl.model.exceptions.VtlScriptException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; +import fr.insee.vtl.parser.VtlParser.AggrFunctionClauseContext; +import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.misc.Interval; @@ -57,7 +60,7 @@ public static String getName(VtlParser.ComponentIDContext context) { } static String getSource(ParserRuleContext ctx) { - var stream = ctx.getStart().getInputStream(); + CharStream stream = ctx.getStart().getInputStream(); return stream.getText(new Interval( ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex() @@ -66,33 +69,33 @@ static String getSource(ParserRuleContext ctx) { private static AggregationExpression convertToAggregation(VtlParser.AggrDatasetContext groupFunctionCtx, ResolvableExpression expression) { if (groupFunctionCtx.SUM() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.sum(numberExpression); } else if (groupFunctionCtx.AVG() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.avg(numberExpression); } else if (groupFunctionCtx.COUNT() != null) { return AggregationExpression.count(); } else if (groupFunctionCtx.MAX() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.max(numberExpression); } else if (groupFunctionCtx.MIN() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.min(numberExpression); } else if (groupFunctionCtx.MEDIAN() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.median(numberExpression); } else if (groupFunctionCtx.STDDEV_POP() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.stdDevPop(numberExpression); } else if (groupFunctionCtx.STDDEV_SAMP() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.stdDevSamp(numberExpression); } else if (groupFunctionCtx.VAR_POP() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.varPop(numberExpression); } else if (groupFunctionCtx.VAR_SAMP() != null) { - var numberExpression = assertNumber(expression, groupFunctionCtx.expr()); + ResolvableExpression numberExpression = assertNumber(expression, groupFunctionCtx.expr()); return AggregationExpression.varSamp(numberExpression); } else { throw new VtlRuntimeException(new VtlScriptException("not implemented", fromContext(groupFunctionCtx))); @@ -102,8 +105,8 @@ private static AggregationExpression convertToAggregation(VtlParser.AggrDatasetC @Override public DatasetExpression visitKeepOrDropClause(VtlParser.KeepOrDropClauseContext ctx) { // Normalize to keep operation. - var keep = ctx.op.getType() == VtlParser.KEEP; - var names = ctx.componentID().stream().map(ClauseVisitor::getName) + boolean keep = ctx.op.getType() == VtlParser.KEEP; + Set names = ctx.componentID().stream().map(ClauseVisitor::getName) .collect(Collectors.toSet()); List columnNames = datasetExpression.getDataStructure().values().stream().map(Dataset.Component::getName) .filter(name -> keep == names.contains(name)) @@ -115,15 +118,15 @@ public DatasetExpression visitKeepOrDropClause(VtlParser.KeepOrDropClauseContext @Override public DatasetExpression visitCalcClause(VtlParser.CalcClauseContext ctx) { - var expressions = new LinkedHashMap(); - var expressionStrings = new LinkedHashMap(); - var roles = new LinkedHashMap(); - var currentDatasetExpression = datasetExpression; + Map expressions = new LinkedHashMap<>(); + Map expressionStrings = new LinkedHashMap<>(); + Map roles = new LinkedHashMap<>(); + DatasetExpression currentDatasetExpression = datasetExpression; // TODO: Refactor so we call the executeCalc for each CalcClauseItemContext the same way we call the // analytics functions. for (VtlParser.CalcClauseItemContext calcCtx : ctx.calcClauseItem()) { - var columnName = getName(calcCtx.componentID()); - var columnRole = calcCtx.componentRole() == null + String columnName = getName(calcCtx.componentID()); + Dataset.Role columnRole = calcCtx.componentRole() == null ? Dataset.Role.MEASURE : Dataset.Role.valueOf(calcCtx.componentRole().getText().toUpperCase()); @@ -162,8 +165,8 @@ public DatasetExpression visitRenameClause(VtlParser.RenameClauseContext ctx) { Map fromTo = new LinkedHashMap<>(); Set renamed = new HashSet<>(); for (VtlParser.RenameClauseItemContext renameCtx : ctx.renameClauseItem()) { - var toNameString = getName(renameCtx.toName); - var fromNameString = getName(renameCtx.fromName); + String toNameString = getName(renameCtx.toName); + String fromNameString = getName(renameCtx.fromName); if (!renamed.add(toNameString)) { throw new VtlRuntimeException(new InvalidArgumentException( String.format("duplicate column: %s", toNameString), fromContext(renameCtx) @@ -178,7 +181,7 @@ public DatasetExpression visitRenameClause(VtlParser.RenameClauseContext ctx) { public DatasetExpression visitAggrClause(VtlParser.AggrClauseContext ctx) { // Normalize the dataset so the expressions are removed from the aggregations. - var aggregationsWithExpressions = ctx.aggregateClause().aggrFunctionClause().stream() + List aggregationsWithExpressions = ctx.aggregateClause().aggrFunctionClause().stream() .filter(agg -> agg.aggrOperatorsGrouping() instanceof VtlParser.AggrDatasetContext) .collect(Collectors.toList()); @@ -198,20 +201,19 @@ public DatasetExpression visitAggrClause(VtlParser.AggrClauseContext ctx) { agg -> getSource(agg.aggrOperatorsGrouping()) )); - var dataStructure = datasetExpression.getDataStructure(); - + Structured.DataStructure dataStructure = datasetExpression.getDataStructure(); DatasetExpression normalizedDataset = processingEngine.executeCalc(this.datasetExpression, expressions, roles, expressionStrings); // Execute the group all as a calc. List groupBy = new ArrayList<>(); - var groupAll = new GroupAllVisitor(componentExpressionVisitor).visit(ctx); + ResolvableExpression groupAll = new GroupAllVisitor(componentExpressionVisitor).visit(ctx); if (groupAll != null) { // TODO, use the name? What if the expression uses multiple columns. normalizedDataset = processingEngine.executeCalc(normalizedDataset, - Map.of("time", groupAll), - Map.of("time", Dataset.Role.IDENTIFIER), - Map.of() + Java8Helpers.mapOf("time", groupAll), + Java8Helpers.mapOf("time", Dataset.Role.IDENTIFIER), + Java8Helpers.mapOf() ); groupBy.add("time"); } @@ -228,15 +230,13 @@ public DatasetExpression visitAggrClause(VtlParser.AggrClauseContext ctx) { }); - - Map collectorMap = new LinkedHashMap<>(); for (VtlParser.AggrFunctionClauseContext functionCtx : ctx.aggregateClause().aggrFunctionClause()) { String alias = getName(functionCtx.componentID()); // TODO: Refactor to avoid this if if (normalizedStructure.containsKey(alias)) { Structured.Component normalizedComponent = normalizedStructure.get(alias); - var aggregationFunction = convertToAggregation( + AggregationExpression aggregationFunction = convertToAggregation( // Note that here we replace the expression by the name of the columns. (VtlParser.AggrDatasetContext) functionCtx.aggrOperatorsGrouping(), new ResolvableExpression(fromContext(ctx)) { diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/GroupByVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/GroupByVisitor.java index cf43bfcf8..8e3349600 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/GroupByVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/GroupByVisitor.java @@ -3,6 +3,7 @@ import fr.insee.vtl.engine.exceptions.InvalidArgumentException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; @@ -33,14 +34,14 @@ private String getName(VtlParser.ComponentIDContext ctx) { @Override protected List defaultResult() { - return List.of(); + return Java8Helpers.listOf(); } @Override public List visitGroupByOrExcept(VtlParser.GroupByOrExceptContext ctx) { List componentNames = new ArrayList<>(ctx.componentID().size()); - for (var component : ctx.componentID()) { - var componentName = getName(component); + for (VtlParser.ComponentIDContext component : ctx.componentID()) { + String componentName = getName(component); if (!dataStructure.containsKey(componentName)) { throw new VtlRuntimeException(new InvalidArgumentException( String.format("unknown component %s", componentName), diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatVisitor.java index 38289f560..9f6e0a70f 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatVisitor.java @@ -1,7 +1,9 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.functions.GenericFunctionsVisitor; +import fr.insee.vtl.model.Positioned; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; import fr.insee.vtl.parser.VtlBaseVisitor; @@ -100,8 +102,8 @@ public static String concat(String valueA, String valueB) { @Override public ResolvableExpression visitArithmeticExprOrConcat(VtlParser.ArithmeticExprOrConcatContext ctx) { try { - var pos = fromContext(ctx); - var parameters = List.of(exprVisitor.visit(ctx.left), exprVisitor.visit(ctx.right)); + Positioned pos = fromContext(ctx); + List parameters = Java8Helpers.listOf(exprVisitor.visit(ctx.left), exprVisitor.visit(ctx.right)); switch (ctx.op.getType()) { case VtlParser.PLUS: return genericFunctionsVisitor.invokeFunction("addition", parameters, pos); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticVisitor.java index 08768115c..9d739caaf 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ArithmeticVisitor.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.functions.GenericFunctionsVisitor; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; @@ -93,7 +94,7 @@ public static Double division(Double valueA, Double valueB) { @Override public ResolvableExpression visitArithmeticExpr(VtlParser.ArithmeticExprContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.left), exprVisitor.visit(ctx.right) ); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/BooleanVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/BooleanVisitor.java index b3c95fb3d..37d18636f 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/BooleanVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/BooleanVisitor.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.functions.GenericFunctionsVisitor; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; @@ -67,7 +68,7 @@ public static Boolean xor(Boolean left, Boolean right) { @Override public ResolvableExpression visitBooleanExpr(VtlParser.BooleanExprContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.left), exprVisitor.visit(ctx.right) ); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ComparisonVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ComparisonVisitor.java index 865b16eed..1022ba6b5 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ComparisonVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ComparisonVisitor.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.ConflictingTypesException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.functions.GenericFunctionsVisitor; import fr.insee.vtl.model.ListExpression; import fr.insee.vtl.model.Positioned; @@ -15,7 +16,6 @@ import java.util.Date; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -136,8 +136,8 @@ public static Boolean notIn(Object obj, List list) { public ResolvableExpression visitComparisonExpr(VtlParser.ComparisonExprContext ctx) { try { Token type = ((TerminalNode) ctx.op.getChild(0)).getSymbol(); - var leftExpression = exprVisitor.visit(ctx.left); - List parameters = List.of( + ResolvableExpression leftExpression = exprVisitor.visit(ctx.left); + List parameters = Java8Helpers.listOf( leftExpression, exprVisitor.visit(ctx.right)); // If a parameter is the null token @@ -177,7 +177,7 @@ public ResolvableExpression visitComparisonExpr(VtlParser.ComparisonExprContext @Override public ResolvableExpression visitInNotInExpr(VtlParser.InNotInExprContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.left), visit(ctx.lists())); Positioned pos = fromContext(ctx); @@ -213,7 +213,7 @@ public ResolvableExpression visitLists(VtlParser.ListsContext ctx) { Set> types = listExpressions.stream().map(TypedExpression::getType) .collect(Collectors.toSet()); - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); if (types.size() > 1) { throw new VtlRuntimeException( @@ -228,7 +228,7 @@ public ResolvableExpression visitLists(VtlParser.ListsContext ctx) { // Since all expression are constant we don't need any context. List values = listExpressions .stream() - .map(expression -> expression.resolve(Map.of())) + .map(expression -> expression.resolve(Java8Helpers.mapOf())) .collect(Collectors.toList()); return ListExpression.withContainedType(values, type, pos); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConditionalVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConditionalVisitor.java index ed294a171..6cff51f86 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConditionalVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConditionalVisitor.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.functions.GenericFunctionsVisitor; import fr.insee.vtl.model.Positioned; import fr.insee.vtl.model.ResolvableExpression; @@ -8,7 +9,6 @@ import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -93,11 +93,11 @@ public static Boolean nvl(Boolean value, Boolean defaultValue) { @Override public ResolvableExpression visitIfExpr(VtlParser.IfExprContext ctx) { try { - var conditionalExpr = exprVisitor.visit(ctx.conditionalExpr); - var thenExpression = exprVisitor.visit(ctx.thenExpr); - var elseExpression = exprVisitor.visit(ctx.elseExpr); + ResolvableExpression conditionalExpr = exprVisitor.visit(ctx.conditionalExpr); + ResolvableExpression thenExpression = exprVisitor.visit(ctx.thenExpr); + ResolvableExpression elseExpression = exprVisitor.visit(ctx.elseExpr); Positioned position = fromContext(ctx); - ResolvableExpression expression = genericFunctionsVisitor.invokeFunction("ifThenElse", List.of(conditionalExpr, thenExpression, elseExpression), position); + ResolvableExpression expression = genericFunctionsVisitor.invokeFunction("ifThenElse", Java8Helpers.listOf(conditionalExpr, thenExpression, elseExpression), position); Class actualType = thenExpression.getType(); return new CastExpression(position, expression, actualType); } catch (VtlScriptException e) { @@ -118,7 +118,7 @@ public ResolvableExpression visitNvlAtom(VtlParser.NvlAtomContext ctx) { ResolvableExpression defaultExpression = exprVisitor.visit(ctx.right); Positioned position = fromContext(ctx); - return genericFunctionsVisitor.invokeFunction("nvl", List.of(expression, defaultExpression), position); + return genericFunctionsVisitor.invokeFunction("nvl", Java8Helpers.listOf(expression, defaultExpression), position); } catch (VtlScriptException e) { throw new VtlRuntimeException(e); } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConstantVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConstantVisitor.java index 91d5715f5..7a325ec9f 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConstantVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConstantVisitor.java @@ -31,7 +31,7 @@ public ConstantExpression visitConstant(VtlParser.ConstantContext ctx) { return new ConstantExpression(Boolean.parseBoolean(ctx.getText()), pos); } if (ctx.STRING_CONSTANT() != null) { - var text = ctx.getText(); + String text = ctx.getText(); return new ConstantExpression(text.substring(1, text.length() - 1), pos); } if (ctx.NULL_CONSTANT() != null) { diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ExpressionVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ExpressionVisitor.java index c994d7d6f..02976a614 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ExpressionVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ExpressionVisitor.java @@ -3,32 +3,17 @@ import fr.insee.vtl.engine.VtlScriptEngine; import fr.insee.vtl.engine.exceptions.UnimplementedException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.AnalyticsVisitor; import fr.insee.vtl.engine.visitors.ClauseVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.ComparisonFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.DistanceFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.GenericFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.JoinFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.NumericFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.SetFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.StringFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.TimeFunctionsVisitor; -import fr.insee.vtl.engine.visitors.expression.functions.ValidationFunctionsVisitor; -import fr.insee.vtl.model.Dataset; -import fr.insee.vtl.model.DatasetExpression; -import fr.insee.vtl.model.ProcessingEngine; -import fr.insee.vtl.model.ResolvableExpression; -import fr.insee.vtl.model.Structured; +import fr.insee.vtl.engine.visitors.expression.functions.*; +import fr.insee.vtl.model.*; import fr.insee.vtl.model.exceptions.InvalidTypeException; import fr.insee.vtl.model.exceptions.VtlScriptException; import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -359,10 +344,10 @@ public ResolvableExpression visitClauseExpr(VtlParser.ClauseExprContext ctx) { @Override public ResolvableExpression visitRatioToReportAn(VtlParser.RatioToReportAnContext ctx) { - var datasetExpression = (DatasetExpression) visit(ctx.expr()); - var dsName = ctx.expr().getText(); - var structure = datasetExpression.getDataStructure(); - var ids = structure.getIdentifiers(); + DatasetExpression datasetExpression = (DatasetExpression) visit(ctx.expr()); + String dsName = ctx.expr().getText(); + Structured.DataStructure structure = datasetExpression.getDataStructure(); + List ids = structure.getIdentifiers(); Map analytics = new LinkedHashMap<>(); for (Structured.Component measure : structure.getMeasures()) { @@ -376,19 +361,19 @@ public ResolvableExpression visitRatioToReportAn(VtlParser.RatioToReportAnContex // Filter out measure we don't need and rename the measure to the name of // the dataset. - var tempDs = processingEngine.executeProject(datasetExpression, + DatasetExpression tempDs = processingEngine.executeProject(datasetExpression, colNames); tempDs = processingEngine.executeRename(tempDs, - Map.of(measure.getName(), dsName)); + Java8Helpers.mapOf(measure.getName(), dsName)); // Execute the calc. String targetColumnName = ctx.op.getText() + "_" + measure.getName(); AnalyticsVisitor analyticsVisitor = new AnalyticsVisitor(processingEngine, tempDs, targetColumnName); - var result = analyticsVisitor.visit(ctx); + DatasetExpression result = analyticsVisitor.visit(ctx); // Rename back to the original name. - result = processingEngine.executeRename(result, Map.of(targetColumnName, measure.getName())); + result = processingEngine.executeRename(result, Java8Helpers.mapOf(targetColumnName, measure.getName())); // Drop the ds1 column. result = processingEngine.executeProject(result, @@ -402,10 +387,10 @@ public ResolvableExpression visitRatioToReportAn(VtlParser.RatioToReportAnContex @Override public ResolvableExpression visitLagOrLeadAn(VtlParser.LagOrLeadAnContext ctx) { - var datasetExpression = (DatasetExpression) visit(ctx.expr()); - var dsName = ctx.expr().getText(); - var structure = datasetExpression.getDataStructure(); - var ids = structure.getIdentifiers(); + DatasetExpression datasetExpression = (DatasetExpression) visit(ctx.expr()); + String dsName = ctx.expr().getText(); + Structured.DataStructure structure = datasetExpression.getDataStructure(); + List ids = structure.getIdentifiers(); Map analytics = new LinkedHashMap<>(); for (Structured.Component measure : structure.getMeasures()) { @@ -419,19 +404,19 @@ public ResolvableExpression visitLagOrLeadAn(VtlParser.LagOrLeadAnContext ctx) { // Filter out measure we don't need and rename the measure to the name of // the dataset. - var tempDs = processingEngine.executeProject(datasetExpression, + DatasetExpression tempDs = processingEngine.executeProject(datasetExpression, colNames); tempDs = processingEngine.executeRename(tempDs, - Map.of(measure.getName(), dsName)); + Java8Helpers.mapOf(measure.getName(), dsName)); // Execute the calc. String targetColumnName = ctx.op.getText() + "_" + measure.getName(); AnalyticsVisitor analyticsVisitor = new AnalyticsVisitor(processingEngine, tempDs, targetColumnName); - var result = analyticsVisitor.visit(ctx); + DatasetExpression result = analyticsVisitor.visit(ctx); // Rename back to the original name. - result = processingEngine.executeRename(result, Map.of(targetColumnName, measure.getName())); + result = processingEngine.executeRename(result, Java8Helpers.mapOf(targetColumnName, measure.getName())); // Drop the ds1 column. result = processingEngine.executeProject(result, @@ -445,10 +430,10 @@ public ResolvableExpression visitLagOrLeadAn(VtlParser.LagOrLeadAnContext ctx) { @Override public DatasetExpression visitAnSimpleFunction(VtlParser.AnSimpleFunctionContext ctx) { - var datasetExpression = (DatasetExpression) visit(ctx.expr()); - var dsName = ctx.expr().getText(); - var structure = datasetExpression.getDataStructure(); - var ids = structure.getIdentifiers(); + DatasetExpression datasetExpression = (DatasetExpression) visit(ctx.expr()); + String dsName = ctx.expr().getText(); + Structured.DataStructure structure = datasetExpression.getDataStructure(); + List ids = structure.getIdentifiers(); Map analytics = new LinkedHashMap<>(); for (Structured.Component measure : structure.getMeasures()) { @@ -462,19 +447,19 @@ public DatasetExpression visitAnSimpleFunction(VtlParser.AnSimpleFunctionContext // Filter out measure we don't need and rename the measure to the name of // the dataset. - var tempDs = processingEngine.executeProject(datasetExpression, + DatasetExpression tempDs = processingEngine.executeProject(datasetExpression, colNames); tempDs = processingEngine.executeRename(tempDs, - Map.of(measure.getName(), dsName)); + Java8Helpers.mapOf(measure.getName(), dsName)); // Execute the calc. String targetColumnName = ctx.op.getText() + "_" + measure.getName(); AnalyticsVisitor analyticsVisitor = new AnalyticsVisitor(processingEngine, tempDs, targetColumnName); - var result = analyticsVisitor.visit(ctx); + DatasetExpression result = analyticsVisitor.visit(ctx); // Rename back to the original name. - result = processingEngine.executeRename(result, Map.of(targetColumnName, measure.getName())); + result = processingEngine.executeRename(result, Java8Helpers.mapOf(targetColumnName, measure.getName())); // Drop the ds1 column. result = processingEngine.executeProject(result, diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/UnaryVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/UnaryVisitor.java index 42a286daa..436d9ebd5 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/UnaryVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/UnaryVisitor.java @@ -1,7 +1,9 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.functions.GenericFunctionsVisitor; +import fr.insee.vtl.model.Positioned; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; import fr.insee.vtl.parser.VtlBaseVisitor; @@ -68,8 +70,8 @@ public static Boolean not(Boolean right) { @Override public ResolvableExpression visitUnaryExpr(VtlParser.UnaryExprContext ctx) { try { - var pos = fromContext(ctx); - var parameters = List.of(exprVisitor.visit(ctx.right)); + Positioned pos = fromContext(ctx); + List parameters = Java8Helpers.listOf(exprVisitor.visit(ctx.right)); switch (ctx.op.getType()) { case VtlParser.PLUS: return genericFunctionsVisitor.invokeFunction("plus", parameters, pos); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/VarIdVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/VarIdVisitor.java index 1c4f74615..b8b6222f3 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/VarIdVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/VarIdVisitor.java @@ -3,11 +3,7 @@ import fr.insee.vtl.engine.exceptions.UndefinedVariableException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; import fr.insee.vtl.engine.expressions.ComponentExpression; -import fr.insee.vtl.model.ConstantExpression; -import fr.insee.vtl.model.Dataset; -import fr.insee.vtl.model.DatasetExpression; -import fr.insee.vtl.model.ResolvableExpression; -import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.*; import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; @@ -36,7 +32,7 @@ public VarIdVisitor(Map context) { @Override public ResolvableExpression visitVarID(VtlParser.VarIDContext ctx) { final String variableName = ctx.getText(); - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); if (!context.containsKey(variableName)) { throw new VtlRuntimeException(new UndefinedVariableException(variableName, pos)); @@ -48,7 +44,7 @@ public ResolvableExpression visitVarID(VtlParser.VarIDContext ctx) { } if (value instanceof Structured.Component) { - var component = (Structured.Component) value; + Structured.Component component = (Structured.Component) value; return new ComponentExpression(component, pos); } diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsVisitor.java index c55e15cfe..7045e232a 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsVisitor.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression.functions; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; @@ -69,7 +70,7 @@ public ComparisonFunctionsVisitor(ExpressionVisitor expressionVisitor, GenericFu @Override public ResolvableExpression visitBetweenAtom(VtlParser.BetweenAtomContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.op), exprVisitor.visit(ctx.from_), exprVisitor.visit(ctx.to_) @@ -89,7 +90,7 @@ public ResolvableExpression visitBetweenAtom(VtlParser.BetweenAtomContext ctx) { @Override public ResolvableExpression visitCharsetMatchAtom(VtlParser.CharsetMatchAtomContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.op), exprVisitor.visit(ctx.pattern) ); @@ -108,7 +109,7 @@ public ResolvableExpression visitCharsetMatchAtom(VtlParser.CharsetMatchAtomCont @Override public ResolvableExpression visitIsNullAtom(VtlParser.IsNullAtomContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.expr()) ); return genericFunctionsVisitor.invokeFunction("isNull", parameters, fromContext(ctx)); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsVisitor.java index f1e5d7546..e46574da1 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsVisitor.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression.functions; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; @@ -47,7 +48,7 @@ public static Long levenshtein(String stringA, String stringB) { @Override public ResolvableExpression visitLevenshteinAtom(VtlParser.LevenshteinAtomContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.left), exprVisitor.visit(ctx.right) ); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsVisitor.java index 237d913f6..f89b5bee0 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsVisitor.java @@ -7,14 +7,9 @@ import fr.insee.vtl.engine.expressions.CastExpression; import fr.insee.vtl.engine.expressions.ComponentExpression; import fr.insee.vtl.engine.expressions.FunctionExpression; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; -import fr.insee.vtl.model.Dataset; -import fr.insee.vtl.model.DatasetExpression; -import fr.insee.vtl.model.Positioned; -import fr.insee.vtl.model.ProcessingEngine; -import fr.insee.vtl.model.ResolvableExpression; -import fr.insee.vtl.model.Structured; -import fr.insee.vtl.model.TypedExpression; +import fr.insee.vtl.model.*; import fr.insee.vtl.model.exceptions.VtlScriptException; import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; @@ -24,12 +19,7 @@ import org.threeten.extra.PeriodDuration; import java.time.Instant; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -114,7 +104,7 @@ public ResolvableExpression invokeFunction(String funcName, List parameterTypes = parameters.stream().map(ResolvableExpression::getType).collect(Collectors.toList()); // Only one parameter, and it's a dataset. We can invoke the function on each measure. // Or method found in global methods - var method = engine.findGlobalMethod(funcName, parameterTypes); + VtlMethod method = engine.findGlobalMethod(funcName, parameterTypes); if (parameters.stream().noneMatch(DatasetExpression.class::isInstance) || method != null) { // Only scalar types. We can invoke the function directly. if (method == null) { @@ -144,7 +134,7 @@ public ResolvableExpression invokeFunction(String funcName, List measures = ((DatasetExpression) finalRes).getMeasures(); if (measures.size() == 1 && measures.get(0).getType().equals(Boolean.class)) { // TODO: refine with constraints matrix - return proc.executeRename((DatasetExpression) finalRes, Map.of(measures.get(0).getName(), "bool_var")); + return proc.executeRename((DatasetExpression) finalRes, Java8Helpers.mapOf(measures.get(0).getName(), "bool_var")); } } return finalRes; @@ -160,19 +150,19 @@ private DatasetExpression invokeFunctionOnDataset(String funcName, List monoExprs = new HashMap<>(); Set measureNames = new HashSet<>(); - var dsExprs = parameters.stream() + Map dsExprs = parameters.stream() .filter(DatasetExpression.class::isInstance) .map(e -> ((DatasetExpression) e)) .map(ds -> { if (Boolean.FALSE.equals(ds.isMonoMeasure())) { throw new VtlRuntimeException(new InvalidArgumentException("mono-measure dataset expected", ds)); } - var uniqueName = "arg" + ds.hashCode(); - var measure = ds.getMeasures().get(0); + String uniqueName = "arg" + ds.hashCode(); + Structured.Component measure = ds.getMeasures().get(0); String measureName = measure.getName(); measureNames.add(measureName); - ds = proc.executeRename(ds, Map.of(measureName, uniqueName)); - var renamedComponent = new Structured.Component(uniqueName, measure.getType(), measure.getRole(), measure.getNullable()); + ds = proc.executeRename(ds, Java8Helpers.mapOf(measureName, uniqueName)); + Structured.Component renamedComponent = new Structured.Component(uniqueName, measure.getType(), measure.getRole(), measure.getNullable()); monoExprs.put(uniqueName, new ComponentExpression(renamedComponent, ds)); return ds; }) @@ -185,7 +175,7 @@ private DatasetExpression invokeFunctionOnDataset(String funcName, List normalizedParams = parameters.stream() .map(e -> monoExprs.getOrDefault("arg" + e.hashCode(), e)) .collect(Collectors.toList()); @@ -193,11 +183,11 @@ private DatasetExpression invokeFunctionOnDataset(String funcName, List parametersTypes = normalizedParams.stream() .map(TypedExpression::getType) .collect(Collectors.toList()); - var method = engine.findMethod(funcName, parametersTypes); - var funcExrp = new FunctionExpression(method, normalizedParams, position); - ds = proc.executeCalc(ds, Map.of(result, funcExrp), Map.of(result, Dataset.Role.MEASURE), Map.of()); + VtlMethod method = engine.findMethod(funcName, parametersTypes); + FunctionExpression funcExrp = new FunctionExpression(method, normalizedParams, position); + ds = proc.executeCalc(ds, Java8Helpers.mapOf(result, funcExrp), Java8Helpers.mapOf(result, Dataset.Role.MEASURE), Java8Helpers.mapOf()); ds = proc.executeProject(ds, Stream.concat(ds.getIdentifiers().stream().map(Structured.Component::getName), Stream.of(result)).collect(Collectors.toList())); - return proc.executeRename(ds, Map.of(result, measureNames.iterator().next())); + return proc.executeRename(ds, Java8Helpers.mapOf(result, measureNames.iterator().next())); } @Override diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsVisitor.java index e0672aa5e..8d6c7df9b 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsVisitor.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.InvalidArgumentException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.DatasetExpression; @@ -41,8 +42,8 @@ public JoinFunctionsVisitor(ExpressionVisitor expressionVisitor, ProcessingEngin public static Optional> checkSameIdentifiers(Collection datasetExpressions) { Set> identifiers = new LinkedHashSet<>(); for (DatasetExpression datasetExpression : datasetExpressions) { - var structure = datasetExpression.getDataStructure(); - var ids = new LinkedHashSet(); + Structured.DataStructure structure = datasetExpression.getDataStructure(); + HashSet ids = new LinkedHashSet(); for (Component component : structure.values()) { if (component.getRole().equals(Role.IDENTIFIER)) { ids.add(component); @@ -75,15 +76,15 @@ private LinkedHashMap normalizeDatasets(List datasets = new LinkedHashMap<>(); List measures = new ArrayList<>(); for (VtlParser.JoinClauseItemContext joinClauseItem : joinClauseItems) { - var datasetExpressionContext = joinClauseItem.expr(); - var alias = joinClauseItem.alias() != null ? joinClauseItem.alias().IDENTIFIER().getText() : null; + VtlParser.ExprContext datasetExpressionContext = joinClauseItem.expr(); + String alias = joinClauseItem.alias() != null ? joinClauseItem.alias().IDENTIFIER().getText() : null; if (alias == null && !(datasetExpressionContext instanceof VtlParser.VarIdExprContext)) { throw new VtlRuntimeException( new InvalidArgumentException("cannot use expression without alias in join clause", fromContext(datasetExpressionContext)) ); } - var datasetExpression = (DatasetExpression) assertTypeExpression( + DatasetExpression datasetExpression = (DatasetExpression) assertTypeExpression( expressionVisitor.visit(datasetExpressionContext), Dataset.class, datasetExpressionContext); List dsMeasures = datasetExpression.getDataStructure().values().stream() @@ -131,8 +132,8 @@ private Map renameDuplicates(List identifi // Use duplicates to rename columns Map result = new LinkedHashMap<>(); for (Map.Entry entry : datasets.entrySet()) { - var name = entry.getKey(); - var dataset = entry.getValue(); + String name = entry.getKey(); + DatasetExpression dataset = entry.getValue(); Map fromTo = new LinkedHashMap<>(); for (String columnName : dataset.getColumnNames()) { if (duplicates.contains(columnName)) { @@ -160,13 +161,13 @@ private DatasetExpression removeComponentAlias(DatasetExpression dataset) { }); Map fromTo = toFrom.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); - var renamed = processingEngine.executeRename(dataset, fromTo); + DatasetExpression renamed = processingEngine.executeRename(dataset, fromTo); return processingEngine.executeProject(renamed, new ArrayList<>(toKeep)); } private DatasetExpression leftJoin(VtlParser.JoinExprContext ctx) { - var joinClauseContext = ctx.joinClause(); - var datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); + VtlParser.JoinClauseContext joinClauseContext = ctx.joinClause(); + LinkedHashMap datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); List ids = getInnerAndLeftJoinIdentifiers(joinClauseContext, datasets); DatasetExpression res = processingEngine.executeLeftJoin(renameDuplicates(ids, datasets), ids); @@ -174,10 +175,10 @@ private DatasetExpression leftJoin(VtlParser.JoinExprContext ctx) { } private DatasetExpression crossJoin(VtlParser.JoinExprContext ctx) { - var joinClauseContext = ctx.joinClauseWithoutUsing(); - var datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); + VtlParser.JoinClauseWithoutUsingContext joinClauseContext = ctx.joinClauseWithoutUsing(); + HashMap datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); - Map renamedDatasets = renameDuplicates(List.of(), datasets); + Map renamedDatasets = renameDuplicates(Java8Helpers.listOf(), datasets); List identifiers = renamedDatasets.values().stream() .flatMap(dsExpr -> dsExpr.getDataStructure().values().stream()) @@ -188,11 +189,11 @@ private DatasetExpression crossJoin(VtlParser.JoinExprContext ctx) { } private DatasetExpression fullJoin(VtlParser.JoinExprContext ctx) { - var joinClauseContext = ctx.joinClauseWithoutUsing(); - var datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); + VtlParser.JoinClauseWithoutUsingContext joinClauseContext = ctx.joinClauseWithoutUsing(); + HashMap datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); // Full join require that all the datasets have one or more common identifiers. - var commonIdentifiers = checkSameIdentifiers(datasets.values()) + List commonIdentifiers = checkSameIdentifiers(datasets.values()) .orElseThrow(() -> new VtlRuntimeException( new InvalidArgumentException(mustHaveCommonIdentifiers, fromContext(joinClauseContext)) )); @@ -202,8 +203,8 @@ private DatasetExpression fullJoin(VtlParser.JoinExprContext ctx) { } private DatasetExpression innerJoin(VtlParser.JoinExprContext ctx) { - var joinClauseContext = ctx.joinClause(); - var datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); + VtlParser.JoinClauseContext joinClauseContext = ctx.joinClause(); + LinkedHashMap datasets = normalizeDatasets(joinClauseContext.joinClauseItem()); List ids = getInnerAndLeftJoinIdentifiers(joinClauseContext, datasets); DatasetExpression res = processingEngine.executeInnerJoin(renameDuplicates(ids, datasets), ids); return removeComponentAlias(res); @@ -226,7 +227,7 @@ private List getInnerAndLeftJoinIdentifiers( // Remove the identifiers if (joinClauseContext.USING() != null) { for (VtlParser.ComponentIDContext usingContext : joinClauseContext.componentID()) { - var name = usingContext.getText(); + String name = usingContext.getText(); for (DatasetExpression datasetExpression : datasets.values()) { List names = datasetExpression.getColumnNames(); if (!names.contains(name)) { diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsVisitor.java index 7f016b1eb..8679080be 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsVisitor.java @@ -1,7 +1,9 @@ package fr.insee.vtl.engine.visitors.expression.functions; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; +import fr.insee.vtl.model.Positioned; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; import fr.insee.vtl.parser.VtlBaseVisitor; @@ -143,7 +145,7 @@ public static Double log(Number operand, Number base) { public ResolvableExpression visitUnaryNumeric(VtlParser.UnaryNumericContext ctx) { try { VtlParser.ExprContext expr = ctx.expr(); - List parameter = List.of(exprVisitor.visit(expr)); + List parameter = Java8Helpers.listOf(exprVisitor.visit(expr)); switch (ctx.op.getType()) { case VtlParser.CEIL: return genericFunctionsVisitor.invokeFunction("ceil", parameter, fromContext(ctx)); @@ -174,8 +176,8 @@ public ResolvableExpression visitUnaryNumeric(VtlParser.UnaryNumericContext ctx) @Override public ResolvableExpression visitUnaryWithOptionalNumeric(VtlParser.UnaryWithOptionalNumericContext ctx) { try { - var pos = fromContext(ctx); - List parameters = List.of( + Positioned pos = fromContext(ctx); + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.expr()), ctx.optionalExpr() == null ? ResolvableExpression.withType(Long.class).withPosition(pos).using(c -> 0L) : @@ -202,7 +204,7 @@ public ResolvableExpression visitUnaryWithOptionalNumeric(VtlParser.UnaryWithOpt @Override public ResolvableExpression visitBinaryNumeric(VtlParser.BinaryNumericContext ctx) { try { - List parameters = List.of( + List parameters = Java8Helpers.listOf( exprVisitor.visit(ctx.left), exprVisitor.visit(ctx.right) ); diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitor.java index a9a7ea68f..018d356d4 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitor.java @@ -3,11 +3,7 @@ import fr.insee.vtl.engine.exceptions.InvalidArgumentException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; -import fr.insee.vtl.model.Dataset; -import fr.insee.vtl.model.DatasetExpression; -import fr.insee.vtl.model.ProcessingEngine; -import fr.insee.vtl.model.ResolvableExpression; -import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.*; import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; import org.antlr.v4.runtime.RuleContext; diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsVisitor.java index 0118d0787..4c30c014a 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsVisitor.java @@ -1,7 +1,9 @@ package fr.insee.vtl.engine.visitors.expression.functions; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; +import fr.insee.vtl.model.Positioned; import fr.insee.vtl.model.ResolvableExpression; import fr.insee.vtl.model.exceptions.VtlScriptException; import fr.insee.vtl.parser.VtlBaseVisitor; @@ -94,7 +96,7 @@ public static String substr(String value, Long start, Long len) { start = start - 1; } - var end = start + len; + Long end = start + len; if (end > value.length()) { return value.substring(Math.toIntExact(start)); } @@ -133,8 +135,8 @@ public static Long instr(String v, String v2, Long start, Long occurence) { @Override public ResolvableExpression visitUnaryStringFunction(VtlParser.UnaryStringFunctionContext ctx) { try { - var pos = fromContext(ctx); - var parameters = List.of(exprVisitor.visit(ctx.expr())); + Positioned pos = fromContext(ctx); + List parameters = Java8Helpers.listOf(exprVisitor.visit(ctx.expr())); switch (ctx.op.getType()) { case VtlParser.TRIM: return genericFunctionsVisitor.invokeFunction("trim", parameters, pos); @@ -166,7 +168,7 @@ public ResolvableExpression visitUnaryStringFunction(VtlParser.UnaryStringFuncti @Override public ResolvableExpression visitSubstrAtom(VtlParser.SubstrAtomContext ctx) { try { - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); ResolvableExpression expr = ctx.expr() == null ? ResolvableExpression.withType(String.class) .withPosition(pos) @@ -179,7 +181,7 @@ public ResolvableExpression visitSubstrAtom(VtlParser.SubstrAtomContext ctx) { ResolvableExpression.withType(Long.class) .withPosition(pos) .using(c -> null) : exprVisitor.visit(ctx.endParameter); - List parameters = List.of(expr, start, len); + List parameters = Java8Helpers.listOf(expr, start, len); return genericFunctionsVisitor.invokeFunction("substr", parameters, fromContext(ctx)); } catch (VtlScriptException e) { throw new VtlRuntimeException(e); @@ -195,7 +197,7 @@ public ResolvableExpression visitSubstrAtom(VtlParser.SubstrAtomContext ctx) { @Override public ResolvableExpression visitReplaceAtom(VtlParser.ReplaceAtomContext ctx) { try { - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); ResolvableExpression expr = ctx.expr(0) == null ? ResolvableExpression.withType(String.class) .withPosition(pos) @@ -208,7 +210,7 @@ public ResolvableExpression visitReplaceAtom(VtlParser.ReplaceAtomContext ctx) { ResolvableExpression.withType(String.class) .withPosition(pos) .using(c -> null) : exprVisitor.visit(ctx.optionalExpr()); - List parameters = List.of(expr, param, optionalExpr); + List parameters = Java8Helpers.listOf(expr, param, optionalExpr); return genericFunctionsVisitor.invokeFunction("replace", parameters, fromContext(ctx)); } catch (VtlScriptException e) { @@ -225,7 +227,7 @@ public ResolvableExpression visitReplaceAtom(VtlParser.ReplaceAtomContext ctx) { @Override public ResolvableExpression visitInstrAtom(VtlParser.InstrAtomContext ctx) { try { - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); ResolvableExpression expr = ctx.expr(0) == null ? ResolvableExpression.withType(String.class) .withPosition(pos) @@ -242,7 +244,7 @@ public ResolvableExpression visitInstrAtom(VtlParser.InstrAtomContext ctx) { ResolvableExpression.withType(Long.class) .withPosition(pos) .using(c -> null) : exprVisitor.visit(ctx.occurrenceParameter); - List parameters = List.of(expr, pattern, start, occurence); + List parameters = Java8Helpers.listOf(expr, pattern, start, occurence); return genericFunctionsVisitor.invokeFunction("instr", parameters, fromContext(ctx)); } catch (VtlScriptException e) { diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsVisitor.java index 76f0d7641..c7c157309 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsVisitor.java @@ -6,6 +6,7 @@ import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.*; import fr.insee.vtl.model.exceptions.VtlScriptException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; import org.antlr.v4.runtime.tree.ParseTree; @@ -16,7 +17,6 @@ import java.time.ZonedDateTime; import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -73,7 +73,7 @@ private ResolvableExpression stockToFlows(VtlParser.FlowAtomContext ctx) { DatasetExpression ds = (DatasetExpression) operand; - var ids = ds.getIdentifiers().stream() + LinkedHashMap ids = ds.getIdentifiers().stream() .collect(Collectors.toMap( Structured.Component::getName, c -> Analytics.Order.ASC, @@ -81,8 +81,8 @@ private ResolvableExpression stockToFlows(VtlParser.FlowAtomContext ctx) { LinkedHashMap::new )); - var time = extractTimeComponent(ctx, ds); - var partition = ids.keySet().stream() + Structured.Component time = extractTimeComponent(ctx, ds); + List partition = ids.keySet().stream() .filter(colName -> !time.getName().equals(colName)) .collect(Collectors.toList()); for (Structured.Component measure : ds.getMeasures()) { @@ -91,28 +91,28 @@ private ResolvableExpression stockToFlows(VtlParser.FlowAtomContext ctx) { continue; } - var measureName = measure.getName(); - var lagColumnName = measure.getName() + "_lag"; + String measureName = measure.getName(); + String lagColumnName = measure.getName() + "_lag"; - var lag = processingEngine.executeLeadOrLagAn(ds, measureName, Analytics.Function.LAG, measureName, 1, partition, ids); - lag = processingEngine.executeRename(lag, Map.of(measureName, lagColumnName)); + DatasetExpression lag = processingEngine.executeLeadOrLagAn(ds, measureName, Analytics.Function.LAG, measureName, 1, partition, ids); + lag = processingEngine.executeRename(lag, Java8Helpers.mapOf(measureName, lagColumnName)); lag = processingEngine.executeProject(lag, Stream.concat(ds.getIdentifiers().stream().map(Structured.Component::getName), Stream.of(lagColumnName)).collect(Collectors.toList()) ); - ds = processingEngine.executeLeftJoin(Map.of("left", ds, "lag", lag), ds.getIdentifiers()); + ds = processingEngine.executeLeftJoin(Java8Helpers.mapOf("left", ds, "lag", lag), ds.getIdentifiers()); // me - nvl(lag, 0) - var measureExpr = new ComponentExpression(ds.getDataStructure().get(measure.getName()), position); - var lagExpr = new ComponentExpression(ds.getDataStructure().get(lagColumnName), position); - var nvlExpr = genericFunctionsVisitor.invokeFunction("nvl", List.of( + ComponentExpression measureExpr = new ComponentExpression(ds.getDataStructure().get(measure.getName()), position); + ComponentExpression lagExpr = new ComponentExpression(ds.getDataStructure().get(lagColumnName), position); + ResolvableExpression nvlExpr = genericFunctionsVisitor.invokeFunction("nvl", Java8Helpers.listOf( lagExpr, new ConstantExpression(0L, position)), position); - var subtractionExpr = genericFunctionsVisitor.invokeFunction("subtraction", List.of( + ResolvableExpression subtractionExpr = genericFunctionsVisitor.invokeFunction("subtraction", Java8Helpers.listOf( measureExpr, nvlExpr ), position); - ds = processingEngine.executeCalc(ds, Map.of(measure.getName(), subtractionExpr), Map.of(), Map.of()); + ds = processingEngine.executeCalc(ds, Java8Helpers.mapOf(measure.getName(), subtractionExpr), Java8Helpers.mapOf(), Java8Helpers.mapOf()); ds = processingEngine.executeProject(ds, ds.getColumnNames().stream().filter(s -> !s.equals(lagColumnName)).collect(Collectors.toList())); } return ds; @@ -135,7 +135,7 @@ private ResolvableExpression flowToStock(VtlParser.FlowAtomContext ctx) { } DatasetExpression ds = (DatasetExpression) operand; - var ids = ds.getIdentifiers().stream() + LinkedHashMap ids = ds.getIdentifiers().stream() .collect(Collectors.toMap( Structured.Component::getName, c -> Analytics.Order.ASC, @@ -143,8 +143,8 @@ private ResolvableExpression flowToStock(VtlParser.FlowAtomContext ctx) { LinkedHashMap::new )); - var time = extractTimeComponent(ctx, ds); - var partition = ids.keySet().stream() + Structured.Component time = extractTimeComponent(ctx, ds); + List partition = ids.keySet().stream() .filter(colName -> !time.getName().equals(colName)) .collect(Collectors.toList()); for (Structured.Component measure : ds.getMeasures()) { @@ -167,7 +167,7 @@ public ResolvableExpression visitTimeShiftAtom(VtlParser.TimeShiftAtomContext ct // Fall through if not dataset. if (!(operand instanceof DatasetExpression)) { - return genericFunctionsVisitor.invokeFunction("timeshift", List.of( + return genericFunctionsVisitor.invokeFunction("timeshift", Java8Helpers.listOf( operand, n ), fromContext(ctx)); @@ -175,13 +175,13 @@ public ResolvableExpression visitTimeShiftAtom(VtlParser.TimeShiftAtomContext ct DatasetExpression ds = (DatasetExpression) operand; - var t = extractTimeComponent(ctx, ds); + Structured.Component t = extractTimeComponent(ctx, ds); - var compExpr = genericFunctionsVisitor.invokeFunction("timeshift", List.of( + ResolvableExpression compExpr = genericFunctionsVisitor.invokeFunction("timeshift", Java8Helpers.listOf( new ComponentExpression(t, fromContext(ctx)), n ), fromContext(ctx)); - return processingEngine.executeCalc(ds, Map.of(t.getName(), compExpr), Map.of(t.getName(), t.getRole()), Map.of()); + return processingEngine.executeCalc(ds, Java8Helpers.mapOf(t.getName(), compExpr), Java8Helpers.mapOf(t.getName(), t.getRole()), Java8Helpers.mapOf()); } catch (VtlScriptException e) { throw new VtlRuntimeException(e); @@ -189,7 +189,7 @@ public ResolvableExpression visitTimeShiftAtom(VtlParser.TimeShiftAtomContext ct } private static Structured.Component extractTimeComponent(ParseTree ctx, DatasetExpression ds) throws InvalidArgumentException { - var t = ds.getIdentifiers().stream() + Structured.Component t = ds.getIdentifiers().stream() .filter(component -> component.getType().equals(Interval.class) || component.getType().equals(Instant.class) || component.getType().equals(ZonedDateTime.class) diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsVisitor.java index d648f21fd..13cc26d5c 100644 --- a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsVisitor.java +++ b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsVisitor.java @@ -4,6 +4,7 @@ import fr.insee.vtl.engine.exceptions.InvalidArgumentException; import fr.insee.vtl.engine.exceptions.UndefinedVariableException; import fr.insee.vtl.engine.exceptions.VtlRuntimeException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor; import fr.insee.vtl.model.*; import fr.insee.vtl.parser.VtlBaseVisitor; @@ -95,7 +96,7 @@ public ResolvableExpression visitValidateDPruleset(VtlParser.ValidateDPrulesetCo }); // Temp create vd column - ds = processingEngine.executeCalc(ds, colsToAdd, Map.of(), Map.of()); + ds = processingEngine.executeCalc(ds, colsToAdd, Java8Helpers.mapOf(), Java8Helpers.mapOf()); // check if dpr variables are in ds structure @@ -122,7 +123,7 @@ public ResolvableExpression visitValidateDPruleset(VtlParser.ValidateDPrulesetCo } ); - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); return processingEngine.executeValidateDPruleset(dpr, ds, output, pos, valuedomaines); } @@ -135,7 +136,7 @@ public ResolvableExpression visitValidateDPruleset(VtlParser.ValidateDPrulesetCo */ @Override public ResolvableExpression visitValidationSimple(VtlParser.ValidationSimpleContext ctx) { - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); DatasetExpression dsExpression = (DatasetExpression) assertTypeExpression(expressionVisitor.visit(ctx.expr()), Dataset.class, ctx.expr()); List exprMeasures = dsExpression.getDataStructure().values().stream() @@ -176,7 +177,7 @@ public ResolvableExpression visitValidationSimple(VtlParser.ValidationSimpleCont // TODO: handle other IDs than componentID? build unique ID tuples to calculate @Override public ResolvableExpression visitValidateHRruleset(VtlParser.ValidateHRrulesetContext ctx) { - var pos = fromContext(ctx); + Positioned pos = fromContext(ctx); DatasetExpression dsExpression = (DatasetExpression) assertTypeExpression(expressionVisitor.visit(ctx.expr()), Dataset.class, ctx.expr()); String datasetName = ctx.expr().getText(); diff --git a/vtl-engine/src/main/java/module-info.java b/vtl-engine/src/main/java/module-info.java deleted file mode 100644 index a960ab585..000000000 --- a/vtl-engine/src/main/java/module-info.java +++ /dev/null @@ -1,42 +0,0 @@ -import fr.insee.vtl.engine.VtlScriptEngineFactory; -import fr.insee.vtl.engine.functions.LevenshteinProvider; -import fr.insee.vtl.engine.processors.InMemoryProcessingEngine; -import fr.insee.vtl.model.FunctionProvider; -import fr.insee.vtl.model.ProcessingEngine; -import fr.insee.vtl.model.ProcessingEngineFactory; - -import javax.script.ScriptEngineFactory; - -/** - * This module contains the actual VTL engine. - */ -module fr.insee.vtl.engine { - exports fr.insee.vtl.engine.exceptions; - - requires transitive java.scripting; - requires transitive fr.insee.vtl.parser; - requires transitive fr.insee.vtl.model; - - uses ProcessingEngine; - uses ProcessingEngineFactory; - uses FunctionProvider; - - // exports fr.insee.vtl.engine.functions; - provides FunctionProvider with LevenshteinProvider; - - exports fr.insee.vtl.engine.processors; - provides ProcessingEngineFactory with InMemoryProcessingEngine.Factory; - - opens fr.insee.vtl.engine; - - requires org.antlr.antlr4.runtime; - - // TODO: Consider removing these. - requires org.apache.commons.lang3; - requires org.apache.commons.text; - requires safety.mirror; - requires org.threeten.extra; - - provides ScriptEngineFactory with VtlScriptEngineFactory; - -} diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/TemporalFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/TemporalFunctionsTest.java index 37b60b0b5..15fc7e014 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/TemporalFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/TemporalFunctionsTest.java @@ -4,6 +4,7 @@ import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; import fr.insee.vtl.model.exceptions.VtlScriptException; +import fr.insee.vtl.model.utils.Java8Helpers; import org.assertj.core.util.Lists; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -15,14 +16,14 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; - import java.time.*; import java.time.temporal.Temporal; import java.util.*; import static fr.insee.vtl.model.Dataset.*; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; class TemporalFunctionsTest { @@ -35,14 +36,14 @@ public void setUp() { @Test public void testAddition() throws ScriptException { - var ts = Lists.list( + List ts = Lists.list( Instant.parse("2023-03-26T00:00:00Z"), ZonedDateTime.parse("2023-03-26T00:00:00+01:00[Europe/Paris]"), // Day before DST starts ZonedDateTime.parse("2023-10-29T00:00:00+02:00[Europe/Paris]"), // Day before DST ends OffsetDateTime.parse("2023-03-26T00:00:00+01:00"), // OffsetDateTime for comparison OffsetDateTime.parse("2023-10-29T00:00:00+02:00") // OffsetDateTime for comparison ); - var d = PeriodDuration.parse("P1DT1H"); + PeriodDuration d = PeriodDuration.parse("P1DT1H"); engine.put("d", d); for (Temporal t : ts) { engine.put("t", t); @@ -58,13 +59,13 @@ public void testAddition() throws ScriptException { @Test public void testSubtraction() throws ScriptException { // Including different temporal types and specific dates for DST testing - var as = List.of( + List as = Java8Helpers.listOf( Instant.parse("2023-04-15T00:00:00Z"), // Standard Instant ZonedDateTime.parse("2023-03-26T00:00:00+01:00[Europe/Paris]"), // Day before DST starts OffsetDateTime.parse("2023-04-15T00:00:00+01:00"), // Standard OffsetDateTime ZonedDateTime.parse("2023-10-29T00:00:00+02:00[Europe/Paris]") // Day before DST ends ); - var bs = List.of( + List bs = Java8Helpers.listOf( Instant.parse("2023-04-14T23:00:00Z"), // 1 hour before the Instant in 'as' ZonedDateTime.parse("2023-03-26T03:00:00+02:00[Europe/Paris]"), // 3 hours after DST starts, same day OffsetDateTime.parse("2023-04-15T01:00:00+01:00"), // 1 hour after the OffsetDateTime in 'as' @@ -83,7 +84,7 @@ public void testSubtraction() throws ScriptException { @Test public void testMultiplication() throws ScriptException { - var ds = List.of( + List ds = Java8Helpers.listOf( PeriodDuration.parse("P1Y"), PeriodDuration.parse("PT1H"), PeriodDuration.parse("P1YT1H") @@ -134,15 +135,15 @@ public void periodIndicator() throws ScriptException { @Test public void testTimeshift() throws ScriptException { Dataset ds1 = new InMemoryDataset( - new DataStructure(List.of( + new DataStructure(Java8Helpers.listOf( new Component("id", String.class, Role.IDENTIFIER), new Component("time", Interval.class, Role.IDENTIFIER), new Component("measure", Long.class, Role.MEASURE) )), - List.of("a", Interval.parse("2010-01-01T00:00:00Z/P1Y"), 1L), - List.of("b", Interval.parse("2011-01-01T00:00:00Z/P1Y"), 2L), - List.of("c", Interval.parse("2012-01-01T00:00:00Z/P1Y"), 4L), - List.of("d", Interval.parse("2013-01-01T00:00:00Z/P1Y"), 8L) + Java8Helpers.listOf("a", Interval.parse("2010-01-01T00:00:00Z/P1Y"), 1L), + Java8Helpers.listOf("b", Interval.parse("2011-01-01T00:00:00Z/P1Y"), 2L), + Java8Helpers.listOf("c", Interval.parse("2012-01-01T00:00:00Z/P1Y"), 4L), + Java8Helpers.listOf("d", Interval.parse("2013-01-01T00:00:00Z/P1Y"), 8L) ); engine.put("ds1", ds1); @@ -150,10 +151,10 @@ public void testTimeshift() throws ScriptException { Object d1 = engine.get("d1"); assertThat(d1).isInstanceOf(Dataset.class); assertThat(((Dataset) d1).getDataAsMap()).containsExactly( - Map.of("id", "a", "measure", 1L, "time", Interval.parse("2009-01-01T00:00:00Z/P1Y")), - Map.of("id", "b", "measure", 2L, "time", Interval.parse("2010-01-01T00:00:00Z/P1Y")), - Map.of("id", "c", "measure", 4L, "time", Interval.parse("2011-01-01T00:00:00Z/P1Y")), - Map.of("id", "d", "measure", 8L, "time", Interval.parse("2012-01-01T00:00:00Z/P1Y")) + Java8Helpers.mapOf("id", "a", "measure", 1L, "time", Interval.parse("2009-01-01T00:00:00Z/P1Y")), + Java8Helpers.mapOf("id", "b", "measure", 2L, "time", Interval.parse("2010-01-01T00:00:00Z/P1Y")), + Java8Helpers.mapOf("id", "c", "measure", 4L, "time", Interval.parse("2011-01-01T00:00:00Z/P1Y")), + Java8Helpers.mapOf("id", "d", "measure", 8L, "time", Interval.parse("2012-01-01T00:00:00Z/P1Y")) ); engine.put("t", Interval.parse("2010-01-01T00:00:00Z/P1Y")); @@ -166,19 +167,19 @@ public void testTimeshift() throws ScriptException { @Test public void testTruncate() throws ScriptException { - var ts = List.of( + List ts = Java8Helpers.listOf( Instant.parse("2023-04-15T01:30:30.1234Z"), // Standard Instant ZonedDateTime.parse("2023-03-26T01:30:30.1234+01:00[Europe/Paris]"), // Day before DST starts OffsetDateTime.parse("2023-04-15T01:30:30.1234+01:30"), // Standard OffsetDateTime ZonedDateTime.parse("2023-10-29T01:30:30.1234+02:00[Europe/Paris]") // Day before DST ends ); - var us = List.of( + List us = Java8Helpers.listOf( "day", "month", "year", "hour", "minute", "second" ); - var r = new ArrayList(); - var rr = new LinkedHashMap>(); + List r = new ArrayList<>(); + LinkedHashMap> rr = new LinkedHashMap<>(); for (Temporal t : ts) { for (String u : us) { engine.put("t", t); @@ -233,7 +234,7 @@ public void testTruncate() throws ScriptException { @Test public void testAtZone() throws ScriptException { - Map testCases = Map.of( + Map testCases = Java8Helpers.mapOf( "Europe/London", ZonedDateTime.ofInstant(Instant.parse("2020-04-04T10:15:30.00Z"), ZoneId.of("Europe/London")), "Asia/Kolkata", ZonedDateTime.ofInstant(Instant.parse("2020-04-04T10:15:30.00Z"), ZoneId.of("Asia/Kolkata")), "America/New_York", ZonedDateTime.ofInstant(Instant.parse("2020-04-04T10:15:30.00Z"), ZoneId.of("America/New_York")) @@ -243,10 +244,10 @@ public void testAtZone() throws ScriptException { engine.put("t", testInstant); for (Map.Entry testCase : testCases.entrySet()) { - var zone = testCase.getKey(); - var expected = testCase.getValue(); + String zone = testCase.getKey(); + ZonedDateTime expected = testCase.getValue(); engine.eval("r := at_zone(t, \"" + zone + "\");"); - var actual = engine.get("r"); + Object actual = engine.get("r"); assertEquals(expected, actual, "Failed for zone: " + zone); } @@ -265,7 +266,7 @@ void testTimeAggregation() throws ScriptException { // This test is an attempt to implement the time aggregate. The state of the group all // prevents us to finish it. See https://github.com/sdmx-twg/vtl/issues/456 - var ds1 = new InMemoryDataset(List.of( + InMemoryDataset ds1 = new InMemoryDataset(Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("t", Interval.class, Dataset.Role.IDENTIFIER), new Structured.Component("me1", Long.class, Dataset.Role.MEASURE) @@ -286,13 +287,13 @@ void testTimeAggregation() throws ScriptException { // Test with own function. engine.eval("res := ds1[aggr test := sum(me1) group all truncate_time(t, \"year\", \"Europe/Oslo\")];"); - var actual = (Dataset) engine.get("res"); + Dataset actual = (Dataset) engine.get("res"); actual.getDataAsMap().forEach(System.out::println); assertThat(actual.getDataAsMap()).containsExactly( - Map.of("time", Interval.parse("2009-12-31T23:00:00Z/2011-01-01T04:49:12Z"), "test", 70L), - Map.of("time", Interval.parse("2012-12-31T23:00:00Z/2014-01-01T04:49:12Z"), "test", 50L), - Map.of("time", Interval.parse("2010-12-31T23:00:00Z/2012-01-01T04:49:12Z"), "test", 30L), - Map.of("time", Interval.parse("2011-12-31T23:00:00Z/2012-12-31T04:49:12Z"), "test", 30L) + Java8Helpers.mapOf("time", Interval.parse("2009-12-31T23:00:00Z/2011-01-01T04:49:12Z"), "test", 70L), + Java8Helpers.mapOf("time", Interval.parse("2012-12-31T23:00:00Z/2014-01-01T04:49:12Z"), "test", 50L), + Java8Helpers.mapOf("time", Interval.parse("2010-12-31T23:00:00Z/2012-01-01T04:49:12Z"), "test", 30L), + Java8Helpers.mapOf("time", Interval.parse("2011-12-31T23:00:00Z/2012-12-31T04:49:12Z"), "test", 30L) ); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/TextFunctions.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/TextFunctions.java index 14ee08184..6b1c4aad3 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/TextFunctions.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/TextFunctions.java @@ -1,11 +1,11 @@ package fr.insee.vtl.engine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; import java.util.Arrays; -import java.util.List; public class TextFunctions { public static String testTrim(String str) { @@ -18,7 +18,7 @@ public static String testUpper(String str) { public static Dataset loadS3() { return new InMemoryDataset( - List.of( + Java8Helpers.listOf( new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("age", Long.class, Dataset.Role.MEASURE), new Structured.Component("weight", Long.class, Dataset.Role.MEASURE) diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineFactoryTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineFactoryTest.java index 656c481c0..3d97573ee 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineFactoryTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineFactoryTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.engine; +import fr.insee.vtl.model.utils.Java8Helpers; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -58,17 +59,17 @@ public String getEngineVersion() { @Override public List getExtensions() { - return List.of("vtl"); + return Java8Helpers.listOf("vtl"); } @Override public List getMimeTypes() { - return List.of("text/x-vtl"); + return Java8Helpers.listOf("text/x-vtl"); } @Override public List getNames() { - return List.of("VTLJava"); + return Java8Helpers.listOf("VTLJava"); } @Override diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java index ec515272d..5dbf0dee9 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/VtlScriptEngineTest.java @@ -2,10 +2,11 @@ import com.github.hervian.reflection.Fun; import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; -import fr.insee.vtl.engine.exceptions.InvalidArgumentException; import fr.insee.vtl.engine.exceptions.UndefinedVariableException; import fr.insee.vtl.engine.exceptions.VtlSyntaxException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.*; +import fr.insee.vtl.model.Positioned.Position; import fr.insee.vtl.model.exceptions.InvalidTypeException; import fr.insee.vtl.model.exceptions.VtlScriptException; import org.assertj.core.api.Condition; @@ -17,9 +18,8 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; +import java.lang.reflect.Method; import java.util.Arrays; -import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -35,8 +35,8 @@ public static Condition atPosition(Integer line, Intege public static Condition atPosition(Integer startLine, Integer endLine, Integer startColumn, Integer endColumn) { return new Condition<>(throwable -> { - var scriptException = (VtlScriptException) throwable; - var position = scriptException.getPosition(); + VtlScriptException scriptException = (VtlScriptException) throwable; + Position position = scriptException.getPosition(); return position.startLine.equals(startLine) && position.endLine.equals(endLine) && position.startColumn.equals(startColumn) && @@ -89,7 +89,7 @@ public void testPersistentAssignmentWithDs() throws ScriptException { VtlScriptEngine engine = (VtlScriptEngine) this.engine; ScriptContext context = engine.getContext(); - var ds = new InMemoryDataset(List.of(), Map.of()); + InMemoryDataset ds = new InMemoryDataset(Java8Helpers.listOf(), Java8Helpers.mapOf()); context.setAttribute("ds", ds, ScriptContext.ENGINE_SCOPE); engine.eval("pds <- ds;"); @@ -108,7 +108,7 @@ public void testPersistentAssignmentWithDs() throws ScriptException { public void testFunctionsExpression() throws NoSuchMethodException, ScriptException { InMemoryDataset ds = new InMemoryDataset( - List.of( + Java8Helpers.listOf( new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("age", Double.class, Dataset.Role.MEASURE), new Structured.Component("weight", Double.class, Dataset.Role.MEASURE) @@ -118,7 +118,7 @@ public void testFunctionsExpression() throws NoSuchMethodException, ScriptExcept Arrays.asList("Nico", 12.34, 12.34) ); - var ceilMethod = MathFunctions.class.getMethod("myCustomCeil", Double.class); + Method ceilMethod = MathFunctions.class.getMethod("myCustomCeil", Double.class); VtlScriptEngine engine = (VtlScriptEngine) this.engine; engine.registerMethod("myCustomCeil", ceilMethod); @@ -131,7 +131,7 @@ public void testFunctionsExpression() throws NoSuchMethodException, ScriptExcept @Test void testMultipleParameterFunctionExpressions() throws ScriptException { - var ds1 = new InMemoryDataset(List.of( + InMemoryDataset ds1 = new InMemoryDataset(Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("m1", Double.class, Dataset.Role.MEASURE) ), @@ -140,7 +140,7 @@ void testMultipleParameterFunctionExpressions() throws ScriptException { Arrays.asList("Nico", 4.567890123), Arrays.asList("NotHere", 9.87654321) ); - var ds2 = new InMemoryDataset(List.of( + InMemoryDataset ds2 = new InMemoryDataset(Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("m1", Long.class, Dataset.Role.MEASURE) ), @@ -154,15 +154,15 @@ void testMultipleParameterFunctionExpressions() throws ScriptException { engine.eval("res := round(ds1#m1, ds2#m1);"); assertThat(((Dataset) engine.get("res")).getDataAsList()).containsExactly( - List.of("Toto", 1.2346), - List.of("Hadrien", 2.35), - List.of("Nico", 4.6) + Java8Helpers.listOf("Toto", 1.2346), + Java8Helpers.listOf("Hadrien", 2.35), + Java8Helpers.listOf("Nico", 4.6) ); } @Test void testMultipleParameterFunctionExpressionsAndConstants() throws ScriptException { - var ds1 = new InMemoryDataset(List.of( + InMemoryDataset ds1 = new InMemoryDataset(Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("m1", Long.class, Dataset.Role.MEASURE) ), @@ -174,15 +174,15 @@ void testMultipleParameterFunctionExpressionsAndConstants() throws ScriptExcepti engine.eval("res := round(0.123456789, ds1#m1);"); assertThat(((Dataset) engine.get("res")).getDataAsList()).containsExactly( - List.of("Toto", 0.12), - List.of("Hadrien", 0.1235), - List.of("Nico", 0.12345679) + Java8Helpers.listOf("Toto", 0.12), + Java8Helpers.listOf("Hadrien", 0.1235), + Java8Helpers.listOf("Nico", 0.12345679) ); } @Test void testFunctionExpressionsWrongType() throws ScriptException { - var ds1 = new InMemoryDataset(List.of( + InMemoryDataset ds1 = new InMemoryDataset(Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("m1", Long.class, Dataset.Role.MEASURE) ), @@ -270,7 +270,7 @@ public void testSerialization() throws Exception { // // var bar = ResolvableExpression.withType(String.class).using(c -> "bar"); // var baz = ResolvableExpression.withType(String.class).using(c -> "baz"); -// var exprVisitor = new ExpressionVisitor(Map.of(), new InMemoryProcessingEngine(), engine); +// var exprVisitor = new ExpressionVisitor(Java8Helpers.mapOf(), new InMemoryProcessingEngine(), engine); // var comparisonVisitor = new ComparisonVisitor(exprVisitor); // var condition = comparisonVisitor.compareExpressions(bar, baz, VtlScriptEngineTest::isEqual); // var expr = ResolvableExpression.withTypeCasting(String.class, (clazz, ctx) -> @@ -286,7 +286,7 @@ public void testSerialization() throws Exception { public void testRegisterGlobal() throws NoSuchMethodException, ScriptException { InMemoryDataset ds = new InMemoryDataset( - List.of( + Java8Helpers.listOf( new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("age", Double.class, Dataset.Role.MEASURE), new Structured.Component("weight", Double.class, Dataset.Role.MEASURE) @@ -296,8 +296,8 @@ public void testRegisterGlobal() throws NoSuchMethodException, ScriptException { Arrays.asList("Nico", 12.34, 12.34) ); - var doNothingMethod = DoNothingClass.class.getMethod("doNothingMethod"); - var ceilMethod = MathFunctions.class.getMethod("myCustomCeil", Double.class); + Method doNothingMethod = DoNothingClass.class.getMethod("doNothingMethod"); + Method ceilMethod = MathFunctions.class.getMethod("myCustomCeil", Double.class); VtlScriptEngine engine = (VtlScriptEngine) this.engine; engine.registerGlobalMethod("doNothingMethod", doNothingMethod); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/exceptions/UnimplementedExceptionTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/exceptions/UnimplementedExceptionTest.java index 6b1b4ad4a..31925da6d 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/exceptions/UnimplementedExceptionTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/exceptions/UnimplementedExceptionTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.engine.exceptions; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -9,7 +10,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.SimpleBindings; -import java.util.List; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; @@ -19,8 +19,8 @@ public class UnimplementedExceptionTest { public void testSimple() { String vtlExpression = "a := first_value(ds over());"; Dataset ds = new InMemoryDataset( - List.of(List.of(1L, 31L)), - List.of(new Structured.Component("ID", Long.class, Dataset.Role.IDENTIFIER), + Java8Helpers.listOf(Java8Helpers.listOf(1L, 31L)), + Java8Helpers.listOf(new Structured.Component("ID", Long.class, Dataset.Role.IDENTIFIER), new Structured.Component("FOO", Long.class, Dataset.Role.MEASURE)) ); SimpleBindings bindings = new SimpleBindings(); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/samples/DatasetSamples.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/samples/DatasetSamples.java index fdaa357b7..7de848582 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/samples/DatasetSamples.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/samples/DatasetSamples.java @@ -1,16 +1,16 @@ package fr.insee.vtl.engine.samples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; import java.util.Arrays; -import java.util.List; public class DatasetSamples { public static InMemoryDataset ds1 = new InMemoryDataset( - List.of( + Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("long1", Long.class, Dataset.Role.MEASURE), new Structured.Component("long2", Long.class, Dataset.Role.MEASURE), @@ -28,7 +28,7 @@ public class DatasetSamples { ); public static InMemoryDataset ds2 = new InMemoryDataset( - List.of( + Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("long1", Long.class, Dataset.Role.MEASURE), new Structured.Component("double1", Double.class, Dataset.Role.MEASURE), diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java index 7ee5779dc..8813f973b 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/ClauseVisitorTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.engine.visitors; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -35,8 +36,8 @@ public void setUp() { @Test public void testFilterClause() throws ScriptException { - var dataset = new InMemoryDataset( - List.of( + InMemoryDataset dataset = new InMemoryDataset( + Java8Helpers.listOf( new Structured.Component("name", String.class, Role.IDENTIFIER), new Structured.Component("age", Long.class, Role.MEASURE), new Structured.Component("weight", Long.class, Role.MEASURE) @@ -53,29 +54,29 @@ public void testFilterClause() throws ScriptException { engine.eval("ds := ds1[filter age > 10 and age < 12];"); assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(Dataset.class); - assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).isEqualTo(List.of( - Map.of("name", "Nico", "age", 11L, "weight", 10L) + assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L) )); engine.eval("ds := ds1[filter age > 10 or age < 12];"); - assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).isEqualTo(List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) )); } @Test public void testManyCalc() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) ); ScriptContext context = engine.getContext(); @@ -83,24 +84,24 @@ public void testManyCalc() throws ScriptException { engine.eval("ds := ds1[rename age to wisdom][calc wisdom := wisdom * 2];"); - var ds = (Dataset) engine.getContext().getAttribute("ds"); + Dataset ds = (Dataset) engine.getContext().getAttribute("ds"); assertThat(ds.getDataAsMap()).contains( - Map.of("name", "Hadrien", "weight", 11L, "wisdom", 20L), - Map.of("name", "Nico", "weight", 10L, "wisdom", 22L), - Map.of("name", "Franck", "weight", 9L, "wisdom", 24L) + Java8Helpers.mapOf("name", "Hadrien", "weight", 11L, "wisdom", 20L), + Java8Helpers.mapOf("name", "Nico", "weight", 10L, "wisdom", 22L), + Java8Helpers.mapOf("name", "Franck", "weight", 9L, "wisdom", 24L) ); } @Test public void testCalcRoleModifier() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) ); ScriptContext context = engine.getContext(); @@ -127,13 +128,13 @@ public void testCalcRoleModifier() throws ScriptException { @Test public void testRenameClause() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) ); ScriptContext context = engine.getContext(); @@ -143,9 +144,9 @@ public void testRenameClause() throws ScriptException { assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("pseudo", "Hadrien", "weight", 10L, "age", 11L), - Map.of("pseudo", "Nico", "weight", 11L, "age", 10L), - Map.of("pseudo", "Franck", "weight", 12L, "age", 9L) + Java8Helpers.mapOf("pseudo", "Hadrien", "weight", 10L, "age", 11L), + Java8Helpers.mapOf("pseudo", "Nico", "weight", 11L, "age", 10L), + Java8Helpers.mapOf("pseudo", "Franck", "weight", 12L, "age", 9L) ); assertThatThrownBy(() -> engine.eval("ds := ds1[rename age to weight, weight to age, name to age];")) @@ -158,13 +159,13 @@ public void testRenameClause() throws ScriptException { public void testCalcClause() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) ); ScriptContext context = engine.getContext(); @@ -174,9 +175,9 @@ public void testCalcClause() throws ScriptException { assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("name", "Hadrien", "res", 15.5, "age", 10L, "weight", 11L), - Map.of("name", "Nico", "res", 16.0, "age", 11L, "weight", 10L), - Map.of("name", "Franck", "res", 16.5, "age", 12L, "weight", 9L) + Java8Helpers.mapOf("name", "Hadrien", "res", 15.5, "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "res", 16.0, "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "res", 16.5, "age", 12L, "weight", 9L) ); } @@ -184,13 +185,13 @@ public void testCalcClause() throws ScriptException { @Test public void testKeepDropClause() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) ); ScriptContext context = engine.getContext(); @@ -200,29 +201,29 @@ public void testKeepDropClause() throws ScriptException { assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("name", "Hadrien", "age", 10L), - Map.of("name", "Nico", "age", 11L), - Map.of("name", "Franck", "age", 12L) + Java8Helpers.mapOf("name", "Hadrien", "age", 10L), + Java8Helpers.mapOf("name", "Nico", "age", 11L), + Java8Helpers.mapOf("name", "Franck", "age", 12L) ); engine.eval("ds := ds1[drop weight];"); assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("name", "Hadrien", "age", 10L), - Map.of("name", "Nico", "age", 11L), - Map.of("name", "Franck", "age", 12L) + Java8Helpers.mapOf("name", "Hadrien", "age", 10L), + Java8Helpers.mapOf("name", "Nico", "age", 11L), + Java8Helpers.mapOf("name", "Franck", "age", 12L) ); } @Test public void testAggregateType() { InMemoryDataset dataset = new InMemoryDataset( - List.of(), - Map.of("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), - Map.of("name", Role.IDENTIFIER, "country", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) + Java8Helpers.listOf(), + Java8Helpers.mapOf("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "country", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) ); - var cases = List.of( + List cases = Java8Helpers.listOf( "res := ds1[aggr a := sum(name) group by country];", "res := ds1[aggr a := avg(name) group by country];", "res := ds1[aggr a := max(name) group by country];", @@ -248,14 +249,14 @@ public void testAggregateType() { public void testAggregate() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), - Map.of("name", "Nico", "country", "france", "age", 11L, "weight", 10D), - Map.of("name", "Franck", "country", "france", "age", 12L, "weight", 9D) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), + Java8Helpers.mapOf("name", "Nico", "country", "france", "age", 11L, "weight", 10D), + Java8Helpers.mapOf("name", "Franck", "country", "france", "age", 12L, "weight", 9D) ), - Map.of("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), - Map.of("name", Role.IDENTIFIER, "country", Role.MEASURE, "age", Role.MEASURE, "weight", Role.MEASURE), - Map.of("name", false, "country", true) + Java8Helpers.mapOf("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "country", Role.MEASURE, "age", Role.MEASURE, "weight", Role.MEASURE), + Java8Helpers.mapOf("name", false, "country", true) ); ScriptContext context = engine.getContext(); @@ -278,26 +279,26 @@ public void testAggregate() throws ScriptException { " group by country];"); assertThat(engine.getContext().getAttribute("res")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("country", "france", "sumAge", 23L, "avgWeight", 11.5, + Java8Helpers.mapOf("country", "france", "sumAge", 23L, "avgWeight", 11.5, "countVal", 2L, "maxAge", 12L, "maxWeight", 10D, "minAge", 11L, "minWeight", 9D, "medianAge", 11.5D, "medianWeight", 9.5D), - Map.of("country", "norway", "sumAge", 10L, "avgWeight", 10.0, + Java8Helpers.mapOf("country", "norway", "sumAge", 10L, "avgWeight", 10.0, "countVal", 1L, "maxAge", 10L, "maxWeight", 11D, "minAge", 10L, "minWeight", 11D, "medianAge", 10D, "medianWeight", 11D) ); InMemoryDataset dataset2 = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), - Map.of("name", "Nico", "country", "france", "age", 9L, "weight", 5D), - Map.of("name", "Franck", "country", "france", "age", 10L, "weight", 15D), - Map.of("name", "Nico1", "country", "france", "age", 11L, "weight", 10D), - Map.of("name", "Franck1", "country", "france", "age", 12L, "weight", 8D) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), + Java8Helpers.mapOf("name", "Nico", "country", "france", "age", 9L, "weight", 5D), + Java8Helpers.mapOf("name", "Franck", "country", "france", "age", 10L, "weight", 15D), + Java8Helpers.mapOf("name", "Nico1", "country", "france", "age", 11L, "weight", 10D), + Java8Helpers.mapOf("name", "Franck1", "country", "france", "age", 12L, "weight", 8D) ), - Map.of("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), - Map.of("name", Role.IDENTIFIER, "country", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), + Java8Helpers.mapOf("name", Role.IDENTIFIER, "country", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) ); context.setAttribute("ds2", dataset2, ScriptContext.ENGINE_SCOPE); @@ -315,7 +316,7 @@ public void testAggregate() throws ScriptException { assertThat(engine.getContext().getAttribute("res")).isInstanceOf(Dataset.class); - var fr = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().get(0); + Map fr = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().get(0); assertThat((Double) fr.get("stddev_popAge")).isCloseTo(1.118, Percentage.withPercentage(2)); assertThat((Double) fr.get("stddev_popWeight")).isCloseTo(3.640, Percentage.withPercentage(2)); @@ -326,7 +327,7 @@ public void testAggregate() throws ScriptException { assertThat((Double) fr.get("var_sampAge")).isCloseTo(1.666, Percentage.withPercentage(2)); assertThat((Double) fr.get("var_sampWeight")).isCloseTo(17.666, Percentage.withPercentage(2)); - var no = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().get(1); + Map no = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().get(1); assertThat((Double) no.get("stddev_popAge")).isEqualTo(0.0); assertThat((Double) no.get("stddev_popWeight")).isEqualTo(0.0); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java index 7af1018d5..5d3bf3e25 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprOrConcatTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -9,7 +10,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -55,10 +55,10 @@ public void testPlus() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := ds1[keep id, long1, long2] + ds1[keep id, long1, long2];"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 60L, "long2", 600L), - Map.of("id", "Hadrien", "long1", 20L, "long2", 2L), - Map.of("id", "Nico", "long1", 40L, "long2", 500L), - Map.of("id", "Franck", "long1", 200L, "long2", 4L) + Java8Helpers.mapOf("id", "Toto", "long1", 60L, "long2", 600L), + Java8Helpers.mapOf("id", "Hadrien", "long1", 20L, "long2", 2L), + Java8Helpers.mapOf("id", "Nico", "long1", 40L, "long2", 500L), + Java8Helpers.mapOf("id", "Franck", "long1", 200L, "long2", 4L) ); // assertThat(((Dataset) res).getDataStructure().get("long2").getType()).isEqualTo(Long.class); } @@ -77,9 +77,9 @@ public void testMinus() throws ScriptException { context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := ds2[keep id, long1] - ds1[keep id, long1] + 1;"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "long1", 141L), - Map.of("id", "Nico", "long1", 1L), - Map.of("id", "Franck", "long1", 1L) + Java8Helpers.mapOf("id", "Hadrien", "long1", 141L), + Java8Helpers.mapOf("id", "Nico", "long1", 1L), + Java8Helpers.mapOf("id", "Franck", "long1", 1L) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Long.class); } @@ -94,9 +94,9 @@ public void testConcat() throws ScriptException { context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := ds2[keep id, string1] || \" \" || ds1[keep id, string1];"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "string1", "hadrien hadrien"), - Map.of("id", "Nico", "string1", "nico nico"), - Map.of("id", "Franck", "string1", "franck franck") + Java8Helpers.mapOf("id", "Hadrien", "string1", "hadrien hadrien"), + Java8Helpers.mapOf("id", "Nico", "string1", "nico nico"), + Java8Helpers.mapOf("id", "Franck", "string1", "franck franck") ); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(String.class); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java index 2f27550fe..3819a6bf8 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ArithmeticExprTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -10,7 +11,6 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -28,8 +28,8 @@ public void testNull() throws ScriptException { ScriptContext context = engine.getContext(); - List operators = List.of("+", "-", "/", "*"); - List values = List.of( + List operators = Java8Helpers.listOf("+", "-", "/", "*"); + List values = Java8Helpers.listOf( "1.1", "1", "cast(null, integer)" ); @@ -64,9 +64,9 @@ public void testArithmeticExpr() throws ScriptException { context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := round(ds1[keep id, long1, double1] * ds2[keep id, long1, double1]);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "long1", 1500.0, "double1", 1.0), - Map.of("id", "Nico", "long1", 400.0, "double1", 27.0), - Map.of("id", "Franck", "long1", 10000.0, "double1", -1.0) + Java8Helpers.mapOf("id", "Hadrien", "long1", 1500.0, "double1", 1.0), + Java8Helpers.mapOf("id", "Nico", "long1", 400.0, "double1", 27.0), + Java8Helpers.mapOf("id", "Franck", "long1", 10000.0, "double1", -1.0) ); // assertThat(((Dataset) res).getDataStructure().get("long2").getType()).isEqualTo(Long.class); @@ -83,9 +83,9 @@ public void testArithmeticExpr() throws ScriptException { context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); res = engine.eval("res := round(ds1[keep id, long1, double1] / ds2[keep id, long1, double1]);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "long1", 0.0, "double1", 1.0), - Map.of("id", "Nico", "long1", 1.0, "double1", 6.0), - Map.of("id", "Franck", "long1", 1.0, "double1", -1.0) + Java8Helpers.mapOf("id", "Hadrien", "long1", 0.0, "double1", 1.0), + Java8Helpers.mapOf("id", "Nico", "long1", 1.0, "double1", 6.0), + Java8Helpers.mapOf("id", "Franck", "long1", 1.0, "double1", -1.0) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java index d1f87fe49..9b9426769 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/BooleanExprTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -12,7 +13,6 @@ import javax.script.ScriptException; import java.util.Arrays; import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -95,24 +95,24 @@ public void testOnDatasets() throws ScriptException { "xorDs := ds1 xor ds2; "); Dataset and = (Dataset) context.getAttribute("andDs"); assertThat(and.getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "bool_var", false), - Map.of("id", "Hadrien", "bool_var", true), - Map.of("id", "Nico", "bool_var", false), - Map.of("id", "Franck", "bool_var", false) + Java8Helpers.mapOf("id", "Toto", "bool_var", false), + Java8Helpers.mapOf("id", "Hadrien", "bool_var", true), + Java8Helpers.mapOf("id", "Nico", "bool_var", false), + Java8Helpers.mapOf("id", "Franck", "bool_var", false) ); Dataset or = (Dataset) context.getAttribute("orDs"); assertThat(or.getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "bool_var", true), - Map.of("id", "Hadrien", "bool_var", true), - Map.of("id", "Nico", "bool_var", true), - Map.of("id", "Franck", "bool_var", false) + Java8Helpers.mapOf("id", "Toto", "bool_var", true), + Java8Helpers.mapOf("id", "Hadrien", "bool_var", true), + Java8Helpers.mapOf("id", "Nico", "bool_var", true), + Java8Helpers.mapOf("id", "Franck", "bool_var", false) ); Dataset xor = (Dataset) context.getAttribute("xorDs"); assertThat(xor.getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "bool_var", true), - Map.of("id", "Hadrien", "bool_var", false), - Map.of("id", "Nico", "bool_var", true), - Map.of("id", "Franck", "bool_var", false) + Java8Helpers.mapOf("id", "Toto", "bool_var", true), + Java8Helpers.mapOf("id", "Hadrien", "bool_var", false), + Java8Helpers.mapOf("id", "Nico", "bool_var", true), + Java8Helpers.mapOf("id", "Franck", "bool_var", false) ); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java index f4bb0e74b..5bdf7b4a5 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ComparisonExprTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.exceptions.VtlScriptException; import org.junit.jupiter.api.BeforeEach; @@ -11,7 +12,6 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -31,12 +31,12 @@ public void testNull() throws ScriptException { ScriptContext context = engine.getContext(); context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); - List operators = List.of("=", "<>", "<", ">", "<=", ">="); - List> values = List.of( - List.of("\"string\"", "string"), - List.of("1.1", "number"), - List.of("1", "integer"), - List.of("cast(null, number)", "number") + List operators = Java8Helpers.listOf("=", "<>", "<", ">", "<=", ">="); + List> values = Java8Helpers.listOf( + Java8Helpers.listOf("\"string\"", "string"), + Java8Helpers.listOf("1.1", "number"), + Java8Helpers.listOf("1", "integer"), + Java8Helpers.listOf("cast(null, number)", "number") ); for (String operator : operators) { @@ -68,11 +68,11 @@ public void testComparisonExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval("equal := ds1[keep id, long1] = ds2[keep id, long1];"); - var equal = engine.getContext().getAttribute("equal"); + Object equal = engine.getContext().getAttribute("equal"); assertThat(((Dataset) equal).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", false), - Map.of("id", "Nico", "bool_var", true), - Map.of("id", "Franck", "bool_var", true) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", false), + Java8Helpers.mapOf("id", "Nico", "bool_var", true), + Java8Helpers.mapOf("id", "Franck", "bool_var", true) ); assertThat(((Dataset) equal).getDataStructure().get("bool_var").getType()).isEqualTo(Boolean.class); @@ -86,11 +86,11 @@ public void testComparisonExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval("notEqual := ds1[keep id, long1] <> ds2[keep id, long1];"); - var notEqual = engine.getContext().getAttribute("notEqual"); + Object notEqual = engine.getContext().getAttribute("notEqual"); assertThat(((Dataset) notEqual).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", true), - Map.of("id", "Nico", "bool_var", false), - Map.of("id", "Franck", "bool_var", false) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", true), + Java8Helpers.mapOf("id", "Nico", "bool_var", false), + Java8Helpers.mapOf("id", "Franck", "bool_var", false) ); assertThat(((Dataset) equal).getDataStructure().get("bool_var").getType()).isEqualTo(Boolean.class); // LT @@ -103,11 +103,11 @@ public void testComparisonExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval("lt := ds1[keep id, long1] < ds2[keep id, long1];"); - var lt = engine.getContext().getAttribute("lt"); + Object lt = engine.getContext().getAttribute("lt"); assertThat(((Dataset) lt).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", true), - Map.of("id", "Nico", "bool_var", false), - Map.of("id", "Franck", "bool_var", false) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", true), + Java8Helpers.mapOf("id", "Nico", "bool_var", false), + Java8Helpers.mapOf("id", "Franck", "bool_var", false) ); assertThat(((Dataset) equal).getDataStructure().get("bool_var").getType()).isEqualTo(Boolean.class); // MT @@ -120,11 +120,11 @@ public void testComparisonExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval("mt := ds1[keep id, long1] > ds2[keep id, long1];"); - var mt = engine.getContext().getAttribute("mt"); + Object mt = engine.getContext().getAttribute("mt"); assertThat(((Dataset) mt).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", false), - Map.of("id", "Nico", "bool_var", false), - Map.of("id", "Franck", "bool_var", false) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", false), + Java8Helpers.mapOf("id", "Nico", "bool_var", false), + Java8Helpers.mapOf("id", "Franck", "bool_var", false) ); assertThat(((Dataset) equal).getDataStructure().get("bool_var").getType()).isEqualTo(Boolean.class); // LE @@ -137,11 +137,11 @@ public void testComparisonExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval("le := ds1[keep id, long1] <= ds2[keep id, long1];"); - var le = engine.getContext().getAttribute("le"); + Object le = engine.getContext().getAttribute("le"); assertThat(((Dataset) le).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", true), - Map.of("id", "Nico", "bool_var", true), - Map.of("id", "Franck", "bool_var", true) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", true), + Java8Helpers.mapOf("id", "Nico", "bool_var", true), + Java8Helpers.mapOf("id", "Franck", "bool_var", true) ); assertThat(((Dataset) equal).getDataStructure().get("bool_var").getType()).isEqualTo(Boolean.class); // ME @@ -155,11 +155,11 @@ public void testComparisonExpr() throws ScriptException { context.setAttribute("ds1", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); context.setAttribute("ds2", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); engine.eval("me := ds1[keep id, long1] >= ds2[keep id, long1];"); - var me = engine.getContext().getAttribute("me"); + Object me = engine.getContext().getAttribute("me"); assertThat(((Dataset) me).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", false), - Map.of("id", "Nico", "bool_var", true), - Map.of("id", "Franck", "bool_var", true) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", false), + Java8Helpers.mapOf("id", "Nico", "bool_var", true), + Java8Helpers.mapOf("id", "Franck", "bool_var", true) ); assertThat(((Dataset) equal).getDataStructure().get("bool_var").getType()).isEqualTo(Boolean.class); } @@ -194,12 +194,12 @@ public void testInNotIn() throws ScriptException { engine.getContext().setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.eval("me := ds[keep id, long1, string1] in {\"toto\", \"franck\"};"); - var in = engine.getContext().getAttribute("me"); + Object in = engine.getContext().getAttribute("me"); assertThat(((Dataset) in).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", false, "string1", true), - Map.of("id", "Hadrien", "long1", false, "string1", false), - Map.of("id", "Nico", "long1", false, "string1", false), - Map.of("id", "Franck", "long1", false, "string1", true) + Java8Helpers.mapOf("id", "Toto", "long1", false, "string1", true), + Java8Helpers.mapOf("id", "Hadrien", "long1", false, "string1", false), + Java8Helpers.mapOf("id", "Nico", "long1", false, "string1", false), + Java8Helpers.mapOf("id", "Franck", "long1", false, "string1", true) ); assertThat(((Dataset) in).getDataStructure().get("string1").getType()).isEqualTo(Boolean.class); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java index 966008ede..207ea1ab2 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/ConditionalExprTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -10,7 +11,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -49,11 +49,11 @@ public void testIfExpr() throws ScriptException { engine.eval("ds1 := ds_1[keep id, long1][rename long1 to bool_var]; " + "ds2 := ds_2[keep id, long1][rename long1 to bool_var]; " + "res := if ds1 > ds2 then ds1 else ds2;"); - var res = engine.getContext().getAttribute("res"); + Object res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", 150L), - Map.of("id", "Nico", "bool_var", 20L), - Map.of("id", "Franck", "bool_var", 100L) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", 150L), + Java8Helpers.mapOf("id", "Nico", "bool_var", 20L), + Java8Helpers.mapOf("id", "Franck", "bool_var", 100L) ); assertThat(((Dataset) res).getDataStructure().get("bool_var").getType()).isEqualTo(Long.class); } @@ -68,12 +68,12 @@ public void testNvlExpr() throws ScriptException { engine.getContext().setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.eval("res := nvl(ds[keep id, long1], 0);"); - var res = engine.getContext().getAttribute("res"); + Object res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 30L), - Map.of("id", "Hadrien", "long1", 10L), - Map.of("id", "Nico", "long1", 20L), - Map.of("id", "Franck", "long1", 100L) + Java8Helpers.mapOf("id", "Toto", "long1", 30L), + Java8Helpers.mapOf("id", "Hadrien", "long1", 10L), + Java8Helpers.mapOf("id", "Nico", "long1", 20L), + Java8Helpers.mapOf("id", "Franck", "long1", 100L) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Long.class); @@ -93,12 +93,12 @@ public void testNvlImplicitCast() throws ScriptException { engine.getContext().setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); engine.eval("res := nvl(ds[keep id, long1], 0.1);"); - var res = engine.getContext().getAttribute("res"); + Object res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 30D), - Map.of("id", "Hadrien", "long1", 10D), - Map.of("id", "Nico", "long1", 20D), - Map.of("id", "Franck", "long1", 100D) + Java8Helpers.mapOf("id", "Toto", "long1", 30D), + Java8Helpers.mapOf("id", "Hadrien", "long1", 10D), + Java8Helpers.mapOf("id", "Nico", "long1", 20D), + Java8Helpers.mapOf("id", "Franck", "long1", 100D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java index bb375abc2..301dedf72 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/UnaryExprTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -10,7 +11,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -47,9 +47,9 @@ public void testUnaryExpr() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := + ds[keep id, long1, double1];"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "long1", 150L, "double1", 1.1D), - Map.of("id", "Nico", "long1", 20L, "double1", 2.2D), - Map.of("id", "Franck", "long1", 100L, "double1", -1.21D) + Java8Helpers.mapOf("id", "Hadrien", "long1", 150L, "double1", 1.1D), + Java8Helpers.mapOf("id", "Nico", "long1", 20L, "double1", 2.2D), + Java8Helpers.mapOf("id", "Franck", "long1", 100L, "double1", -1.21D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Long.class); @@ -61,9 +61,9 @@ public void testUnaryExpr() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); res = engine.eval("res := - ds[keep id, long1, double1];"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "long1", -150L, "double1", -1.1D), - Map.of("id", "Nico", "long1", -20L, "double1", -2.2D), - Map.of("id", "Franck", "long1", -100L, "double1", 1.21D) + Java8Helpers.mapOf("id", "Hadrien", "long1", -150L, "double1", -1.1D), + Java8Helpers.mapOf("id", "Nico", "long1", -20L, "double1", -2.2D), + Java8Helpers.mapOf("id", "Franck", "long1", -100L, "double1", 1.21D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Long.class); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java index afbc26fc4..684637920 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ComparisonFunctionsTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -10,7 +11,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -53,10 +53,10 @@ public void testBetweenAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := between(ds[keep id, long1, double2], 5, 15);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", false, "double2", false), - Map.of("id", "Hadrien", "long1", true, "double2", true), - Map.of("id", "Nico", "long1", false, "double2", false), - Map.of("id", "Franck", "long1", false, "double2", false) + Java8Helpers.mapOf("id", "Toto", "long1", false, "double2", false), + Java8Helpers.mapOf("id", "Hadrien", "long1", true, "double2", true), + Java8Helpers.mapOf("id", "Nico", "long1", false, "double2", false), + Java8Helpers.mapOf("id", "Franck", "long1", false, "double2", false) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Boolean.class); @@ -81,10 +81,10 @@ public void testCharsetMatchAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := match_characters(ds[keep id, string1, string2], \"(.*)o(.*)\");"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "string1", true, "string2", false), - Map.of("id", "Hadrien", "string1", false, "string2", false), - Map.of("id", "Nico", "string1", true, "string2", false), - Map.of("id", "Franck", "string1", false, "string2", false) + Java8Helpers.mapOf("id", "Toto", "string1", true, "string2", false), + Java8Helpers.mapOf("id", "Hadrien", "string1", false, "string2", false), + Java8Helpers.mapOf("id", "Nico", "string1", true, "string2", false), + Java8Helpers.mapOf("id", "Franck", "string1", false, "string2", false) ); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(Boolean.class); @@ -109,10 +109,10 @@ public void testIsNullAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := isnull(ds[keep id, string1, bool1]);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "string1", false, "bool1", false), - Map.of("id", "Hadrien", "string1", false, "bool1", false), - Map.of("id", "Nico", "string1", false, "bool1", false), - Map.of("id", "Franck", "string1", false, "bool1", false) + Java8Helpers.mapOf("id", "Toto", "string1", false, "bool1", false), + Java8Helpers.mapOf("id", "Hadrien", "string1", false, "bool1", false), + Java8Helpers.mapOf("id", "Nico", "string1", false, "bool1", false), + Java8Helpers.mapOf("id", "Franck", "string1", false, "bool1", false) ); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(Boolean.class); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java index 7d1c2a840..2dfb548fe 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/DistanceFunctionsTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -10,7 +11,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -44,10 +44,10 @@ public void testLevenshteinAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := levenshtein(ds[keep id, string1], ds[keep id, string2][rename string2 to string1])[rename string1 to lev];"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "lev", 3L), - Map.of("id", "Hadrien", "lev", 7L), - Map.of("id", "Nico", "lev", 4L), - Map.of("id", "Franck", "lev", 5L) + Java8Helpers.mapOf("id", "Toto", "lev", 3L), + Java8Helpers.mapOf("id", "Hadrien", "lev", 7L), + Java8Helpers.mapOf("id", "Nico", "lev", 4L), + Java8Helpers.mapOf("id", "Franck", "lev", 5L) ); assertThat(((Dataset) res).getDataStructure().get("lev").getType()).isEqualTo(Long.class); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsTest.java index b3f42c116..857d3fa8f 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsTest.java @@ -10,10 +10,8 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.time.Duration; import java.time.Instant; import java.time.Period; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -53,15 +51,15 @@ public void testDuration() throws ScriptException { // but examples around line 1507 indicates that this is the case. The threeten-extra classes are used to // simplify parsing of duration and intervals. engine.eval("d1 := cast(\"P1Y2M10DT2H30M\", duration);"); - var d1 = engine.getContext().getAttribute("d1"); + Object d1 = engine.getContext().getAttribute("d1"); assertThat(d1).isInstanceOf(PeriodDuration.class); - assertThat((PeriodDuration)d1).isEqualTo(PeriodDuration.parse("P1Y2M10DT2H30M")); + assertThat((PeriodDuration) d1).isEqualTo(PeriodDuration.parse("P1Y2M10DT2H30M")); // Normalization engine.eval("d2 := cast(\"P1Y15M2DT086401S\", duration);"); - var d2 = engine.getContext().getAttribute("d2"); + Object d2 = engine.getContext().getAttribute("d2"); assertThat(d2).isInstanceOf(PeriodDuration.class); - assertThat((PeriodDuration)d2).isEqualTo(PeriodDuration.parse("P2Y3M3DT1S")); + assertThat((PeriodDuration) d2).isEqualTo(PeriodDuration.parse("P2Y3M3DT1S")); } @@ -69,21 +67,21 @@ public void testDuration() throws ScriptException { public void testTimePeriod() throws ScriptException { // Start/End format engine.eval("p1 := cast(\"2015-03-03T09:30:45Z/2018-04-05T12:30:15Z\", time_period);"); - var p1 = engine.getContext().getAttribute("p1"); + Object p1 = engine.getContext().getAttribute("p1"); assertThat(p1).isInstanceOf(Interval.class); - assertThat((Interval)p1).isEqualTo(Interval.parse("2015-03-03T09:30:45Z/2018-04-05T12:30:15Z")); + assertThat((Interval) p1).isEqualTo(Interval.parse("2015-03-03T09:30:45Z/2018-04-05T12:30:15Z")); // Start/Duration format engine.eval("p2 := cast(\"2007-03-01T13:00:00Z/P1Y2M10DT2H30M\", time_period);"); - var p2 = engine.getContext().getAttribute("p2"); + Object p2 = engine.getContext().getAttribute("p2"); assertThat(p2).isInstanceOf(Interval.class); - assertThat((Interval)p2).isEqualTo(Interval.parse("2007-03-01T13:00:00Z/2008-05-11T15:30:00Z")); + assertThat((Interval) p2).isEqualTo(Interval.parse("2007-03-01T13:00:00Z/2008-05-11T15:30:00Z")); // Duration/End format engine.eval("p3 := cast(\"P1Y2M10DT2H30M/2008-05-11T15:30:00Z\", time_period);"); - var p3 = engine.getContext().getAttribute("p3"); + Object p3 = engine.getContext().getAttribute("p3"); assertThat(p3).isInstanceOf(Interval.class); - assertThat((Interval)p3).isEqualTo(Interval.parse("2007-03-01T13:00:00Z/2008-05-11T15:30:00Z")); + assertThat((Interval) p3).isEqualTo(Interval.parse("2007-03-01T13:00:00Z/2008-05-11T15:30:00Z")); } @Test diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java index 85cd2bfb2..7869683d3 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/JoinFunctionsTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.engine.visitors.expression.functions; import fr.insee.vtl.engine.exceptions.InvalidArgumentException; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -13,7 +14,6 @@ import javax.script.ScriptException; import java.util.Arrays; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import static fr.insee.vtl.engine.VtlScriptEngineTest.atPosition; @@ -24,30 +24,30 @@ public class JoinFunctionsTest { private final InMemoryDataset ds1 = new InMemoryDataset( - List.of( - List.of("a", 1L, 1L), - List.of("a", 2L, 2L), - List.of("b", 1L, 3L), - List.of("b", 2L, 4L), - List.of("c", 1L, 5L), - List.of("c", 2L, 6L) + Java8Helpers.listOf( + Java8Helpers.listOf("a", 1L, 1L), + Java8Helpers.listOf("a", 2L, 2L), + Java8Helpers.listOf("b", 1L, 3L), + Java8Helpers.listOf("b", 2L, 4L), + Java8Helpers.listOf("c", 1L, 5L), + Java8Helpers.listOf("c", 2L, 6L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("id1", String.class, Role.IDENTIFIER), new Structured.Component("id2", Long.class, Role.IDENTIFIER), new Structured.Component("m1", Long.class, Role.MEASURE) ) ); private final InMemoryDataset ds2 = new InMemoryDataset( - List.of( - List.of("a", 1L, 7L), - List.of("a", 2L, 8L), - List.of("b", 1L, 9L), - List.of("b", 2L, 10L), - List.of("d", 3L, 11L), - List.of("d", 4L, 12L) + Java8Helpers.listOf( + Java8Helpers.listOf("a", 1L, 7L), + Java8Helpers.listOf("a", 2L, 8L), + Java8Helpers.listOf("b", 1L, 9L), + Java8Helpers.listOf("b", 2L, 10L), + Java8Helpers.listOf("d", 3L, 11L), + Java8Helpers.listOf("d", 4L, 12L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("id1", String.class, Role.IDENTIFIER), new Structured.Component("id2", Long.class, Role.IDENTIFIER), new Structured.Component("m2", Long.class, Role.MEASURE) @@ -86,7 +86,7 @@ public void testLeftJoinWithUsing() throws ScriptException { engine.getContext().setAttribute("ds2", ds2, ScriptContext.ENGINE_SCOPE); engine.eval("result := left_join(ds1, ds2 as aliasDs using id1);"); - var result = (Dataset) engine.getContext().getAttribute("result"); + Dataset result = (Dataset) engine.getContext().getAttribute("result"); assertThat(result.getColumnNames()).containsExactlyInAnyOrder( "id1", "id2", "m1", "m2" ); @@ -107,22 +107,22 @@ public void testLeftJoinWithUsing() throws ScriptException { @Test public void testLeftJoinWithDifferentIdentifiers() throws ScriptException { InMemoryDataset dataset1 = new InMemoryDataset( - List.of( - List.of(1L, 1L), - List.of(1L, 2L), - List.of(2L, 1L) + Java8Helpers.listOf( + Java8Helpers.listOf(1L, 1L), + Java8Helpers.listOf(1L, 2L), + Java8Helpers.listOf(2L, 1L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", Long.class, Role.IDENTIFIER), new Structured.Component("Id_2", Long.class, Role.IDENTIFIER) ) ); InMemoryDataset dataset2 = new InMemoryDataset( - List.of( - List.of(1L, "X"), - List.of(2L, "Y") + Java8Helpers.listOf( + Java8Helpers.listOf(1L, "X"), + Java8Helpers.listOf(2L, "Y") ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_2", Long.class, Role.IDENTIFIER), new Structured.Component("Me_1", String.class, Role.MEASURE) ) @@ -135,9 +135,9 @@ public void testLeftJoinWithDifferentIdentifiers() throws ScriptException { engine.eval("result := left_join(ds_1, ds_2 using Id_2);"); Dataset result = ((Dataset) engine.getContext().getAttribute("result")); assertThat(result.getDataAsMap()).containsExactlyInAnyOrder( - Map.of("Id_1", 1L, "Id_2", 1L, "Me_1", "X"), - Map.of("Id_1", 1L, "Id_2", 2L, "Me_1", "Y"), - Map.of("Id_1", 2L, "Id_2", 1L, "Me_1", "X") + Java8Helpers.mapOf("Id_1", 1L, "Id_2", 1L, "Me_1", "X"), + Java8Helpers.mapOf("Id_1", 1L, "Id_2", 2L, "Me_1", "Y"), + Java8Helpers.mapOf("Id_1", 2L, "Id_2", 1L, "Me_1", "X") ); assertThat(result.getDataStructure().get("Id_1").getRole()).isEqualTo(Role.IDENTIFIER); assertThat(result.getDataStructure().get("Id_2").getRole()).isEqualTo(Role.IDENTIFIER); @@ -152,21 +152,21 @@ public void testLeftJoinWithDifferentIdentifiers() throws ScriptException { @Test public void testLeftJoinWithDouble() throws ScriptException { InMemoryDataset dataset1 = new InMemoryDataset( - List.of( - List.of("Hadrien", 10L), - List.of("Nico", 11L) + Java8Helpers.listOf( + Java8Helpers.listOf("Hadrien", 10L), + Java8Helpers.listOf("Nico", 11L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("name", String.class, Role.IDENTIFIER), new Structured.Component("age", Long.class, Role.MEASURE) ) ); InMemoryDataset dataset2 = new InMemoryDataset( - List.of( - List.of("Hadrien", 20L), - List.of("Nico", 22L) + Java8Helpers.listOf( + Java8Helpers.listOf("Hadrien", 20L), + Java8Helpers.listOf("Nico", 22L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("name", String.class, Role.IDENTIFIER), new Structured.Component("age", Long.class, Role.MEASURE) ) @@ -182,8 +182,8 @@ public void testLeftJoinWithDouble() throws ScriptException { engine.eval("result := left_join(ds_1 as ds1, ds_2 as ds2);"); assertThat(((Dataset) engine.getContext().getAttribute("result")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("name", "Hadrien", "age", 20L), - Map.of("name", "Nico", "age", 22L) + Java8Helpers.mapOf("name", "Hadrien", "age", 20L), + Java8Helpers.mapOf("name", "Nico", "age", 22L) ); } @@ -192,21 +192,21 @@ public void testLeftJoinWithDouble() throws ScriptException { public void testLeftJoinMixedStructure() throws ScriptException { InMemoryDataset dataset1 = new InMemoryDataset( - List.of( - Map.of("id", "K001", "measure1", 1L, "measure2", 9L, "color", "yellow"), - Map.of("id", "K002", "measure1", 2L, "measure2", 8L, "color", "blue") + Java8Helpers.listOf( + Java8Helpers.mapOf("id", "K001", "measure1", 1L, "measure2", 9L, "color", "yellow"), + Java8Helpers.mapOf("id", "K002", "measure1", 2L, "measure2", 8L, "color", "blue") ), - Map.of("id", String.class, "measure1", Long.class, "measure2", Long.class, "color", String.class), - Map.of("id", Role.IDENTIFIER, "measure1", Role.MEASURE, "measure2", Role.MEASURE, "color", Role.MEASURE) + Java8Helpers.mapOf("id", String.class, "measure1", Long.class, "measure2", Long.class, "color", String.class), + Java8Helpers.mapOf("id", Role.IDENTIFIER, "measure1", Role.MEASURE, "measure2", Role.MEASURE, "color", Role.MEASURE) ); InMemoryDataset dataset2 = new InMemoryDataset( - List.of( - Map.of("id", "K001", "intermezzo", 0L, "measure2", 11L, "measure1", 19L, "stale", ""), - Map.of("id", "K003", "intermezzo", 0L, "measure2", 7L, "measure1", 3L, "stale", "") + Java8Helpers.listOf( + Java8Helpers.mapOf("id", "K001", "intermezzo", 0L, "measure2", 11L, "measure1", 19L, "stale", ""), + Java8Helpers.mapOf("id", "K003", "intermezzo", 0L, "measure2", 7L, "measure1", 3L, "stale", "") ), - Map.of("id", String.class, "intermezzo", Long.class, "measure2", Long.class, "measure1", Long.class, "stale", String.class), - Map.of("id", Role.IDENTIFIER, "intermezzo", Role.MEASURE, "measure2", Role.MEASURE, "measure1", Role.MEASURE, "stale", Role.MEASURE) + Java8Helpers.mapOf("id", String.class, "intermezzo", Long.class, "measure2", Long.class, "measure1", Long.class, "stale", String.class), + Java8Helpers.mapOf("id", Role.IDENTIFIER, "intermezzo", Role.MEASURE, "measure2", Role.MEASURE, "measure1", Role.MEASURE, "stale", Role.MEASURE) ); engine.getContext().setAttribute("ds1", dataset1, ScriptContext.ENGINE_SCOPE); @@ -219,16 +219,15 @@ public void testLeftJoinMixedStructure() throws ScriptException { Dataset joinData = (Dataset) engine.getBindings(ScriptContext.ENGINE_SCOPE).get("joinData"); // Build Map with null value - Map thirdLine = new LinkedHashMap<>() {{ - put("id", "K003"); - put("measure1", 3L); - put("measure2", 7L); - put("color", null); - }}; + Map thirdLine = new LinkedHashMap<>(); + thirdLine.put("id", "K003"); + thirdLine.put("measure1", 3L); + thirdLine.put("measure2", 7L); + thirdLine.put("color", null); assertThat(joinData.getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "K001", "measure1", 1L, "measure2", 9L, "color", "yellow"), - Map.of("id", "K002", "measure1", 2L, "measure2", 8L, "color", "blue"), + Java8Helpers.mapOf("id", "K001", "measure1", 1L, "measure2", 9L, "color", "yellow"), + Java8Helpers.mapOf("id", "K002", "measure1", 2L, "measure2", 8L, "color", "blue"), thirdLine ); } @@ -240,7 +239,7 @@ public void testInnerJoin() throws ScriptException { engine.getContext().setAttribute("ds_2", ds2, ScriptContext.ENGINE_SCOPE); engine.eval("result := inner_join(ds_1[keep id1, id2, m1] as ds1, ds_2 as ds2);"); - var result = (Dataset) engine.getContext().getAttribute("result"); + Dataset result = (Dataset) engine.getContext().getAttribute("result"); assertThat(result.getColumnNames()).containsExactlyInAnyOrder( "id1", "id2", "m1", "m2" ); @@ -278,8 +277,8 @@ public void testInnerJoin() throws ScriptException { public void testFullJoin() throws ScriptException { ScriptContext context = engine.getContext(); - var ds1 = new InMemoryDataset( - List.of( + InMemoryDataset ds1 = new InMemoryDataset( + Java8Helpers.listOf( new Structured.Component("id", String.class, Role.IDENTIFIER), new Structured.Component("m1", Long.class, Role.MEASURE) ), @@ -288,8 +287,8 @@ public void testFullJoin() throws ScriptException { Arrays.asList("d", 3L) ); - var ds2 = new InMemoryDataset( - List.of( + InMemoryDataset ds2 = new InMemoryDataset( + Java8Helpers.listOf( new Structured.Component("id", String.class, Role.IDENTIFIER), new Structured.Component("m1", Long.class, Role.MEASURE) ), @@ -298,8 +297,8 @@ public void testFullJoin() throws ScriptException { Arrays.asList("c", 6L) ); - var ds3 = new InMemoryDataset( - List.of( + InMemoryDataset ds3 = new InMemoryDataset( + Java8Helpers.listOf( new Structured.Component("id", String.class, Role.IDENTIFIER), new Structured.Component("m1", Long.class, Role.MEASURE) ), @@ -313,7 +312,7 @@ public void testFullJoin() throws ScriptException { engine.eval("result := full_join(ds_1 as ds1, ds_2 as ds2, ds_3 as ds3);"); - var result = (Dataset) context.getAttribute("result"); + Dataset result = (Dataset) context.getAttribute("result"); assertThat(result.getDataStructure().values()).containsExactly( new Structured.Component("id", String.class, Role.IDENTIFIER), @@ -333,14 +332,14 @@ public void testFullJoin() throws ScriptException { public void testCrossJoin() throws ScriptException { InMemoryDataset ds3 = new InMemoryDataset( - List.of( - List.of("a", 1L, 1L), - List.of("a", 2L, 2L), - List.of("a", 3L, 3L), - List.of("b", 1L, 3L), - List.of("b", 2L, 4L) + Java8Helpers.listOf( + Java8Helpers.listOf("a", 1L, 1L), + Java8Helpers.listOf("a", 2L, 2L), + Java8Helpers.listOf("a", 3L, 3L), + Java8Helpers.listOf("b", 1L, 3L), + Java8Helpers.listOf("b", 2L, 4L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("id1", String.class, Role.IDENTIFIER), new Structured.Component("id2", Long.class, Role.IDENTIFIER), new Structured.Component("m1", Long.class, Role.MEASURE) @@ -348,11 +347,11 @@ public void testCrossJoin() throws ScriptException { ); InMemoryDataset ds4 = new InMemoryDataset( - List.of( - List.of("a", 1L, 7L), - List.of("a", 2L, 8L) + Java8Helpers.listOf( + Java8Helpers.listOf("a", 1L, 7L), + Java8Helpers.listOf("a", 2L, 8L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("id1", String.class, Role.IDENTIFIER), new Structured.Component("id2", Long.class, Role.IDENTIFIER), new Structured.Component("m2", Long.class, Role.MEASURE) @@ -363,7 +362,7 @@ public void testCrossJoin() throws ScriptException { engine.getContext().setAttribute("ds_4", ds4, ScriptContext.ENGINE_SCOPE); engine.eval("result := cross_join(ds_3 as ds3, ds_4 as ds4);"); - var result = (Dataset) engine.getContext().getAttribute("result"); + Dataset result = (Dataset) engine.getContext().getAttribute("result"); assertThat(result.getColumnNames()).containsExactly( "ds3#id1", "ds3#id2", "ds4#id1", "ds4#id2", "m1", "m2" ); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java index c8fd76ed9..308c50e4a 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/NumericFunctionsTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.assertj.core.data.Percentage; import org.junit.jupiter.api.BeforeEach; @@ -11,7 +12,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -85,9 +85,9 @@ public void testCeil() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := ceil(ds[keep id, long1, double1]);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "long1", 150L, "double1", 2L), - Map.of("id", "Nico", "long1", 20L, "double1", 3L), - Map.of("id", "Franck", "long1", 100L, "double1", -1L) + Java8Helpers.mapOf("id", "Hadrien", "long1", 150L, "double1", 2L), + Java8Helpers.mapOf("id", "Nico", "long1", 20L, "double1", 3L), + Java8Helpers.mapOf("id", "Franck", "long1", 100L, "double1", -1L) ); assertThatThrownBy(() -> { engine.eval("e := ceil(\"ko\");"); @@ -110,9 +110,9 @@ public void testFloor() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := floor(ds[keep id, double1]);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "double1", 1L), - Map.of("id", "Nico", "double1", 2L), - Map.of("id", "Franck", "double1", -2L) + Java8Helpers.mapOf("id", "Hadrien", "double1", 1L), + Java8Helpers.mapOf("id", "Nico", "double1", 2L), + Java8Helpers.mapOf("id", "Franck", "double1", -2L) ); assertThatThrownBy(() -> { engine.eval("e := floor(\"ko\");"); @@ -131,9 +131,9 @@ public void testAbs() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := abs(ds[keep id, double1]);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "double1", 1.1D), - Map.of("id", "Nico", "double1", 2.2D), - Map.of("id", "Franck", "double1", 1.21D) + Java8Helpers.mapOf("id", "Hadrien", "double1", 1.1D), + Java8Helpers.mapOf("id", "Nico", "double1", 2.2D), + Java8Helpers.mapOf("id", "Franck", "double1", 1.21D) ); assertThatThrownBy(() -> { engine.eval("c := abs(\"ko\");"); @@ -156,9 +156,9 @@ public void testExp() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := floor(exp(ds[keep id, double1, long1]));"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "double1", 3L, "long1", 9223372036854775807L), - Map.of("id", "Nico", "double1", 9L, "long1", 485165195L), - Map.of("id", "Franck", "double1", 0L, "long1", 9223372036854775807L) + Java8Helpers.mapOf("id", "Hadrien", "double1", 3L, "long1", 9223372036854775807L), + Java8Helpers.mapOf("id", "Nico", "double1", 9L, "long1", 485165195L), + Java8Helpers.mapOf("id", "Franck", "double1", 0L, "long1", 9223372036854775807L) ); assertThatThrownBy(() -> { engine.eval("e := exp(\"ko\");"); @@ -181,9 +181,9 @@ public void testLn() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds2, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := floor(ln(abs(ds[keep id, double1])));"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "double1", 0L), - Map.of("id", "Nico", "double1", 0L), - Map.of("id", "Franck", "double1", 0L) + Java8Helpers.mapOf("id", "Hadrien", "double1", 0L), + Java8Helpers.mapOf("id", "Nico", "double1", 0L), + Java8Helpers.mapOf("id", "Franck", "double1", 0L) ); assertThatThrownBy(() -> { engine.eval("e := ln(\"ko\");"); @@ -208,10 +208,10 @@ public void testRound() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := round(ds[keep id, long1, double2], 1);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 30.0D, "double2", 1.2D), - Map.of("id", "Hadrien", "long1", 10.0D, "double2", 10.1D), - Map.of("id", "Nico", "long1", 20.0D, "double2", 21.1D), - Map.of("id", "Franck", "long1", 100.0D, "double2", 100.9D) + Java8Helpers.mapOf("id", "Toto", "long1", 30.0D, "double2", 1.2D), + Java8Helpers.mapOf("id", "Hadrien", "long1", 10.0D, "double2", 10.1D), + Java8Helpers.mapOf("id", "Nico", "long1", 20.0D, "double2", 21.1D), + Java8Helpers.mapOf("id", "Franck", "long1", 100.0D, "double2", 100.9D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); assertThatThrownBy(() -> { @@ -241,10 +241,10 @@ public void testTrunc() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := trunc(ds[keep id, long1, double2], 1);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 30.0D, "double2", 1.2D), - Map.of("id", "Hadrien", "long1", 10.0D, "double2", 10.1D), - Map.of("id", "Nico", "long1", 20.0D, "double2", 21.1D), - Map.of("id", "Franck", "long1", 100.0D, "double2", 100.9D) + Java8Helpers.mapOf("id", "Toto", "long1", 30.0D, "double2", 1.2D), + Java8Helpers.mapOf("id", "Hadrien", "long1", 10.0D, "double2", 10.1D), + Java8Helpers.mapOf("id", "Nico", "long1", 20.0D, "double2", 21.1D), + Java8Helpers.mapOf("id", "Franck", "long1", 100.0D, "double2", 100.9D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); assertThatThrownBy(() -> { @@ -268,10 +268,10 @@ public void testSqrt() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := round(sqrt(ds[keep id, long1, double2]));"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 5.0D, "double2", 1D), - Map.of("id", "Hadrien", "long1", 3.0D, "double2", 3.0D), - Map.of("id", "Nico", "long1", 4.0D, "double2", 5.0D), - Map.of("id", "Franck", "long1", 10.0D, "double2", 10.0D) + Java8Helpers.mapOf("id", "Toto", "long1", 5.0D, "double2", 1D), + Java8Helpers.mapOf("id", "Hadrien", "long1", 3.0D, "double2", 3.0D), + Java8Helpers.mapOf("id", "Nico", "long1", 4.0D, "double2", 5.0D), + Java8Helpers.mapOf("id", "Franck", "long1", 10.0D, "double2", 10.0D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); @@ -298,10 +298,10 @@ public void testMod() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := trunc(mod(ds[keep id, long1, double2], 2), 1);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 0.0D, "double2", 1.2D), - Map.of("id", "Hadrien", "long1", 0.0D, "double2", 0.1D), - Map.of("id", "Nico", "long1", 0.0D, "double2", 1.1D), - Map.of("id", "Franck", "long1", 0.0D, "double2", 0.9D) + Java8Helpers.mapOf("id", "Toto", "long1", 0.0D, "double2", 1.2D), + Java8Helpers.mapOf("id", "Hadrien", "long1", 0.0D, "double2", 0.1D), + Java8Helpers.mapOf("id", "Nico", "long1", 0.0D, "double2", 1.1D), + Java8Helpers.mapOf("id", "Franck", "long1", 0.0D, "double2", 0.9D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); @@ -328,10 +328,10 @@ public void testPower() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := trunc(power(ds[keep id, long1, double2], 2), 1);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 900.0D, "double2", 1.4D), - Map.of("id", "Hadrien", "long1", 100.0D, "double2", 102.2D), - Map.of("id", "Nico", "long1", 400.0D, "double2", 445.2D), - Map.of("id", "Franck", "long1", 10000.0D, "double2", 10180.8D) + Java8Helpers.mapOf("id", "Toto", "long1", 900.0D, "double2", 1.4D), + Java8Helpers.mapOf("id", "Hadrien", "long1", 100.0D, "double2", 102.2D), + Java8Helpers.mapOf("id", "Nico", "long1", 400.0D, "double2", 445.2D), + Java8Helpers.mapOf("id", "Franck", "long1", 10000.0D, "double2", 10180.8D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); @@ -356,10 +356,10 @@ public void testLog() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := trunc(log(ds[keep id, long1, double2], 2), 1);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "long1", 4.9D, "double2", 0.2D), - Map.of("id", "Hadrien", "long1", 3.3D, "double2", 3.3D), - Map.of("id", "Nico", "long1", 4.3D, "double2", 4.3D), - Map.of("id", "Franck", "long1", 6.6D, "double2", 6.6D) + Java8Helpers.mapOf("id", "Toto", "long1", 4.9D, "double2", 0.2D), + Java8Helpers.mapOf("id", "Hadrien", "long1", 3.3D, "double2", 3.3D), + Java8Helpers.mapOf("id", "Nico", "long1", 4.3D, "double2", 4.3D), + Java8Helpers.mapOf("id", "Franck", "long1", 6.6D, "double2", 6.6D) ); assertThat(((Dataset) res).getDataStructure().get("long1").getType()).isEqualTo(Double.class); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java index 2a77832c6..e7b12c512 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/SetFunctionsVisitorTest.java @@ -1,17 +1,14 @@ package fr.insee.vtl.engine.visitors.expression.functions; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import javax.script.ScriptContext; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; +import javax.script.*; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import static fr.insee.vtl.model.Structured.Component; @@ -31,24 +28,24 @@ public void setUp() { public void testUnionIncompatibleStructure() { InMemoryDataset dataset1 = new InMemoryDataset( - List.of(), - List.of( + Java8Helpers.listOf(), + Java8Helpers.listOf( new Component("name", String.class, Dataset.Role.IDENTIFIER, null), new Component("age", Long.class, Dataset.Role.MEASURE, null), new Component("weight", Long.class, Dataset.Role.MEASURE, null) ) ); InMemoryDataset dataset2 = new InMemoryDataset( - List.of(), - List.of( + Java8Helpers.listOf(), + Java8Helpers.listOf( new Component("age", Long.class, Dataset.Role.MEASURE), new Component("name", String.class, Dataset.Role.IDENTIFIER), new Component("weight", Long.class, Dataset.Role.MEASURE) ) ); InMemoryDataset dataset3 = new InMemoryDataset( - List.of(), - List.of( + Java8Helpers.listOf(), + Java8Helpers.listOf( new Component("name2", String.class, Dataset.Role.IDENTIFIER), new Component("age", Long.class, Dataset.Role.MEASURE), new Component("weight", Long.class, Dataset.Role.MEASURE) @@ -67,13 +64,13 @@ public void testUnionIncompatibleStructure() { public void testUnionSimple() throws ScriptException { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); ScriptContext context = engine.getContext(); context.getBindings(ScriptContext.ENGINE_SCOPE).put("ds1", dataset); @@ -90,10 +87,10 @@ public void testUnionSimple() throws ScriptException { @Test public void testUnionDifferentStructure() throws ScriptException { - var structure = new Structured.DataStructure(Map.of("id", String.class), Map.of("id", Dataset.Role.IDENTIFIER)); - InMemoryDataset ds1 = new InMemoryDataset(structure, List.of("1"), List.of("2"), List.of("3"), List.of("4")); - InMemoryDataset ds2 = new InMemoryDataset(structure, List.of("3"), List.of("4"), List.of("5"), List.of("6")); - var bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); + Structured.DataStructure structure = new Structured.DataStructure(Java8Helpers.mapOf("id", String.class), Java8Helpers.mapOf("id", Dataset.Role.IDENTIFIER)); + InMemoryDataset ds1 = new InMemoryDataset(structure, Java8Helpers.listOf("1"), Java8Helpers.listOf("2"), Java8Helpers.listOf("3"), Java8Helpers.listOf("4")); + InMemoryDataset ds2 = new InMemoryDataset(structure, Java8Helpers.listOf("3"), Java8Helpers.listOf("4"), Java8Helpers.listOf("5"), Java8Helpers.listOf("6")); + Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); bindings.put("ds1", ds1); bindings.put("ds2", ds2); @@ -106,40 +103,40 @@ public void testUnionDifferentStructure() throws ScriptException { "ds3 := union(ds1, ds2);\n" + "ds4 := union(ds2, ds1);"); - var ds3 = (Dataset) bindings.get("ds3"); - var ds4 = (Dataset) bindings.get("ds4"); + Dataset ds3 = (Dataset) bindings.get("ds3"); + Dataset ds4 = (Dataset) bindings.get("ds4"); - var onlyA = ds3.getDataAsMap().stream() + List onlyA = ds3.getDataAsMap().stream() .map(m -> m.get("A")) .distinct() .collect(Collectors.toList()); assertThat(onlyA).containsExactly("A"); - var onlyB = ds4.getDataAsMap().stream() + List onlyB = ds4.getDataAsMap().stream() .map(m -> m.get("B")) .distinct() .collect(Collectors.toList()); assertThat(onlyB).containsExactly("B"); - var onlyAList = ds3.getDataAsList().stream() + List onlyAList = ds3.getDataAsList().stream() .map(l -> l.get(ds3.getDataStructure().indexOfKey("A"))) .distinct() .collect(Collectors.toList()); assertThat(onlyAList).containsExactly("A"); - var onlyBList = ds4.getDataAsList().stream() + List onlyBList = ds4.getDataAsList().stream() .map(l -> l.get(ds4.getDataStructure().indexOfKey("B"))) .distinct() .collect(Collectors.toList()); assertThat(onlyBList).containsExactly("B"); - var onlyADatapoint = ds3.getDataPoints().stream() + List onlyADatapoint = ds3.getDataPoints().stream() .map(d -> d.get("A")) .distinct() .collect(Collectors.toList()); assertThat(onlyADatapoint).containsExactly("A"); - var onlyBDatapoint = ds4.getDataPoints().stream() + List onlyBDatapoint = ds4.getDataPoints().stream() .map(d -> d.get("B")) .distinct() .collect(Collectors.toList()); @@ -151,20 +148,20 @@ public void testUnionDifferentStructure() throws ScriptException { public void testUnion() throws ScriptException { InMemoryDataset dataset1 = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); InMemoryDataset dataset2 = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "weight", 11L, "age", 10L), - Map.of("name", "Franck", "weight", 9L, "age", 12L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "weight", 11L, "age", 10L), + Java8Helpers.mapOf("name", "Franck", "weight", 9L, "age", 12L) ), - Map.of("name", String.class, "weight", Long.class, "age", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "weight", Long.class, "age", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); ScriptContext context = engine.getContext(); context.getBindings(ScriptContext.ENGINE_SCOPE).put("ds1", dataset1); @@ -174,9 +171,9 @@ public void testUnion() throws ScriptException { Object result = engine.getContext().getAttribute("result"); assertThat(result).isInstanceOf(Dataset.class); assertThat(((Dataset) result).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ); } @@ -185,28 +182,28 @@ public void testUnion() throws ScriptException { public void testUnionMultiple() throws ScriptException { InMemoryDataset dataset1 = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); InMemoryDataset dataset2 = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien2", "age", 10L, "weight", 11L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien2", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); InMemoryDataset dataset3 = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Franck2", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Franck2", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); ScriptContext context = engine.getContext(); @@ -218,11 +215,11 @@ public void testUnionMultiple() throws ScriptException { Object result = engine.getContext().getAttribute("result"); assertThat(result).isInstanceOf(Dataset.class); assertThat(((Dataset) result).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L), - Map.of("name", "Franck2", "age", 12L, "weight", 9L), - Map.of("name", "Hadrien2", "age", 10L, "weight", 11L) + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L), + Java8Helpers.mapOf("name", "Franck2", "age", 12L, "weight", 9L), + Java8Helpers.mapOf("name", "Hadrien2", "age", 10L, "weight", 11L) ); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java index 244b3f78e..1a85e1612 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/StringFunctionsTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.engine.exceptions.FunctionNotFoundException; import fr.insee.vtl.engine.samples.DatasetSamples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -10,7 +11,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -93,23 +93,23 @@ public void testUnaryStringFunction() throws ScriptException { "dsLower := lower(ds[keep id, string1]); " + "dsLen := length(ds[keep id, string1]);"); assertThat(((Dataset) context.getAttribute("dsTrim")).getDataAsMap().get(0)).isEqualTo( - Map.of("id", "Toto", "string1", "toto") + Java8Helpers.mapOf("id", "Toto", "string1", "toto") ); assertThat(((Dataset) context.getAttribute("dsLTrim")).getDataAsMap().get(0)).isEqualTo( - Map.of("id", "Toto", "string1", "toto") + Java8Helpers.mapOf("id", "Toto", "string1", "toto") ); assertThat(((Dataset) context.getAttribute("dsRTrim")).getDataAsMap().get(0)).isEqualTo( - Map.of("id", "Toto", "string1", "toto") + Java8Helpers.mapOf("id", "Toto", "string1", "toto") ); assertThat(((Dataset) context.getAttribute("dsUpper")).getDataAsMap().get(0)).isEqualTo( - Map.of("id", "Toto", "string1", "TOTO") + Java8Helpers.mapOf("id", "Toto", "string1", "TOTO") ); assertThat(((Dataset) context.getAttribute("dsLower")).getDataAsMap().get(0)).isEqualTo( - Map.of("id", "Toto", "string1", "toto") + Java8Helpers.mapOf("id", "Toto", "string1", "toto") ); assertThat(((Dataset) context.getAttribute("dsLower")).getDataStructure().get("string1").getType()).isEqualTo(String.class); assertThat(((Dataset) context.getAttribute("dsLen")).getDataAsMap().get(0)).isEqualTo( - Map.of("id", "Toto", "string1", 4L) + Java8Helpers.mapOf("id", "Toto", "string1", 4L) ); assertThat(((Dataset) context.getAttribute("dsLen")).getDataStructure().get("string1").getType()).isEqualTo(Long.class); } @@ -131,10 +131,10 @@ public void testSubstrAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := substr(ds[keep id, string1, string2], 2, 4);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "string1", "oto", "string2", ""), - Map.of("id", "Hadrien", "string1", "adri", "string2", ""), - Map.of("id", "Nico", "string1", "ico", "string2", ""), - Map.of("id", "Franck", "string1", "ranc", "string2", "") + Java8Helpers.mapOf("id", "Toto", "string1", "oto", "string2", ""), + Java8Helpers.mapOf("id", "Hadrien", "string1", "adri", "string2", ""), + Java8Helpers.mapOf("id", "Nico", "string1", "ico", "string2", ""), + Java8Helpers.mapOf("id", "Franck", "string1", "ranc", "string2", "") ); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(String.class); } @@ -150,10 +150,10 @@ public void testReplaceAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := replace(ds[keep id, string1, string2], \"o\", \"O\");"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "string1", "tOtO", "string2", "t"), - Map.of("id", "Hadrien", "string1", "hadrien", "string2", "k"), - Map.of("id", "Nico", "string1", "nicO", "string2", "l"), - Map.of("id", "Franck", "string1", "franck", "string2", "c") + Java8Helpers.mapOf("id", "Toto", "string1", "tOtO", "string2", "t"), + Java8Helpers.mapOf("id", "Hadrien", "string1", "hadrien", "string2", "k"), + Java8Helpers.mapOf("id", "Nico", "string1", "nicO", "string2", "l"), + Java8Helpers.mapOf("id", "Franck", "string1", "franck", "string2", "c") ); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(String.class); @@ -182,10 +182,10 @@ public void testInstrAtom() throws ScriptException { context.setAttribute("ds", DatasetSamples.ds1, ScriptContext.ENGINE_SCOPE); Object res = engine.eval("res := instr(ds[keep id, string1, string2], \"o\", 0, 2);"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Toto", "string1", 4L, "string2", 0L), - Map.of("id", "Hadrien", "string1", 0L, "string2", 0L), - Map.of("id", "Nico", "string1", 0L, "string2", 0L), - Map.of("id", "Franck", "string1", 0L, "string2", 0L) + Java8Helpers.mapOf("id", "Toto", "string1", 4L, "string2", 0L), + Java8Helpers.mapOf("id", "Hadrien", "string1", 0L, "string2", 0L), + Java8Helpers.mapOf("id", "Nico", "string1", 0L, "string2", 0L), + Java8Helpers.mapOf("id", "Franck", "string1", 0L, "string2", 0L) ); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(Long.class); diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsTest.java index b1b16e40e..3bd9c7f15 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/TimeFunctionsTest.java @@ -2,6 +2,7 @@ import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; +import fr.insee.vtl.model.utils.Java8Helpers; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -12,7 +13,6 @@ import javax.script.ScriptException; import java.time.Instant; import java.time.OffsetDateTime; -import java.util.List; import static fr.insee.vtl.model.Dataset.Component; import static fr.insee.vtl.model.Dataset.Role; @@ -38,21 +38,21 @@ public void testCurrentDateAtom() throws ScriptException { @Disabled @Test public void testFlowToStock() throws ScriptException { - var ds = new InMemoryDataset( - List.of( + InMemoryDataset ds = new InMemoryDataset( + Java8Helpers.listOf( new Component("id1", String.class, Role.IDENTIFIER), new Component("id2", OffsetDateTime.class, Role.IDENTIFIER), new Component("me1", Long.class, Role.MEASURE) ), - List.of("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 2L), - List.of("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 5L), - List.of("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), -3L), - List.of("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 9L), - List.of("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 4L) + Java8Helpers.listOf("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 2L), + Java8Helpers.listOf("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 5L), + Java8Helpers.listOf("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), -3L), + Java8Helpers.listOf("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 9L), + Java8Helpers.listOf("A", OffsetDateTime.parse("2010-01-01T00:00:00+01:00"), 4L) ); engine.put("ds", ds); engine.eval("res := flow_to_stock(ds);"); - var actual = (Dataset) engine.get("res"); + Dataset actual = (Dataset) engine.get("res"); actual.getDataAsMap().forEach(System.out::println); assertThat(engine.get("r")).isNotNull(); } diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsTest.java index f6507a3bc..745646124 100644 --- a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsTest.java +++ b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.engine.visitors.expression.functions; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -9,20 +10,19 @@ import javax.script.ScriptContext; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; -import java.util.List; import static org.assertj.core.api.Assertions.assertThatThrownBy; public class ValidationFunctionsTest { private final Dataset dataset = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", 10L), - List.of("2011", "I", "DEBIT", -2L), - List.of("2012", "I", "CREDIT", 10L), - List.of("2012", "I", "DEBIT", 2L) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", 10L), + Java8Helpers.listOf("2011", "I", "DEBIT", -2L), + Java8Helpers.listOf("2012", "I", "CREDIT", 10L), + Java8Helpers.listOf("2012", "I", "DEBIT", 2L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), @@ -31,13 +31,13 @@ public class ValidationFunctionsTest { ); private final Dataset dsExprOk = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", true), - List.of("2011", "I", "DEBIT", false), - List.of("2012", "I", "CREDIT", false), - List.of("2012", "I", "DEBIT", true) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", true), + Java8Helpers.listOf("2011", "I", "DEBIT", false), + Java8Helpers.listOf("2012", "I", "CREDIT", false), + Java8Helpers.listOf("2012", "I", "DEBIT", true) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), @@ -45,13 +45,13 @@ public class ValidationFunctionsTest { ) ); private final Dataset dsExprKo1 = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", true, true), - List.of("2011", "I", "DEBIT", false, true), - List.of("2012", "I", "CREDIT", false, true), - List.of("2012", "I", "DEBIT", true, true) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", true, true), + Java8Helpers.listOf("2011", "I", "DEBIT", false, true), + Java8Helpers.listOf("2012", "I", "CREDIT", false, true), + Java8Helpers.listOf("2012", "I", "DEBIT", true, true) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), @@ -60,13 +60,13 @@ public class ValidationFunctionsTest { ) ); private final Dataset dsExprKo2 = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", 1L), - List.of("2011", "I", "DEBIT", 1L), - List.of("2012", "I", "CREDIT", 1L), - List.of("2012", "I", "DEBIT", 1L) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", 1L), + Java8Helpers.listOf("2011", "I", "DEBIT", 1L), + Java8Helpers.listOf("2012", "I", "CREDIT", 1L), + Java8Helpers.listOf("2012", "I", "DEBIT", 1L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), @@ -74,13 +74,13 @@ public class ValidationFunctionsTest { ) ); private final Dataset dsImbalanceKo = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", 1L, 1L), - List.of("2011", "I", "DEBIT", 2L, 1L), - List.of("2012", "I", "CREDIT", 2L, 1L), - List.of("2012", "I", "DEBIT", 3L, 1L) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", 1L, 1L), + Java8Helpers.listOf("2011", "I", "DEBIT", 2L, 1L), + Java8Helpers.listOf("2012", "I", "CREDIT", 2L, 1L), + Java8Helpers.listOf("2012", "I", "DEBIT", 3L, 1L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), diff --git a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/BindingsDeserializer.java b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/BindingsDeserializer.java index 06da7d790..ae14e8768 100644 --- a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/BindingsDeserializer.java +++ b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/BindingsDeserializer.java @@ -33,15 +33,15 @@ protected BindingsDeserializer() { @Override public Bindings deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - var bindings = new SimpleBindings(); - var token = p.currentToken(); + SimpleBindings bindings = new SimpleBindings(); + JsonToken token = p.currentToken(); if (!token.isStructStart()) { ctxt.handleUnexpectedToken(Bindings.class, p); } while (p.nextToken() == JsonToken.FIELD_NAME) { - var name = p.currentName(); - var value = p.nextValue(); + String name = p.currentName(); + JsonToken value = p.nextValue(); if (value.isStructStart()) { // Dataset bindings.put(name, ctxt.readValue(p, Dataset.class)); diff --git a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentDeserializer.java b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentDeserializer.java index 1df28eef1..139fe8039 100644 --- a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentDeserializer.java +++ b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentDeserializer.java @@ -2,9 +2,11 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import java.io.IOException; import java.util.Map; @@ -14,7 +16,7 @@ */ public class ComponentDeserializer extends StdDeserializer { - private static final Map> TYPES = Map.of( + private static final Map> TYPES = Java8Helpers.mapOf( "STRING", String.class, "INTEGER", Long.class, "NUMBER", Double.class, @@ -38,11 +40,11 @@ protected ComponentDeserializer() { */ @Override public Structured.Component deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - var node = ctxt.readTree(p); - var name = node.get("name").asText(); - var type = node.get("type").asText(); - var role = Dataset.Role.valueOf(node.get("role").asText()); - var nullable = node.get("nullable") != null ? node.get("nullable").asBoolean() : null; + JsonNode node = ctxt.readTree(p); + String name = node.get("name").asText(); + String type = node.get("type").asText(); + Dataset.Role role = Dataset.Role.valueOf(node.get("role").asText()); + Boolean nullable = node.get("nullable") != null ? node.get("nullable").asBoolean() : null; return new Dataset.Component(name, asType(type), role, nullable); } diff --git a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentSerializer.java b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentSerializer.java index e891de8f9..26acbc7a3 100644 --- a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentSerializer.java +++ b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/ComponentSerializer.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import java.io.IOException; import java.util.Map; @@ -13,7 +14,7 @@ */ public class ComponentSerializer extends StdSerializer { - private static final Map, String> TYPES = Map.of( + private static final Map, String> TYPES = Java8Helpers.mapOf( String.class, "STRING", Long.class, "INTEGER", Double.class, "NUMBER", diff --git a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/DatasetDeserializer.java b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/DatasetDeserializer.java index aec5883e7..6ad712874 100644 --- a/vtl-jackson/src/main/java/fr/insee/vtl/jackson/DatasetDeserializer.java +++ b/vtl-jackson/src/main/java/fr/insee/vtl/jackson/DatasetDeserializer.java @@ -5,10 +5,12 @@ import com.fasterxml.jackson.core.ObjectCodec; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.type.CollectionLikeType; import com.fasterxml.jackson.databind.util.TokenBuffer; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import java.io.IOException; import java.util.ArrayList; @@ -24,8 +26,8 @@ */ public class DatasetDeserializer extends StdDeserializer { - private static final Set STRUCTURE_NAMES = Set.of("structure", "dataStructure"); - private static final Set DATAPOINT_NAMES = Set.of("data", "dataPoints"); + private static final Set STRUCTURE_NAMES = Java8Helpers.setOf("structure", "dataStructure"); + private static final Set DATAPOINT_NAMES = Java8Helpers.setOf("data", "dataPoints"); /** * Base constructor. @@ -81,20 +83,20 @@ private void convertDataPoints(JsonParser p, List> objects, List object : objects) { for (int i = 0; i < object.size(); i++) { - var converted = deserializers.get(i).convert(p, object.get(i)); + Object converted = deserializers.get(i).convert(p, object.get(i)); object.set(i, converted); } } } private List> deserializeDataPoints(JsonParser p, DeserializationContext ctxt, List components) throws IOException { - var fieldName = p.currentName(); + String fieldName = p.currentName(); if (!DATAPOINT_NAMES.contains(fieldName)) { ctxt.handleUnexpectedToken(Dataset.class, p); } // row != array. - var token = p.nextToken(); + JsonToken token = p.nextToken(); if (token != JsonToken.START_ARRAY) { ctxt.handleUnexpectedToken(Dataset.class, p); } @@ -107,8 +109,8 @@ private List> deserializeDataPoints(JsonParser p, DeserializationCo List> dataPoints = new ArrayList<>(); while (p.nextToken() == JsonToken.START_ARRAY) { - var row = new ArrayList<>(); - for (var deserializer : deserializers) { + List row = new ArrayList<>(); + for (PointDeserializer deserializer : deserializers) { p.nextValue(); row.add(deserializer.deserialize(p, ctxt)); } @@ -124,26 +126,26 @@ private List> deserializeDataPoints(JsonParser p, DeserializationCo } private List> deserializeUncheckedDataPoint(JsonParser p, DeserializationContext ctxt) throws IOException { - var fieldName = p.currentName(); + String fieldName = p.currentName(); if (!DATAPOINT_NAMES.contains(fieldName)) { ctxt.handleUnexpectedToken(Dataset.class, p); } p.nextToken(); - var listOfComponentType = ctxt.getTypeFactory().constructCollectionLikeType(List.class, List.class); + CollectionLikeType listOfComponentType = ctxt.getTypeFactory().constructCollectionLikeType(List.class, List.class); return ctxt.readValue(p, listOfComponentType); } private List deserializeStructure(JsonParser p, DeserializationContext ctxt) throws IOException { - var fieldName = p.currentName(); + String fieldName = p.currentName(); if (!STRUCTURE_NAMES.contains(fieldName)) { ctxt.handleUnexpectedToken(Dataset.class, p); } p.nextToken(); - var listOfComponentType = ctxt.getTypeFactory().constructCollectionLikeType(List.class, Dataset.Component.class); + CollectionLikeType listOfComponentType = ctxt.getTypeFactory().constructCollectionLikeType(List.class, Dataset.Component.class); return ctxt.readValue(p, listOfComponentType); } @@ -160,7 +162,7 @@ Object convert(JsonParser p, Object object) throws IOException { } Object convert(ObjectCodec codec, Object node, Class type) throws IOException { - var buf = new TokenBuffer(codec, false); + TokenBuffer buf = new TokenBuffer(codec, false); codec.writeValue(buf, node); return buf.asParser().readValueAs(type); } diff --git a/vtl-jackson/src/main/java/module-info.java b/vtl-jackson/src/main/java/module-info.java deleted file mode 100644 index c183e09c0..000000000 --- a/vtl-jackson/src/main/java/module-info.java +++ /dev/null @@ -1,10 +0,0 @@ -/** - * This module exposes a Jackson module for the VTL model. - */ - -module fr.insee.vtl.jackson { - requires java.scripting; - requires fr.insee.vtl.model; - requires com.fasterxml.jackson.databind; - exports fr.insee.vtl.jackson; -} \ No newline at end of file diff --git a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/BindingsDeserializerTest.java b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/BindingsDeserializerTest.java index 43d747eb2..d555da765 100644 --- a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/BindingsDeserializerTest.java +++ b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/BindingsDeserializerTest.java @@ -5,6 +5,7 @@ import javax.script.Bindings; import java.io.IOException; +import java.io.InputStream; import static org.assertj.core.api.Assertions.assertThat; @@ -13,9 +14,9 @@ public class BindingsDeserializerTest extends AbstractMapperTest { @Test public void testSupportsBindinds() throws IOException { - var jsonStream = getClass().getResourceAsStream("/bindings.json"); + InputStream jsonStream = getClass().getResourceAsStream("/bindings.json"); - var bindings = mapper.readValue(jsonStream, Bindings.class); + Bindings bindings = mapper.readValue(jsonStream, Bindings.class); assertThat(bindings) .containsEntry("string", "string") diff --git a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentDeserializerTest.java b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentDeserializerTest.java index 035fc79f7..69e8b37d4 100644 --- a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentDeserializerTest.java +++ b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentDeserializerTest.java @@ -12,13 +12,13 @@ public class ComponentDeserializerTest extends AbstractMapperTest { @Test public void testCanDeserialize() throws JsonProcessingException { - var stringComponent = new Dataset.Component("NAME", String.class, Dataset.Role.IDENTIFIER, null); - var measureComponent = new Dataset.Component("NAME", String.class, Dataset.Role.MEASURE, false); - var attributeComponent = new Dataset.Component("NAME", String.class, Dataset.Role.ATTRIBUTE, true); + Structured.Component stringComponent = new Dataset.Component("NAME", String.class, Dataset.Role.IDENTIFIER, null); + Structured.Component measureComponent = new Dataset.Component("NAME", String.class, Dataset.Role.MEASURE, false); + Structured.Component attributeComponent = new Dataset.Component("NAME", String.class, Dataset.Role.ATTRIBUTE, true); - var doubleComponent = new Dataset.Component("NAME", Double.class, Dataset.Role.IDENTIFIER, true); - var longComponent = new Dataset.Component("NAME", Long.class, Dataset.Role.IDENTIFIER, true); - var booleanComponent = new Dataset.Component("NAME", Boolean.class, Dataset.Role.IDENTIFIER, true); + Structured.Component doubleComponent = new Dataset.Component("NAME", Double.class, Dataset.Role.IDENTIFIER, true); + Structured.Component longComponent = new Dataset.Component("NAME", Long.class, Dataset.Role.IDENTIFIER, true); + Structured.Component booleanComponent = new Dataset.Component("NAME", Boolean.class, Dataset.Role.IDENTIFIER, true); Dataset.Component component; diff --git a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentSerializerTest.java b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentSerializerTest.java index d5ffb7ff0..dafa5662a 100644 --- a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentSerializerTest.java +++ b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/ComponentSerializerTest.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.vtl.model.Dataset; +import fr.insee.vtl.model.utils.Java8Helpers; import org.junit.jupiter.api.Test; import java.util.List; @@ -13,7 +14,7 @@ public class ComponentSerializerTest extends AbstractMapperTest { @Test public void testSerializeComponents() throws JsonProcessingException { - List cases = List.of( + List cases = Java8Helpers.listOf( new Dataset.Component("name", Long.class, Dataset.Role.IDENTIFIER), new Dataset.Component("name", Long.class, Dataset.Role.MEASURE), new Dataset.Component("name", Long.class, Dataset.Role.ATTRIBUTE), diff --git a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetDeserializerTest.java b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetDeserializerTest.java index 195cf2c37..0c7fe2b39 100644 --- a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetDeserializerTest.java +++ b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetDeserializerTest.java @@ -1,9 +1,11 @@ package fr.insee.vtl.jackson; import fr.insee.vtl.model.Dataset; +import fr.insee.vtl.model.utils.Java8Helpers; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -13,9 +15,9 @@ public class DatasetDeserializerTest extends AbstractMapperTest { @Test public void testDeserializeDataset() throws IOException { - var jsonStream = getClass().getResourceAsStream("/dataset.json"); + InputStream jsonStream = getClass().getResourceAsStream("/dataset.json"); - var dataset = mapper.readValue(jsonStream, Dataset.class); + Dataset dataset = mapper.readValue(jsonStream, Dataset.class); assertThat(dataset.getDataStructure().values()).containsExactly( new Dataset.Component("CONTINENT", String.class, Dataset.Role.IDENTIFIER, false), @@ -30,9 +32,9 @@ public void testDeserializeDataset() throws IOException { nulls.add(null); nulls.add(null); assertThat(dataset.getDataAsList()).containsExactly( - List.of("Europe", "France", 67063703L, 643.801), - List.of("Europe", "Norway", 5372191L, 385.203), - List.of("Oceania", "New Zealand", 4917000L, 268.021), + Java8Helpers.listOf("Europe", "France", 67063703L, 643.801), + Java8Helpers.listOf("Europe", "Norway", 5372191L, 385.203), + Java8Helpers.listOf("Oceania", "New Zealand", 4917000L, 268.021), nulls ); @@ -40,9 +42,9 @@ public void testDeserializeDataset() throws IOException { @Test public void testDeserializeInvertedDataset() throws IOException { - var jsonStream = getClass().getResourceAsStream("/dataset-inverted.json"); + InputStream jsonStream = getClass().getResourceAsStream("/dataset-inverted.json"); - var dataset = mapper.readValue(jsonStream, Dataset.class); + Dataset dataset = mapper.readValue(jsonStream, Dataset.class); assertThat(dataset.getDataStructure().values()).containsExactly( new Dataset.Component("CONTINENT", String.class, Dataset.Role.IDENTIFIER, false), @@ -57,9 +59,9 @@ public void testDeserializeInvertedDataset() throws IOException { nulls.add(null); nulls.add(null); assertThat(dataset.getDataAsList()).containsExactly( - List.of("Europe", "France", 67063703L, 643.801), - List.of("Europe", "Norway", 5372191L, 385.203), - List.of("Oceania", "New Zealand", 4917000L, 268.021), + Java8Helpers.listOf("Europe", "France", 67063703L, 643.801), + Java8Helpers.listOf("Europe", "Norway", 5372191L, 385.203), + Java8Helpers.listOf("Oceania", "New Zealand", 4917000L, 268.021), nulls ); diff --git a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetSerializerTest.java b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetSerializerTest.java index 72747854a..fa10e00f9 100644 --- a/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetSerializerTest.java +++ b/vtl-jackson/src/test/java/fr/insee/vtl/jackson/DatasetSerializerTest.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import fr.insee.vtl.model.Dataset; +import fr.insee.vtl.model.utils.Java8Helpers; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -19,7 +20,7 @@ public class DatasetSerializerTest extends AbstractMapperTest { @BeforeEach public void setUp() throws IOException { super.setUp(); - original = getClass().getResourceAsStream("/dataset-std.json").readAllBytes(); + original = Java8Helpers.readAllBytes(getClass().getResourceAsStream("/dataset-std.json")); dataset = mapper.readValue(original, Dataset.class); } diff --git a/vtl-jdbc/src/main/java/fr/insee/vtl/jdbc/JDBCDataset.java b/vtl-jdbc/src/main/java/fr/insee/vtl/jdbc/JDBCDataset.java index 1e8278f87..7cf0445f0 100644 --- a/vtl-jdbc/src/main/java/fr/insee/vtl/jdbc/JDBCDataset.java +++ b/vtl-jdbc/src/main/java/fr/insee/vtl/jdbc/JDBCDataset.java @@ -73,7 +73,7 @@ public static Class toVtlType(Integer sqlType) { public static DataStructure toDataStructure(ResultSetMetaData metaData) throws SQLException { List components = new ArrayList<>(); for (int columnIdx = 1; columnIdx <= metaData.getColumnCount(); columnIdx++) { - var name = metaData.getColumnName(columnIdx); + String name = metaData.getColumnName(columnIdx); Class type = toVtlType(metaData.getColumnType(columnIdx)); // TODO: refine nullable strategy components.add(new Component(name, type, Role.MEASURE, true)); @@ -83,8 +83,8 @@ public static DataStructure toDataStructure(ResultSetMetaData metaData) throws S @Override public List getDataPoints() { - try (var resultSet = this.resultSetSupplier.get()) { - var result = new ArrayList(); + try (ResultSet resultSet = this.resultSetSupplier.get()) { + List result = new ArrayList<>(); while (resultSet.next()) { result.add(toDataPoint(resultSet)); } @@ -125,7 +125,7 @@ private DataStructure getDataStructure(ResultSet resultSet) throws SQLException @Override public DataStructure getDataStructure() { - try (var resultSet = this.resultSetSupplier.get()) { + try (ResultSet resultSet = this.resultSetSupplier.get()) { return getDataStructure(resultSet); } catch (SQLException se) { throw new RuntimeException(se); diff --git a/vtl-jdbc/src/main/java/module-info.java b/vtl-jdbc/src/main/java/module-info.java deleted file mode 100644 index ef6ddabdb..000000000 --- a/vtl-jdbc/src/main/java/module-info.java +++ /dev/null @@ -1,10 +0,0 @@ -/** - * This module contains tools for SQL data sources. - */ -module fr.insee.vtl.jdbc { - requires fr.insee.vtl.model; - requires java.sql; - requires java.scripting; - - exports fr.insee.vtl.jdbc; -} diff --git a/vtl-jdbc/src/test/java/fr/insee/vtl/jdbc/JDBCDatasetTest.java b/vtl-jdbc/src/test/java/fr/insee/vtl/jdbc/JDBCDatasetTest.java index c507e1301..94c96e7fd 100644 --- a/vtl-jdbc/src/test/java/fr/insee/vtl/jdbc/JDBCDatasetTest.java +++ b/vtl-jdbc/src/test/java/fr/insee/vtl/jdbc/JDBCDatasetTest.java @@ -1,19 +1,17 @@ package fr.insee.vtl.jdbc; import fr.insee.vtl.model.Dataset; +import fr.insee.vtl.model.utils.Java8Helpers; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import javax.script.ScriptContext; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; +import javax.script.*; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.Map; +import java.sql.Statement; import static org.assertj.core.api.Assertions.assertThat; @@ -24,10 +22,10 @@ public class JDBCDatasetTest { @BeforeEach public void setUp() throws SQLException, IOException { - var databaseFile = File.createTempFile("vtl-test", "h2"); + File databaseFile = File.createTempFile("vtl-test", "h2"); databaseFile.deleteOnExit(); connection = DriverManager.getConnection("jdbc:h2:" + databaseFile); - var statement = connection.createStatement(); + Statement statement = connection.createStatement(); statement.executeUpdate("" + "create table if not exists ds1 (" + " id integer," + @@ -51,8 +49,8 @@ public void setUp() throws SQLException, IOException { @Test public void testReadSql() throws ScriptException, SQLException { - var statement = connection.createStatement(); - var jdbcDataset = new JDBCDataset(() -> { + Statement statement = connection.createStatement(); + JDBCDataset jdbcDataset = new JDBCDataset(() -> { try { return statement.executeQuery("select * from ds1;"); } catch (SQLException se) { @@ -60,20 +58,20 @@ public void testReadSql() throws ScriptException, SQLException { } }); - var bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); + Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); bindings.put("ds1", jdbcDataset); engine.eval("ds2 := ds1[calc identifier ID := ID, COL4 := if COL3 then COL2 + 1.0 else COL2 - 1.0];"); - var ds2 = (Dataset) bindings.get("ds2"); + Dataset ds2 = (Dataset) bindings.get("ds2"); assertThat(ds2.getDataAsMap()).containsExactly( - Map.of("ID", 1L, "COL1", "string1", "COL2", 1.2D, + Java8Helpers.mapOf("ID", 1L, "COL1", "string1", "COL2", 1.2D, "COL3", true, "COL4", 2.2D), - Map.of("ID", 2L, "COL1", "string2", "COL2", 5.2D, + Java8Helpers.mapOf("ID", 2L, "COL1", "string2", "COL2", 5.2D, "COL3", false, "COL4", 4.2D), - Map.of("ID", 3L, "COL1", "string3", "COL2", 3.2D, + Java8Helpers.mapOf("ID", 3L, "COL1", "string3", "COL2", 3.2D, "COL3", true, "COL4", 4.2D), - Map.of("ID", 4L, "COL1", "string4", "COL2", 4.2D, + Java8Helpers.mapOf("ID", 4L, "COL1", "string4", "COL2", 4.2D, "COL3", false, "COL4", 3.2D) ); } diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/AggregationExpression.java b/vtl-model/src/main/java/fr/insee/vtl/model/AggregationExpression.java index e91f58acd..ae2c85f98 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/AggregationExpression.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/AggregationExpression.java @@ -1,11 +1,6 @@ package fr.insee.vtl.model; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; import java.util.function.Function; diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/Dataset.java b/vtl-model/src/main/java/fr/insee/vtl/model/Dataset.java index 276bce3ed..ab66fa537 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/Dataset.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/Dataset.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -26,7 +27,7 @@ public interface Dataset extends Structured { List getDataPoints(); default List> getDataAsList() { - var columns = getDataStructure().keySet(); + Set columns = getDataStructure().keySet(); return getDataPoints().stream() .map(dataPoint -> columns.stream() .map(dataPoint::get).collect(Collectors.toList()) diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/InMemoryDataset.java b/vtl-model/src/main/java/fr/insee/vtl/model/InMemoryDataset.java index dce16b162..47faf5404 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/InMemoryDataset.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/InMemoryDataset.java @@ -1,8 +1,8 @@ package fr.insee.vtl.model; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import fr.insee.vtl.model.utils.Java8Helpers; + +import java.util.*; import java.util.stream.Collectors; /** @@ -52,7 +52,7 @@ public InMemoryDataset(List> data, Map> typ public InMemoryDataset(DataStructure structures, List... data) { this.structure = structures; - this.data = convertList(List.of(data)); + this.data = convertList(Java8Helpers.listOf(data)); } /** @@ -62,7 +62,7 @@ public InMemoryDataset(DataStructure structures, List... data) { * @param structure The list of structure components forming the structure of the dataset. */ public InMemoryDataset(List structure, List... dataPoints) { - this(List.of(dataPoints), structure); + this(Java8Helpers.listOf(dataPoints), structure); } /** diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/IndexedHashMap.java b/vtl-model/src/main/java/fr/insee/vtl/model/IndexedHashMap.java index a0f2b6806..6b2942a48 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/IndexedHashMap.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/IndexedHashMap.java @@ -1,10 +1,6 @@ package fr.insee.vtl.model; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * The DoubleExpression class is a delegated LinkedHashMap that has indexOf methods. diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/ListExpression.java b/vtl-model/src/main/java/fr/insee/vtl/model/ListExpression.java index fba0f667d..829fea4f1 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/ListExpression.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/ListExpression.java @@ -1,8 +1,10 @@ package fr.insee.vtl.model; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * The ListExpression class is an abstract representation of an expression of type List. @@ -21,7 +23,7 @@ public ListExpression(Positioned position) { * @return A new list expression resolving to a list with the elements and type provided. */ public static ListExpression withContainedType(Collection elements, Class containedType, Positioned position) { - List list = List.copyOf(elements); + List list = Collections.unmodifiableList(elements.stream().collect(Collectors.toList())); return new ListExpression(position) { @Override public List resolve(Map context) { diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/ResolvableExpression.java b/vtl-model/src/main/java/fr/insee/vtl/model/ResolvableExpression.java index 7b6a76131..f281a7512 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/ResolvableExpression.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/ResolvableExpression.java @@ -50,7 +50,7 @@ public ResolvableExpression tryCast(Class clazz) { return ResolvableExpression.withType(clazz).withPosition(this).using(ctx -> null); } return ResolvableExpression.withType(clazz).withPosition(this).using(ctx -> { - var value = this.resolve(ctx); + Object value = this.resolve(ctx); try { return clazz.cast(value); } catch (ClassCastException cce) { diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/Structured.java b/vtl-model/src/main/java/fr/insee/vtl/model/Structured.java index a358ea9d3..91d471bd0 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/Structured.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/Structured.java @@ -288,8 +288,8 @@ public DataStructure(Collection components) { super(components.size()); Set duplicates = new HashSet<>(); for (Component component : components) { - var newComponent = new Component(component); - var old = put(newComponent.getName(), newComponent); + Component newComponent = new Component(component); + Component old = put(newComponent.getName(), newComponent); if (old != null) { duplicates.add(old); } diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/exceptions/InvalidTypeException.java b/vtl-model/src/main/java/fr/insee/vtl/model/exceptions/InvalidTypeException.java index f3c332a4d..ee2b246ed 100644 --- a/vtl-model/src/main/java/fr/insee/vtl/model/exceptions/InvalidTypeException.java +++ b/vtl-model/src/main/java/fr/insee/vtl/model/exceptions/InvalidTypeException.java @@ -1,7 +1,10 @@ package fr.insee.vtl.model.exceptions; import fr.insee.vtl.model.Positioned; +import fr.insee.vtl.model.utils.Java8Helpers; +import java.util.Collections; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -26,7 +29,7 @@ public InvalidTypeException(Class expectedType, Class receivedType, Positi receivedType.getSimpleName(), expectedType.getSimpleName() ), position); this.expectedType = expectedType; - this.expectedTypes = Set.of(expectedType); + this.expectedTypes = Java8Helpers.setOf(expectedType); this.receivedType = receivedType; } diff --git a/vtl-model/src/main/java/fr/insee/vtl/model/utils/Java8Helpers.java b/vtl-model/src/main/java/fr/insee/vtl/model/utils/Java8Helpers.java new file mode 100644 index 000000000..9b93533eb --- /dev/null +++ b/vtl-model/src/main/java/fr/insee/vtl/model/utils/Java8Helpers.java @@ -0,0 +1,169 @@ +package fr.insee.vtl.model.utils; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public class Java8Helpers { + + public static class MapEntry { + + private final K key; + private final V value; + + public MapEntry(K key, V value) { + this.key = key; + this.value = value; + } + + public static MapEntry of(K key, V value) { + return new MapEntry<>(key, value); + } + } + + @SafeVarargs // only read access + public static List listOf(T... items) { + List tmpList = new ArrayList<>(); + Collections.addAll(tmpList, items); + return Collections.unmodifiableList(tmpList); + } + + @SafeVarargs // only read access + public static Set setOf(T... items) { + Set tmpSet = new HashSet<>(); + Collections.addAll(tmpSet, items); + return Collections.unmodifiableSet(tmpSet); + } + + public static Map mapOf() { + return Collections.unmodifiableMap(new HashMap<>()); + } + + public static Map mapOf(K k1, V v1) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + tmpMap.put(k4, v4); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + tmpMap.put(k4, v4); + tmpMap.put(k5, v5); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5, K k6, V v6) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + tmpMap.put(k4, v4); + tmpMap.put(k5, v5); + tmpMap.put(k6, v6); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5, K k6, V v6, K k7, V v7) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + tmpMap.put(k4, v4); + tmpMap.put(k5, v5); + tmpMap.put(k6, v6); + tmpMap.put(k7, v7); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5, K k6, V v6, K k7, V v7, K k8, V v8) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + tmpMap.put(k4, v4); + tmpMap.put(k5, v5); + tmpMap.put(k6, v6); + tmpMap.put(k7, v7); + tmpMap.put(k8, v8); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5, K k6, V v6, K k7, V v7, K k8, V v8, K k9, V v9) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + tmpMap.put(k4, v4); + tmpMap.put(k5, v5); + tmpMap.put(k6, v6); + tmpMap.put(k7, v7); + tmpMap.put(k8, v8); + tmpMap.put(k9, v9); + return Collections.unmodifiableMap(tmpMap); + } + + public static Map mapOf(K k1, V v1, K k2, V v2, K k3, V v3, K k4, V v4, K k5, V v5, K k6, V v6, K k7, V v7, K k8, V v8, K k9, V v9, K k10, V v10) { + Map tmpMap = new HashMap<>(); + tmpMap.put(k1, v1); + tmpMap.put(k2, v2); + tmpMap.put(k3, v3); + tmpMap.put(k4, v4); + tmpMap.put(k5, v5); + tmpMap.put(k6, v6); + tmpMap.put(k7, v7); + tmpMap.put(k8, v8); + tmpMap.put(k9, v9); + tmpMap.put(k10, v10); + return Collections.unmodifiableMap(tmpMap); + } + + @SafeVarargs // only read access + public static Map mapOfEntries(MapEntry... entries) { + Map tmpMap = new HashMap<>(); + Arrays.stream(entries).forEach(entry -> tmpMap.put(entry.key, entry.value)); + return Collections.unmodifiableMap(tmpMap); + } + + public static Stream streamIterator(Iterator iterator) { + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false + ); + } + + public static byte[] readAllBytes(InputStream inputStream) throws IOException { + byte[] bytes = new byte[inputStream.available()]; + DataInputStream dataInputStream = new DataInputStream(inputStream); + dataInputStream.readFully(bytes); + return bytes; + } +} \ No newline at end of file diff --git a/vtl-model/src/main/java/module-info.java b/vtl-model/src/main/java/module-info.java deleted file mode 100644 index 0afdd04e6..000000000 --- a/vtl-model/src/main/java/module-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/** - * This module contains the model describing VTL expressions and bindings. - */ -module fr.insee.vtl.model { - requires java.scripting; - exports fr.insee.vtl.model; - exports fr.insee.vtl.model.exceptions; -} \ No newline at end of file diff --git a/vtl-model/src/test/java/fr/insee/vtl/model/IndexedHashMapTest.java b/vtl-model/src/test/java/fr/insee/vtl/model/IndexedHashMapTest.java index a42130230..e31800418 100644 --- a/vtl-model/src/test/java/fr/insee/vtl/model/IndexedHashMapTest.java +++ b/vtl-model/src/test/java/fr/insee/vtl/model/IndexedHashMapTest.java @@ -8,8 +8,8 @@ public class IndexedHashMapTest { @Test public void testEquals() { - var map1 = new IndexedHashMap<>(); - var map2 = new IndexedHashMap<>(); + IndexedHashMap map1 = new IndexedHashMap<>(); + IndexedHashMap map2 = new IndexedHashMap<>(); map1.put("a", "1"); map1.put("b", "2"); diff --git a/vtl-parser/src/main/java/module-info.java b/vtl-parser/src/main/java/module-info.java deleted file mode 100644 index 9ee49778f..000000000 --- a/vtl-parser/src/main/java/module-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/** - * This module exposes the constructs generated by Antlr from the VTL grammar files. - */ -module fr.insee.vtl.parser { - requires org.antlr.antlr4.runtime; - exports fr.insee.vtl.parser; - opens fr.insee.vtl.parser; -} \ No newline at end of file diff --git a/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/SDMXVTLWorkflow.java b/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/SDMXVTLWorkflow.java index 6c79f3e8c..54bcfd05f 100644 --- a/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/SDMXVTLWorkflow.java +++ b/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/SDMXVTLWorkflow.java @@ -4,6 +4,7 @@ import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.PersistentDataset; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import io.sdmx.api.io.ReadableDataLocation; import io.sdmx.api.sdmx.model.beans.SdmxBeans; import io.sdmx.api.sdmx.model.beans.base.INamedBean; @@ -57,8 +58,8 @@ private Map getRulesets() { ) .flatMap(v -> v.getItems().stream()) .collect(Collectors.toMap(INamedBean::getId, v -> { - var desc = toComment(v.getDescription()); - var definition = addSemi(v.getRulesetDefinition()); + String desc = toComment(v.getDescription()); + String definition = addSemi(v.getRulesetDefinition()); return desc + definition; }, (u, v) -> { @@ -80,10 +81,10 @@ public Map getTransformations() { v -> v.getItems().stream() ) .collect(Collectors.toMap(ITransformationBean::getResult, v -> { - var varName = v.getResult(); - var comment = toComment(v.getDescription()); - var expression = addSemi(v.getExpression()); - var assignment = (v.isPersistent() ? " <- " : " := "); + String varName = v.getResult(); + String comment = toComment(v.getDescription()); + String expression = addSemi(v.getExpression()); + String assignment = (v.isPersistent() ? " <- " : " := "); return comment + varName + assignment + expression; }, (u, v) -> { @@ -101,7 +102,7 @@ public Map getEmptyDatasets() { return TrevasSDMXUtils.parseDataStructure(sdmxBeans).entrySet().stream() .map(e -> new AbstractMap.SimpleEntry<>( e.getKey(), - new InMemoryDataset(List.of(), (Map) e.getValue()) + new InMemoryDataset(Java8Helpers.listOf(), (Map) e.getValue()) )).collect(Collectors.toMap( Map.Entry::getKey, Map.Entry::getValue, diff --git a/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/TrevasSDMXUtils.java b/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/TrevasSDMXUtils.java index aa1a9be19..c6147f45e 100644 --- a/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/TrevasSDMXUtils.java +++ b/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/TrevasSDMXUtils.java @@ -100,7 +100,7 @@ private static Dataset.Role convertTypeToRole(ComponentBean.COMPONENT_TYPE type) } public static Structured.DataStructure buildStructureFromSDMX3(SdmxBeans beans, String structureID) { - var structures = parseDataStructure(beans); + Map structures = parseDataStructure(beans); return structures.get(structureID); } diff --git a/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/package-info.java b/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/package-info.java deleted file mode 100644 index f4fa80587..000000000 --- a/vtl-sdmx/src/main/java/fr/insee/vtl/sdmx/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * This package contains tools for SDMX data sources. - */ -package fr.insee.vtl.sdmx; \ No newline at end of file diff --git a/vtl-sdmx/src/main/java/module-info.java b/vtl-sdmx/src/main/java/module-info.java deleted file mode 100644 index 41e03d00a..000000000 --- a/vtl-sdmx/src/main/java/module-info.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * This module exposes SDMX tools for VTL. - */ - -module fr.insee.vtl.sdmx { - requires fr.insee.vtl.model; - requires fusion.api.sdmx; - requires fusion.api; - requires java.scripting; - requires fusion.sdmx.ml; - requires fusion.utils; - exports fr.insee.vtl.sdmx; -} \ No newline at end of file diff --git a/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java b/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java index 44a0a56ab..c01217942 100644 --- a/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java +++ b/vtl-sdmx/src/test/java/fr/insee/vtl/BPETest.java @@ -4,6 +4,7 @@ import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.PersistentDataset; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.sdmx.SDMXVTLWorkflow; import fr.insee.vtl.sdmx.TrevasSDMXUtils; import fr.insee.vtl.spark.SparkDataset; @@ -223,7 +224,7 @@ public void bpeV2() { .csv("src/test/resources/LEGAL_POP_NUTS3.csv"), censusStructure ); - Map inputs = Map.of("BPE_DETAIL_VTL", bpeDetailDs, "LEGAL_POP", censusNuts); + Map inputs = Java8Helpers.mapOf("BPE_DETAIL_VTL", bpeDetailDs, "LEGAL_POP", censusNuts); ReadableDataLocation rdl = new ReadableDataLocationTmp("src/test/resources/DSD_BPE_CENSUS.xml"); SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, inputs); Map bindings = sdmxVtlWorkflow.run(); diff --git a/vtl-sdmx/src/test/java/fr/insee/vtl/DataStructureTest.java b/vtl-sdmx/src/test/java/fr/insee/vtl/DataStructureTest.java index a6cb431bf..f64f17634 100644 --- a/vtl-sdmx/src/test/java/fr/insee/vtl/DataStructureTest.java +++ b/vtl-sdmx/src/test/java/fr/insee/vtl/DataStructureTest.java @@ -16,7 +16,7 @@ public class DataStructureTest { public void buildDataset() throws IOException { DataStructure structure = TrevasSDMXUtils.buildStructureFromSDMX3("src/test/resources/DSD_BPE_TOWN.xml", "BPE_CUBE_2021"); - var dataset = new CSVDataset(structure, new FileReader("src/test/resources/BPE_TOWN_SAMPLE.csv")); + CSVDataset dataset = new CSVDataset(structure, new FileReader("src/test/resources/BPE_TOWN_SAMPLE.csv")); dataset.getDataAsMap().forEach(System.out::println); } diff --git a/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java b/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java index b71c09244..e855df1ec 100644 --- a/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java +++ b/vtl-sdmx/src/test/java/fr/insee/vtl/SDMXVTLWorkflowTest.java @@ -4,6 +4,7 @@ import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.PersistentDataset; import fr.insee.vtl.model.Structured; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.sdmx.SDMXVTLWorkflow; import io.sdmx.api.io.ReadableDataLocation; import io.sdmx.utils.core.io.ReadableDataLocationTmp; @@ -14,8 +15,6 @@ import javax.script.ScriptContext; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; - -import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -40,11 +39,11 @@ void testGetEmptyDataset() { .getOrCreate(); ScriptEngineManager mgr = new ScriptEngineManager(); - var engine = mgr.getEngineByExtension("vtl"); + ScriptEngine engine = mgr.getEngineByExtension("vtl"); engine.put(VtlScriptEngine.PROCESSING_ENGINE_NAMES, "spark"); ReadableDataLocation rdl = new ReadableDataLocationTmp("src/test/resources/DSD_BPE_CENSUS.xml"); - SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Map.of()); + SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Java8Helpers.mapOf()); Map emptyDatasets = sdmxVtlWorkflow.getEmptyDatasets(); engine.getBindings(ScriptContext.ENGINE_SCOPE).putAll(emptyDatasets); @@ -54,13 +53,13 @@ void testGetEmptyDataset() { ); assertThat(result.get("BPE_CENSUS_NUTS3_2021").getDataStructure()).isEqualTo( - new Structured.DataStructure(List.of( + new Structured.DataStructure(Java8Helpers.listOf( new Structured.Component("nuts3", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("pract_per_10000_inhabitants", Double.class, Dataset.Role.MEASURE) )) ); assertThat(result.get("BPE_MUNICIPALITY").getDataStructure()).isEqualTo( - new Structured.DataStructure(List.of( + new Structured.DataStructure(Java8Helpers.listOf( new Structured.Component("facility_type", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("municipality", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("year", String.class, Dataset.Role.IDENTIFIER), @@ -68,7 +67,7 @@ void testGetEmptyDataset() { )) ); assertThat(result.get("BPE_NUTS3").getDataStructure()).isEqualTo( - new Structured.DataStructure(List.of( + new Structured.DataStructure(Java8Helpers.listOf( new Structured.Component("year", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("facility_type", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("nuts3", String.class, Dataset.Role.IDENTIFIER), @@ -80,7 +79,7 @@ void testGetEmptyDataset() { @Test public void testGetRulesetsVTL() { ReadableDataLocation rdl = new ReadableDataLocationTmp("src/test/resources/DSD_BPE_CENSUS.xml"); - SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Map.of()); + SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Java8Helpers.mapOf()); assertThat(sdmxVtlWorkflow.getRulesetsVTL()).isEqualTo( "define datapoint ruleset UNIQUE_MUNICIPALITY (valuedomain CL_DEPCOM) is\n" + " MUNICIPALITY_FORMAT_RULE : match_characters(CL_DEPCOM, \"[0-9]{5}|2[A-B][0-9]{3}\") errorcode \"Municipality code is not in the correct format\"\n" + @@ -95,7 +94,7 @@ public void testGetRulesetsVTL() { @Test public void testGetTransformationsVTL() { ReadableDataLocation rdl = new ReadableDataLocationTmp("src/test/resources/DSD_BPE_CENSUS.xml"); - SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Map.of()); + SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, Java8Helpers.mapOf()); assertThat(sdmxVtlWorkflow.getTransformationsVTL()).isEqualTo( "// Validation of municipality code in input file\n" + "CHECK_MUNICIPALITY := check_datapoint(BPE_DETAIL_VTL, UNIQUE_MUNICIPALITY invalid);\n" + diff --git a/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkDataset.java b/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkDataset.java index 2c03dd5ed..7a7850d09 100644 --- a/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkDataset.java +++ b/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkDataset.java @@ -1,5 +1,6 @@ package fr.insee.vtl.spark; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Structured; import org.apache.spark.sql.Row; @@ -35,8 +36,8 @@ public class SparkDataset implements Dataset { * @param roles a map between component names and their roles in the dataset. */ public SparkDataset(org.apache.spark.sql.Dataset sparkDataset, Map roles) { - var castedSparkDataset = castIfNeeded(Objects.requireNonNull(sparkDataset)); - var dataStructure = fromSparkSchema(sparkDataset.schema(), roles, Map.of()); + org.apache.spark.sql.Dataset castedSparkDataset = castIfNeeded(Objects.requireNonNull(sparkDataset)); + DataStructure dataStructure = fromSparkSchema(sparkDataset.schema(), roles, Java8Helpers.mapOf()); this.sparkDataset = addMetadata(castedSparkDataset, dataStructure); this.roles = Objects.requireNonNull(roles); } @@ -48,7 +49,7 @@ public SparkDataset(org.apache.spark.sql.Dataset sparkDataset, Map sparkDataset, DataStructure structure) { - var castedSparkDataset = castIfNeeded(Objects.requireNonNull(sparkDataset)); + org.apache.spark.sql.Dataset castedSparkDataset = castIfNeeded(Objects.requireNonNull(sparkDataset)); this.sparkDataset = addMetadata(castedSparkDataset, structure); this.roles = Objects.requireNonNull( structure.entrySet() @@ -94,7 +95,7 @@ public SparkDataset(Dataset vtlDataset, Map roles, SparkSession sp * Cast integer and float types to long and double. */ private static org.apache.spark.sql.Dataset castIfNeeded(org.apache.spark.sql.Dataset sparkDataset) { - var casted = sparkDataset; + org.apache.spark.sql.Dataset casted = sparkDataset; StructType schema = sparkDataset.schema(); for (StructField field : JavaConverters.asJavaCollection(schema)) { if (IntegerType.sameType(field.dataType())) { @@ -112,7 +113,7 @@ private static org.apache.spark.sql.Dataset castIfNeeded(org.apache.spark.s } private static org.apache.spark.sql.Dataset addMetadata(org.apache.spark.sql.Dataset sparkDataset, DataStructure structure) { - var casted = sparkDataset; + org.apache.spark.sql.Dataset casted = sparkDataset; for (StructField field : JavaConverters.asJavaCollection(toSparkSchema(structure))) { String name = field.name(); casted = casted.withColumn(name, casted.col(name), field.metadata()); @@ -131,11 +132,8 @@ public static StructType toSparkSchema(DataStructure structure) { for (Component component : structure.values()) { Object vd = null == component.getValuedomain() ? null : component.getValuedomain(); // TODO: refine nullable strategy - - Map map = new HashMap<>(); - map.put("vtlRole", component.getRole().name()); - map.put("vtlValuedomain", vd); - var md = mapAsScalaMap(map).toMap(Predef.$conforms()); + scala.collection.immutable.Map md = mapAsScalaMap(Java8Helpers.mapOf("vtlRole", component.getRole().name(), "vtlValuedomain", vd)) + .toMap(Predef.$conforms()); schema.add(DataTypes.createStructField( component.getName(), fromVtlType(component.getType()), @@ -169,7 +167,7 @@ private static Role handleRole(StructField field, Map roles) { if (roles.containsKey(field.name())) { fieldRole = roles.get(field.name()); } else if (field.metadata().contains("vtlRole")) { - var roleName = field.metadata().getString("vtlRole"); + String roleName = field.metadata().getString("vtlRole"); fieldRole = Role.valueOf(roleName); } else { fieldRole = Role.MEASURE; diff --git a/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkFilterFunction.java b/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkFilterFunction.java index 15163fd74..bf481b8ea 100644 --- a/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkFilterFunction.java +++ b/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkFilterFunction.java @@ -22,7 +22,7 @@ public SparkFilterFunction(ResolvableExpression expression) { @Override public boolean call(Row row) { - var res = expression.resolve(new SparkRowMap(row)); + Object res = expression.resolve(new SparkRowMap(row)); if (res == null) return false; return (boolean) res; } diff --git a/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkProcessingEngine.java b/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkProcessingEngine.java index b151bff21..c24fda299 100644 --- a/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkProcessingEngine.java +++ b/vtl-spark/src/main/java/fr/insee/vtl/spark/SparkProcessingEngine.java @@ -1,5 +1,6 @@ package fr.insee.vtl.spark; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.*; import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; @@ -115,13 +116,13 @@ private static WindowSpec buildWindowSpec(List partitionBy, Map orderBy, Analytics.WindowSpec window) { if (partitionBy == null) { - partitionBy = List.of(); + partitionBy = Java8Helpers.listOf(); } WindowSpec windowSpec = Window.partitionBy(colNameToCol(partitionBy)); if (orderBy == null) { - orderBy = Map.of(); + orderBy = Java8Helpers.mapOf(); } windowSpec = windowSpec.orderBy(buildOrderCol(orderBy)); @@ -164,9 +165,9 @@ private static List identifierNames(List components) { private SparkDataset asSparkDataset(DatasetExpression expression) { if (expression instanceof SparkDatasetExpression) { - return ((SparkDatasetExpression) expression).resolve(Map.of()); + return ((SparkDatasetExpression) expression).resolve(Java8Helpers.mapOf()); } else { - var dataset = expression.resolve(Map.of()); + fr.insee.vtl.model.Dataset dataset = expression.resolve(Java8Helpers.mapOf()); if (dataset instanceof SparkDataset) { return (SparkDataset) dataset; } else { @@ -185,7 +186,7 @@ public DatasetExpression executeCalc(DatasetExpression expression, Map aliasesToName = new HashMap<>(); Map renamedExpressions = new LinkedHashMap<>(); Map renamedExpressionString = new LinkedHashMap<>(); - for (var name : expressions.keySet()) { + for (String name : expressions.keySet()) { String alias = name + "_" + aliasesToName.size(); renamedExpressions.put(alias, expressions.get(name)); renamedExpressionString.put(alias, expressionStrings.get(name)); @@ -202,16 +203,16 @@ public DatasetExpression executeCalc(DatasetExpression expression, Map renamed = rename(evaluated, aliasesToName); // Create the new role map. - var roleMap = getRoleMap(dataset); + Map roleMap = getRoleMap(dataset); roleMap.putAll(roles); return new SparkDatasetExpression(new SparkDataset(renamed, roleMap), expression); } private Dataset executeCalcEvaluated(Dataset interpreted, Map expressions) { - var columnNames = Set.of(interpreted.columns()); + Set columnNames = Java8Helpers.setOf(interpreted.columns()); Column structColumns = struct(columnNames.stream().map(colName -> col(colName)).toArray(Column[]::new)); - for (var name : expressions.keySet()) { + for (String name : expressions.keySet()) { // Ignore the columns that already exist. if (columnNames.contains(name)) { continue; @@ -262,10 +263,10 @@ public DatasetExpression executeFilter(DatasetExpression expression, ResolvableE public DatasetExpression executeRename(DatasetExpression expression, Map fromTo) { SparkDataset dataset = asSparkDataset(expression); - var result = rename(dataset.getSparkDataset(), fromTo); + Dataset result = rename(dataset.getSparkDataset(), fromTo); - var originalRoles = getRoleMap(dataset); - var renamedRoles = new LinkedHashMap<>(originalRoles); + Map originalRoles = getRoleMap(dataset); + Map renamedRoles = new LinkedHashMap<>(originalRoles); for (Map.Entry fromToEntry : fromTo.entrySet()) { renamedRoles.put(fromToEntry.getValue(), originalRoles.get(fromToEntry.getKey())); } @@ -424,7 +425,7 @@ public DatasetExpression executeSimpleAnalytic( throw UNKNOWN_ANALYTIC_FUNCTION; } - var result = sparkDataset.getSparkDataset().withColumn(targetColName, column); + Dataset result = sparkDataset.getSparkDataset().withColumn(targetColName, column); return new SparkDatasetExpression(new SparkDataset(result), dataset); } @@ -454,7 +455,7 @@ public DatasetExpression executeLeadOrLagAn( default: throw UNKNOWN_ANALYTIC_FUNCTION; } - var result = sparkDataset.getSparkDataset().withColumn(targetColName, column); + Dataset result = sparkDataset.getSparkDataset().withColumn(targetColName, column); return new SparkDatasetExpression(new SparkDataset(result), dataset); } @@ -505,7 +506,7 @@ public DatasetExpression executeRankAn( public DatasetExpression executeInnerJoin(Map datasets, List components) { List> sparkDatasets = toAliasedDatasets(datasets); List identifiers = identifierNames(components); - var innerJoin = executeJoin(sparkDatasets, identifiers, "inner"); + Dataset innerJoin = executeJoin(sparkDatasets, identifiers, "inner"); DatasetExpression datasetExpression = datasets.entrySet().iterator().next().getValue(); return new SparkDatasetExpression(new SparkDataset(innerJoin, getRoleMap(components)), datasetExpression); } @@ -514,7 +515,7 @@ public DatasetExpression executeInnerJoin(Map dataset public DatasetExpression executeLeftJoin(Map datasets, List components) { List> sparkDatasets = toAliasedDatasets(datasets); List identifiers = identifierNames(components); - var innerJoin = executeJoin(sparkDatasets, identifiers, "left"); + Dataset innerJoin = executeJoin(sparkDatasets, identifiers, "left"); DatasetExpression datasetExpression = datasets.entrySet().iterator().next().getValue(); return new SparkDatasetExpression(new SparkDataset(innerJoin, getRoleMap(components)), datasetExpression); } @@ -522,7 +523,7 @@ public DatasetExpression executeLeftJoin(Map datasets @Override public DatasetExpression executeCrossJoin(Map datasets, List identifiers) { List> sparkDatasets = toAliasedDatasets(datasets); - var crossJoin = executeJoin(sparkDatasets, List.of(), "cross"); + Dataset crossJoin = executeJoin(sparkDatasets, Java8Helpers.listOf(), "cross"); DatasetExpression datasetExpression = datasets.entrySet().iterator().next().getValue(); return new SparkDatasetExpression(new SparkDataset(crossJoin, getRoleMap(identifiers)), datasetExpression); } @@ -531,7 +532,7 @@ public DatasetExpression executeCrossJoin(Map dataset public DatasetExpression executeFullJoin(Map datasets, List identifiers) { List> sparkDatasets = toAliasedDatasets(datasets); List identifierNames = identifierNames(identifiers); - var crossJoin = executeJoin(sparkDatasets, identifierNames, "outer"); + Dataset crossJoin = executeJoin(sparkDatasets, identifierNames, "outer"); DatasetExpression datasetExpression = datasets.entrySet().iterator().next().getValue(); return new SparkDatasetExpression(new SparkDataset(crossJoin, getRoleMap(identifiers)), datasetExpression); } @@ -547,7 +548,7 @@ public DatasetExpression executeValidateDPruleset(DataPointRuleset dpr, DatasetE DatasetExpression sparkDsExpr = new SparkDatasetExpression(sparkDs, pos); Structured.DataStructure dataStructure = sparkDs.getDataStructure(); - var roleMap = getRoleMap(sparkDataset); + Map roleMap = getRoleMap(sparkDataset); roleMap.put(RULEID, IDENTIFIER); roleMap.put(BOOLVAR, MEASURE); roleMap.put(ERRORLEVEL, MEASURE); @@ -607,7 +608,7 @@ public DatasetExpression executeValidateDPruleset(DataPointRuleset dpr, DatasetE resolvableExpressions.put(ERRORLEVEL, errorLevelExpression); resolvableExpressions.put(ERRORCODE, errorCodeExpression); // do we need to use execute executeCalcInterpreted too? - return executeCalc(sparkDsExpr, resolvableExpressions, roleMap, Map.of()); + return executeCalc(sparkDsExpr, resolvableExpressions, roleMap, Java8Helpers.mapOf()); } ).collect(Collectors.toList()); @@ -639,14 +640,14 @@ public DatasetExpression executeValidationSimple(DatasetExpression dsExpr, Dataset sparkImbalanceDatasetRow = sparkImbalanceDataset.getSparkDataset(); String imbalanceMonomeasureName = imbalanceExpr.getDataStructure().values() .stream().filter(Component::isMeasure).map(Component::getName).collect(Collectors.toList()).get(0); - Map varsToRename = Map.ofEntries(Map.entry(imbalanceMonomeasureName, IMBALANCE)); + Map varsToRename = Java8Helpers.mapOfEntries(Java8Helpers.MapEntry.of(imbalanceMonomeasureName, IMBALANCE)); Dataset renamed = rename(sparkImbalanceDatasetRow, varsToRename); - var imbalanceRoleMap = getRoleMap(sparkImbalanceDataset); + Map imbalanceRoleMap = getRoleMap(sparkImbalanceDataset); SparkDatasetExpression imbalanceRenamedExpr = new SparkDatasetExpression(new SparkDataset(renamed, imbalanceRoleMap), pos); // Join expr ds & imbalance ds - Map datasetExpressions = Map.ofEntries( - Map.entry("dsExpr", dsExpr), - Map.entry("imbalanceExpr", imbalanceRenamedExpr) + Map datasetExpressions = Java8Helpers.mapOfEntries( + Java8Helpers.MapEntry.of("dsExpr", dsExpr), + Java8Helpers.MapEntry.of("imbalanceExpr", imbalanceRenamedExpr) ); List components = dsExpr.getDataStructure().values().stream() .filter(Component::isIdentifier) @@ -676,13 +677,13 @@ public DatasetExpression executeValidationSimple(DatasetExpression dsExpr, return boolVar ? null : errorLevelType.cast(erLevel); }); - var roleMap = getRoleMap(sparkDataset); + Map roleMap = getRoleMap(sparkDataset); roleMap.put(ERRORLEVEL, MEASURE); roleMap.put(ERRORCODE, MEASURE); - Map resolvableExpressions = Map.ofEntries( - Map.entry(ERRORLEVEL, errorLevelExpression), - Map.entry(ERRORCODE, errorCodeExpression) + Map resolvableExpressions = Java8Helpers.mapOfEntries( + Java8Helpers.MapEntry.of(ERRORLEVEL, errorLevelExpression), + Java8Helpers.MapEntry.of(ERRORCODE, errorCodeExpression) ); Dataset calculatedDataset = executeCalcEvaluated(ds, resolvableExpressions); @@ -710,7 +711,7 @@ public DatasetExpression executeHierarchicalValidation(DatasetExpression dsE, Hi } // Create "bindings" (componentID column values) - fr.insee.vtl.model.Dataset ds = dsE.resolve(Map.of()); + fr.insee.vtl.model.Dataset ds = dsE.resolve(Java8Helpers.mapOf()); Map bindings = ds.getDataAsMap().stream() .collect( @@ -722,7 +723,7 @@ public DatasetExpression executeHierarchicalValidation(DatasetExpression dsE, Hi Component measure = dsE.getDataStructure().getMeasures().get(0); Class measureType = measure.getType(); - var roleMap = getRoleMap(ds); + Map roleMap = getRoleMap(ds); roleMap.put(RULEID, IDENTIFIER); roleMap.put(BOOLVAR, MEASURE); roleMap.put(IMBALANCE, MEASURE); @@ -832,14 +833,14 @@ public DatasetExpression executeHierarchicalValidation(DatasetExpression dsE, Hi resolvableExpressions.put(ERRORLEVEL, errorLevelExpression); resolvableExpressions.put(ERRORCODE, errorCodeExpression); - datasetsExpression.add(executeCalc(filteredDataset, resolvableExpressions, roleMap, Map.of())); + datasetsExpression.add(executeCalc(filteredDataset, resolvableExpressions, roleMap, Java8Helpers.mapOf())); } ); DatasetExpression datasetExpression; if (datasetsExpression.size() == 0) { InMemoryDataset emptyCHDataset = new InMemoryDataset( - List.of(), - Map.of(measure.getName(), measureType, RULEID, String.class, componentID, String.class, BOOLVAR, Boolean.class, IMBALANCE, Double.class, ERRORLEVEL, errorLevelType, ERRORCODE, errorCodeType), + Java8Helpers.listOf(), + Java8Helpers.mapOf(measure.getName(), measureType, RULEID, String.class, componentID, String.class, BOOLVAR, Boolean.class, IMBALANCE, Double.class, ERRORLEVEL, errorLevelType, ERRORCODE, errorCodeType), roleMap ); datasetExpression = DatasetExpression.of(emptyCHDataset, pos); @@ -939,12 +940,12 @@ private Map buildBindingsWithDefault(Map binding bindingsWithDefault.put(i, bindings.get(i)); } else { // don't need to handle non_null, items are always in bindings - if (List.of(NON_ZERO, PARTIAL_ZERO, ALWAYS_ZERO).contains(validationMode)) { + if (Java8Helpers.listOf(NON_ZERO, PARTIAL_ZERO, ALWAYS_ZERO).contains(validationMode)) { if (measureType.isAssignableFrom(Long.class)) { bindingsWithDefault.put(i, 0L); } else bindingsWithDefault.put(i, 0D); } - if (List.of(PARTIAL_NULL, ALWAYS_NULL).contains(validationMode)) { + if (Java8Helpers.listOf(PARTIAL_NULL, ALWAYS_NULL).contains(validationMode)) { bindingsWithDefault.put(i, null); } } @@ -961,7 +962,7 @@ private Map invertMap(Map map) { private List> toAliasedDatasets(Map datasets) { List> sparkDatasets = new ArrayList<>(); for (Map.Entry dataset : datasets.entrySet()) { - var sparkDataset = asSparkDataset(dataset.getValue()) + Dataset sparkDataset = asSparkDataset(dataset.getValue()) .getSparkDataset() .as(dataset.getKey()); sparkDatasets.add(sparkDataset); @@ -978,8 +979,8 @@ private List> toAliasedDatasets(Map data * @return The dataset resulting from the join operation. */ public Dataset executeJoin(List> sparkDatasets, List identifiers, String type) { - var iterator = sparkDatasets.iterator(); - var result = iterator.next(); + Iterator> iterator = sparkDatasets.iterator(); + Dataset result = iterator.next(); while (iterator.hasNext()) { if (type.equals("cross")) result = result.crossJoin(iterator.next()); else result = result.join( @@ -1006,7 +1007,7 @@ public String getName() { @Override public ProcessingEngine getProcessingEngine(ScriptEngine engine) { // Try to find the session in the script engine. - var session = engine.get(SPARK_SESSION); + Object session = engine.get(SPARK_SESSION); if (session != null) { if (session instanceof SparkSession) { return new SparkProcessingEngine((SparkSession) session); @@ -1014,7 +1015,7 @@ public ProcessingEngine getProcessingEngine(ScriptEngine engine) { throw new IllegalArgumentException(SPARK_SESSION + " was not a spark session"); } } else { - var activeSession = SparkSession.active(); + SparkSession activeSession = SparkSession.active(); if (activeSession != null) { return new SparkProcessingEngine(activeSession); } else { diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java index a099958ec..d47848a9b 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkDatasetTest.java @@ -1,13 +1,10 @@ package fr.insee.vtl.spark; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.api.java.function.FilterFunction; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.RowFactory; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.*; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.BeforeEach; @@ -19,8 +16,6 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; import java.nio.file.Path; -import java.util.List; -import java.util.Map; import static fr.insee.vtl.model.Structured.Component; import static org.assertj.core.api.Assertions.assertThat; @@ -45,14 +40,14 @@ public void setUp() { @Test public void testFilters() { - StructType schema = DataTypes.createStructType(List.of( + StructType schema = DataTypes.createStructType(Java8Helpers.listOf( DataTypes.createStructField("string", DataTypes.StringType, false), DataTypes.createStructField("integer", DataTypes.LongType, false), DataTypes.createStructField("boolean", DataTypes.BooleanType, false), DataTypes.createStructField("float", DataTypes.DoubleType, false) )); - Dataset dataFrame = spark.createDataFrame(List.of( + Dataset dataFrame = spark.createDataFrame(Java8Helpers.listOf( RowFactory.create("string", 1L, true, 1.5D) ), schema); @@ -65,14 +60,14 @@ public void testFilters() { @Test void testVtlCanReadSpark() { - StructType schema = DataTypes.createStructType(List.of( + StructType schema = DataTypes.createStructType(Java8Helpers.listOf( DataTypes.createStructField("string", DataTypes.StringType, false), DataTypes.createStructField("integer", DataTypes.LongType, false), DataTypes.createStructField("boolean", DataTypes.BooleanType, false), DataTypes.createStructField("float", DataTypes.DoubleType, false) )); - Dataset dataFrame = spark.createDataFrame(List.of( + Dataset dataFrame = spark.createDataFrame(Java8Helpers.listOf( RowFactory.create("string", 1L, true, 1.5D) ), schema); @@ -80,8 +75,8 @@ void testVtlCanReadSpark() { fr.insee.vtl.model.Dataset sparkDataset = new SparkDataset(dataFrame); fr.insee.vtl.model.Dataset expectedDataset = new InMemoryDataset( - List.of(List.of("string", 1L, true, 1.5D)), - List.of( + Java8Helpers.listOf(Java8Helpers.listOf("string", 1L, true, 1.5D)), + Java8Helpers.listOf( new Component("string", String.class, fr.insee.vtl.model.Dataset.Role.MEASURE), new Component("integer", Long.class, fr.insee.vtl.model.Dataset.Role.MEASURE), new Component("boolean", Boolean.class, fr.insee.vtl.model.Dataset.Role.MEASURE), @@ -98,7 +93,7 @@ void testVtlCanReadSpark() { public void testParquetMetadataReading(@TempDir Path tmpDirectory) { SparkDataset sparkDataset = new SparkDataset( spark.read().parquet("src/main/resources/input_sample"), - Map.of( + Java8Helpers.mapOf( "year", fr.insee.vtl.model.Dataset.Role.IDENTIFIER, "student_number", fr.insee.vtl.model.Dataset.Role.ATTRIBUTE ) diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkSQLTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkSQLTest.java index 5bd88cbc0..61bfcee2f 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkSQLTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/SparkSQLTest.java @@ -1,23 +1,23 @@ package fr.insee.vtl.spark; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; +import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import javax.script.ScriptContext; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; +import javax.script.*; import java.io.File; import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; +import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @@ -32,8 +32,8 @@ public class SparkSQLTest { public void setUp() throws SQLException, IOException { databaseFile = File.createTempFile("vtl-test", "h2"); databaseFile.deleteOnExit(); - var connection = DriverManager.getConnection("jdbc:h2:" + databaseFile); - var statement = connection.createStatement(); + Connection connection = DriverManager.getConnection("jdbc:h2:" + databaseFile); + Statement statement = connection.createStatement(); statement.executeUpdate("create table if not exists ds1 (" + " id integer," + " colChar char(7), " + @@ -70,14 +70,14 @@ public void setUp() throws SQLException, IOException { public void testReadSql() throws ScriptException { // See https://spark.apache.org/docs/latest/sql-data-sources-jdbc.html - var ds1 = spark.read().format("jdbc") + org.apache.spark.sql.Dataset ds1 = spark.read().format("jdbc") .option("url", "jdbc:h2:" + databaseFile) //.option("dbtable", "DS1") .option("query", "select * from DS1") .load(); - var bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); - bindings.put("ds1", new SparkDataset(ds1, Map.of("id", Dataset.Role.IDENTIFIER))); + Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); + bindings.put("ds1", new SparkDataset(ds1, Java8Helpers.mapOf("id", Dataset.Role.IDENTIFIER))); engine.eval("ds2 := ds1[calc " + "col1 := COLCHAR || \" ok\", " + @@ -94,7 +94,7 @@ public void testReadSql() throws ScriptException { "]" + "[keep ID, col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11];"); - var ds2 = (Dataset) bindings.get("ds2"); + Dataset ds2 = (Dataset) bindings.get("ds2"); List> roundedDs2 = ds2.getDataAsList().stream().map(line -> line.stream().map(element -> { if (element instanceof Double) { @@ -107,10 +107,10 @@ public void testReadSql() throws ScriptException { ).collect(Collectors.toList()); assertThat(roundedDs2).containsExactlyInAnyOrder( - List.of(1L, "string1 ok", "string1 ok", 2L, 100L, 1.1D, 1.1991D, false, 1.2001D, 10099L, "2001", "2001"), - List.of(2L, "string2 ok", "string2 ok", 3L, 200L, 5.1D, 5.1991D, true, 5.2001D, 10199L, "2002", "2002"), - List.of(3L, "string3 ok", "string3 ok", 4L, 300L, 3.1D, 3.1991D, false, 3.2001D, 10299L, "2003", "2003"), - List.of(4L, "string4 ok", "string4 ok", 5L, 400L, 4.1D, 4.1991D, true, 4.2001D, 10399L, "2004", "2004") + Java8Helpers.listOf(1L, "string1 ok", "string1 ok", 2L, 100L, 1.1D, 1.1991D, false, 1.2001D, 10099L, "2001", "2001"), + Java8Helpers.listOf(2L, "string2 ok", "string2 ok", 3L, 200L, 5.1D, 5.1991D, true, 5.2001D, 10199L, "2002", "2002"), + Java8Helpers.listOf(3L, "string3 ok", "string3 ok", 4L, 300L, 3.1D, 3.1991D, false, 3.2001D, 10299L, "2003", "2003"), + Java8Helpers.listOf(4L, "string4 ok", "string4 ok", 5L, 400L, 4.1D, 4.1991D, true, 4.2001D, 10399L, "2004", "2004") ); } } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/TemporalTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/TemporalTest.java index 7a0fe0dfa..5223edc46 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/TemporalTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/TemporalTest.java @@ -3,6 +3,7 @@ import fr.insee.vtl.engine.VtlScriptEngine; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; +import fr.insee.vtl.model.utils.Java8Helpers; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -11,7 +12,6 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; import java.time.Instant; -import java.util.List; import static fr.insee.vtl.model.Structured.Component; import static org.assertj.core.api.Assertions.assertThat; @@ -34,63 +34,63 @@ public void setUp() { @Test public void testFlowToStock() throws ScriptException { - var ds = new InMemoryDataset( - List.of( + InMemoryDataset ds = new InMemoryDataset( + Java8Helpers.listOf( new Component("id1", String.class, Dataset.Role.IDENTIFIER), new Component("id2", Instant.class, Dataset.Role.IDENTIFIER), new Component("me1", Long.class, Dataset.Role.MEASURE) ), - List.of("A", Instant.parse("2009-01-01T00:00:00Z"), 2L), - List.of("A", Instant.parse("2011-01-01T00:00:00Z"), 5L), - List.of("A", Instant.parse("2012-01-01T00:00:00Z"), -3L), - List.of("B", Instant.parse("2010-01-01T00:00:00Z"), 9L), - List.of("B", Instant.parse("2011-01-01T00:00:00Z"), 4L), - List.of("B", Instant.parse("2013-01-01T00:00:00Z"), -6L) + Java8Helpers.listOf("A", Instant.parse("2009-01-01T00:00:00Z"), 2L), + Java8Helpers.listOf("A", Instant.parse("2011-01-01T00:00:00Z"), 5L), + Java8Helpers.listOf("A", Instant.parse("2012-01-01T00:00:00Z"), -3L), + Java8Helpers.listOf("B", Instant.parse("2010-01-01T00:00:00Z"), 9L), + Java8Helpers.listOf("B", Instant.parse("2011-01-01T00:00:00Z"), 4L), + Java8Helpers.listOf("B", Instant.parse("2013-01-01T00:00:00Z"), -6L) ); engine.put("ds", ds); engine.eval("r := flow_to_stock(ds);"); assertThat(engine.get("r")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.get("r")).getDataAsList()).containsExactlyInAnyOrder( - List.of("A", Instant.parse("2009-01-01T00:00:00Z"), 2L), - List.of("A", Instant.parse("2011-01-01T00:00:00Z"), 7L), - List.of("A", Instant.parse("2012-01-01T00:00:00Z"), 4L), - List.of("B", Instant.parse("2010-01-01T00:00:00Z"), 9L), - List.of("B", Instant.parse("2011-01-01T00:00:00Z"), 13L), - List.of("B", Instant.parse("2013-01-01T00:00:00Z"), 7L) + Java8Helpers.listOf("A", Instant.parse("2009-01-01T00:00:00Z"), 2L), + Java8Helpers.listOf("A", Instant.parse("2011-01-01T00:00:00Z"), 7L), + Java8Helpers.listOf("A", Instant.parse("2012-01-01T00:00:00Z"), 4L), + Java8Helpers.listOf("B", Instant.parse("2010-01-01T00:00:00Z"), 9L), + Java8Helpers.listOf("B", Instant.parse("2011-01-01T00:00:00Z"), 13L), + Java8Helpers.listOf("B", Instant.parse("2013-01-01T00:00:00Z"), 7L) ); } @Test public void testStockToFlow() throws ScriptException { - var ds = new InMemoryDataset( - List.of( + InMemoryDataset ds = new InMemoryDataset( + Java8Helpers.listOf( new Component("id1", String.class, Dataset.Role.IDENTIFIER), new Component("id2", Instant.class, Dataset.Role.IDENTIFIER), new Component("me1", Long.class, Dataset.Role.MEASURE), new Component("me2", String.class, Dataset.Role.MEASURE) ), - List.of("A", Instant.parse("2010-01-01T00:00:00Z"), 2L, "foo"), - List.of("A", Instant.parse("2011-01-01T00:00:00Z"), 7L, "foo"), - List.of("A", Instant.parse("2012-01-01T00:00:00Z"), 4L, "foo"), - List.of("A", Instant.parse("2013-01-01T00:00:00Z"), 13L, "foo"), - List.of("B", Instant.parse("2010-01-01T00:00:00Z"), 4L, "foo"), - List.of("B", Instant.parse("2011-01-01T00:00:00Z"), -4L, "foo"), - List.of("B", Instant.parse("2012-01-01T00:00:00Z"), -4L, "foo"), - List.of("B", Instant.parse("2013-01-01T00:00:00Z"), 2L, "foo") + Java8Helpers.listOf("A", Instant.parse("2010-01-01T00:00:00Z"), 2L, "foo"), + Java8Helpers.listOf("A", Instant.parse("2011-01-01T00:00:00Z"), 7L, "foo"), + Java8Helpers.listOf("A", Instant.parse("2012-01-01T00:00:00Z"), 4L, "foo"), + Java8Helpers.listOf("A", Instant.parse("2013-01-01T00:00:00Z"), 13L, "foo"), + Java8Helpers.listOf("B", Instant.parse("2010-01-01T00:00:00Z"), 4L, "foo"), + Java8Helpers.listOf("B", Instant.parse("2011-01-01T00:00:00Z"), -4L, "foo"), + Java8Helpers.listOf("B", Instant.parse("2012-01-01T00:00:00Z"), -4L, "foo"), + Java8Helpers.listOf("B", Instant.parse("2013-01-01T00:00:00Z"), 2L, "foo") ); engine.put("ds", ds); engine.eval("res := stock_to_flow(ds);"); assertThat(engine.get("res")).isInstanceOf(Dataset.class); ((Dataset) engine.get("res")).getDataAsMap().forEach(System.out::println); assertThat(((Dataset) engine.get("res")).getDataAsList()).containsExactly( - List.of("A", Instant.parse("2010-01-01T00:00:00Z"), "foo", 2L), - List.of("A", Instant.parse("2011-01-01T00:00:00Z"), "foo", 5L), - List.of("A", Instant.parse("2012-01-01T00:00:00Z"), "foo", -3L), - List.of("A", Instant.parse("2013-01-01T00:00:00Z"), "foo", 9L), - List.of("B", Instant.parse("2010-01-01T00:00:00Z"), "foo", 4L), - List.of("B", Instant.parse("2011-01-01T00:00:00Z"), "foo", -8L), - List.of("B", Instant.parse("2012-01-01T00:00:00Z"), "foo", 0L), - List.of("B", Instant.parse("2013-01-01T00:00:00Z"), "foo", 6L) + Java8Helpers.listOf("A", Instant.parse("2010-01-01T00:00:00Z"), "foo", 2L), + Java8Helpers.listOf("A", Instant.parse("2011-01-01T00:00:00Z"), "foo", 5L), + Java8Helpers.listOf("A", Instant.parse("2012-01-01T00:00:00Z"), "foo", -3L), + Java8Helpers.listOf("A", Instant.parse("2013-01-01T00:00:00Z"), "foo", 9L), + Java8Helpers.listOf("B", Instant.parse("2010-01-01T00:00:00Z"), "foo", 4L), + Java8Helpers.listOf("B", Instant.parse("2011-01-01T00:00:00Z"), "foo", -8L), + Java8Helpers.listOf("B", Instant.parse("2012-01-01T00:00:00Z"), "foo", 0L), + Java8Helpers.listOf("B", Instant.parse("2013-01-01T00:00:00Z"), "foo", 6L) ); } } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/CalcTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/CalcTest.java index 8e569c725..54b2b51aa 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/CalcTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/CalcTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -13,8 +14,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -45,13 +44,13 @@ public void tearDown() { } InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); @Test @@ -61,12 +60,12 @@ public void testCalcClause() throws ScriptException, InterruptedException { engine.eval("ds := ds1[calc test := between(age, 10, 11), age := age * 2, attribute wisdom := (weight + age) / 2];"); - var ds = (Dataset) engine.getContext().getAttribute("ds"); + Dataset ds = (Dataset) engine.getContext().getAttribute("ds"); assertThat(ds).isInstanceOf(Dataset.class); - assertThat(ds.getDataAsMap()).isEqualTo(List.of( - Map.of("name", "Hadrien", "age", 20L, "test", true, "weight", 11L, "wisdom", 10.5D), - Map.of("name", "Nico", "age", 22L, "test", true, "weight", 10L, "wisdom", 10.5D), - Map.of("name", "Franck", "age", 24L, "test", false, "weight", 9L, "wisdom", 10.5D) + assertThat(ds.getDataAsMap()).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 20L, "test", true, "weight", 11L, "wisdom", 10.5D), + Java8Helpers.mapOf("name", "Nico", "age", 22L, "test", true, "weight", 10L, "wisdom", 10.5D), + Java8Helpers.mapOf("name", "Franck", "age", 24L, "test", false, "weight", 9L, "wisdom", 10.5D) )); assertThat(ds.getDataStructure()).containsValues( new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/FilterTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/FilterTest.java index 8217e8296..1d57b33da 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/FilterTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/FilterTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -13,21 +14,19 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; public class FilterTest { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); private SparkSession spark; private ScriptEngine engine; @@ -62,9 +61,9 @@ public void testFilterClause() throws ScriptException { engine.eval("ds := ds1[filter age > 10 and age < 12];"); assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(Dataset.class); - var ds = (Dataset) engine.getContext().getAttribute("ds"); - assertThat(ds.getDataAsMap()).isEqualTo(List.of( - Map.of("name", "Nico", "age", 11L, "weight", 10L) + Dataset ds = (Dataset) engine.getContext().getAttribute("ds"); + assertThat(ds.getDataAsMap()).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L) )); assertThat(ds.getDataStructure()).containsValues( diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/JoinTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/JoinTest.java index 41b2a3e89..18ca6775f 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/JoinTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/JoinTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -14,46 +15,45 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; import java.util.Arrays; -import java.util.List; import static org.assertj.core.api.Assertions.assertThat; public class JoinTest { private final InMemoryDataset dataset1 = new InMemoryDataset( - List.of( - List.of("a", 1L, 2L), - List.of("b", 3L, 4L), - List.of("c", 5L, 6L), - List.of("d", 7L, 8L) + Java8Helpers.listOf( + Java8Helpers.listOf("a", 1L, 2L), + Java8Helpers.listOf("b", 3L, 4L), + Java8Helpers.listOf("c", 5L, 6L), + Java8Helpers.listOf("d", 7L, 8L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("age", Long.class, Dataset.Role.MEASURE), new Structured.Component("weight", Long.class, Dataset.Role.MEASURE) ) ); private final InMemoryDataset dataset2 = new InMemoryDataset( - List.of( - List.of(9L, "a", 10L), - List.of(11L, "b", 12L), - List.of(12L, "c", 13L), - List.of(14L, "c", 15L) + Java8Helpers.listOf( + Java8Helpers.listOf(9L, "a", 10L), + Java8Helpers.listOf(11L, "b", 12L), + Java8Helpers.listOf(12L, "c", 13L), + Java8Helpers.listOf(14L, "c", 15L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("age2", Long.class, Dataset.Role.MEASURE), new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("weight2", Long.class, Dataset.Role.MEASURE) ) ); private final InMemoryDataset dataset3 = new InMemoryDataset( - List.of( - List.of(16L, "a", 17L), - List.of(18L, "b", 19L), - List.of(20L, "c", 21L), - List.of(22L, "c", 23L) + Java8Helpers.listOf( + Java8Helpers.listOf(16L, "a", 17L), + Java8Helpers.listOf(18L, "b", 19L), + Java8Helpers.listOf(20L, "c", 21L), + Java8Helpers.listOf(22L, "c", 23L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("age3", Long.class, Dataset.Role.MEASURE), new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("weight3", Long.class, Dataset.Role.MEASURE) @@ -93,7 +93,7 @@ public void testLeftJoin() throws ScriptException { engine.eval("result := left_join(ds_1 as ds1, ds_2 as ds2, ds_3 as ds3);"); - var result = (Dataset) context.getAttribute("result"); + Dataset result = (Dataset) context.getAttribute("result"); assertThat(result.getDataAsList()).containsExactlyInAnyOrder( Arrays.asList("a", 2L, 16L, 17L, 10L, 1L, 9L), Arrays.asList("b", 4L, 18L, 19L, 12L, 3L, 11L), @@ -124,7 +124,7 @@ public void testInnerJoin() throws ScriptException { engine.eval("result := inner_join(ds1 as dsOne, ds2, ds3);"); - var resultInner = (Dataset) context.getAttribute("result"); + Dataset resultInner = (Dataset) context.getAttribute("result"); assertThat(resultInner.getDataAsList()).containsExactlyInAnyOrder( Arrays.asList("a", 2L, 16L, 17L, 10L, 1L, 9L), Arrays.asList("b", 4L, 18L, 19L, 12L, 3L, 11L), @@ -149,8 +149,8 @@ public void testInnerJoin() throws ScriptException { public void testFullJoin() throws ScriptException { ScriptContext context = engine.getContext(); - var ds1 = new InMemoryDataset( - List.of( + InMemoryDataset ds1 = new InMemoryDataset( + Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("m1", Long.class, Dataset.Role.MEASURE) ), @@ -159,8 +159,8 @@ public void testFullJoin() throws ScriptException { Arrays.asList("d", 3L) ); - var ds2 = new InMemoryDataset( - List.of( + InMemoryDataset ds2 = new InMemoryDataset( + Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("m1", Long.class, Dataset.Role.MEASURE) ), @@ -169,8 +169,8 @@ public void testFullJoin() throws ScriptException { Arrays.asList("c", 6L) ); - var ds3 = new InMemoryDataset( - List.of( + InMemoryDataset ds3 = new InMemoryDataset( + Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("m1", Long.class, Dataset.Role.MEASURE) ), @@ -184,7 +184,7 @@ public void testFullJoin() throws ScriptException { engine.eval("result := full_join(ds_1 as ds1, ds_2 as ds2, ds_3 as ds3);"); - var result = (Dataset) context.getAttribute("result"); + Dataset result = (Dataset) context.getAttribute("result"); assertThat(result.getDataStructure().values()).containsExactlyInAnyOrder( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), @@ -209,7 +209,7 @@ public void testCrossJoin() throws ScriptException { engine.eval("result := cross_join(ds1 as dsOne, ds2, ds3);"); - var resultCross = (Dataset) context.getAttribute("result"); + Dataset resultCross = (Dataset) context.getAttribute("result"); assertThat(resultCross.getDataAsList()).containsExactlyInAnyOrder( Arrays.asList("a", 1L, 2L, 9L, "a", 10L, 16L, "a", 17L), Arrays.asList("a", 1L, 2L, 9L, "a", 10L, 18L, "b", 19L), diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java index aea44dc66..22435130a 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/OperatorsTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.spark.samples.DatasetSamples; import org.apache.spark.sql.SparkSession; @@ -12,7 +13,6 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -74,7 +74,7 @@ public void testOperators() throws ScriptException { "ds_1 := ds1[keep id, string1, string2]; ds_2 := ds2[keep id, string1][calc string2 := string1]; " + "res := ds_1 || ds_2; " ); - var res = engine.getContext().getAttribute("res"); + Object res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataStructure().get("string1").getType()).isEqualTo(String.class); } @@ -85,11 +85,11 @@ public void testPlan() throws ScriptException { engine.eval("ds1 := ds_1[keep id, long1][rename long1 to bool_var]; " + "ds2 := ds_2[keep id, long1][rename long1 to bool_var]; " + "res := if ds1 > ds2 then ds1 else ds2;"); - var res = engine.getContext().getAttribute("res"); + Object res = engine.getContext().getAttribute("res"); assertThat(((Dataset) res).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("id", "Hadrien", "bool_var", 150L), - Map.of("id", "Nico", "bool_var", 20L), - Map.of("id", "Franck", "bool_var", 100L) + Java8Helpers.mapOf("id", "Hadrien", "bool_var", 150L), + Java8Helpers.mapOf("id", "Nico", "bool_var", 20L), + Java8Helpers.mapOf("id", "Franck", "bool_var", 100L) ); assertThat(((Dataset) res).getDataStructure().get("bool_var").getType()).isEqualTo(Long.class); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java index 0e6e03445..b8db3a8c1 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ProjectTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -13,21 +14,19 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; public class ProjectTest { private final InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); private SparkSession spark; private ScriptEngine engine; @@ -64,19 +63,19 @@ public void testProjection() throws ScriptException { assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(fr.insee.vtl.model.Dataset.class); assertThat(((fr.insee.vtl.model.Dataset) engine.getContext().getAttribute("ds")).getDataAsMap()).containsExactly( - Map.of("name", "Hadrien", "age", 10L), - Map.of("name", "Nico", "age", 11L), - Map.of("name", "Franck", "age", 12L) + Java8Helpers.mapOf("name", "Hadrien", "age", 10L), + Java8Helpers.mapOf("name", "Nico", "age", 11L), + Java8Helpers.mapOf("name", "Franck", "age", 12L) ); engine.eval("ds := ds1[drop weight];"); assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(fr.insee.vtl.model.Dataset.class); - var ds = (Dataset) engine.getContext().getAttribute("ds"); + Dataset ds = (Dataset) engine.getContext().getAttribute("ds"); assertThat(ds.getDataAsMap()).containsExactly( - Map.of("name", "Hadrien", "age", 10L), - Map.of("name", "Nico", "age", 11L), - Map.of("name", "Franck", "age", 12L) + Java8Helpers.mapOf("name", "Hadrien", "age", 10L), + Java8Helpers.mapOf("name", "Nico", "age", 11L), + Java8Helpers.mapOf("name", "Franck", "age", 12L) ); assertThat(ds.getDataStructure()).containsValues( new Structured.Component("name", String.class, Dataset.Role.IDENTIFIER), diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/RenameTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/RenameTest.java index 2622afc9d..d84012ee8 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/RenameTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/RenameTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -13,21 +14,19 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; public class RenameTest { private final InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "age", 10L, "weight", 11L), - Map.of("name", "Nico", "age", 11L, "weight", 10L), - Map.of("name", "Franck", "age", 12L, "weight", 9L) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "age", 10L, "weight", 11L), + Java8Helpers.mapOf("name", "Nico", "age", 11L, "weight", 10L), + Java8Helpers.mapOf("name", "Franck", "age", 12L, "weight", 9L) ), - Map.of("name", String.class, "age", Long.class, "weight", Long.class), - Map.of("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "age", Long.class, "weight", Long.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); private SparkSession spark; private ScriptEngine engine; @@ -63,11 +62,11 @@ public void testRename() throws ScriptException { engine.eval("ds := ds1[rename age to weight, weight to age, name to pseudo];"); assertThat(engine.getContext().getAttribute("ds")).isInstanceOf(Dataset.class); - var ds = (Dataset) engine.getContext().getAttribute("ds"); + Dataset ds = (Dataset) engine.getContext().getAttribute("ds"); assertThat(ds.getDataAsMap()).containsExactlyInAnyOrder( - Map.of("pseudo", "Hadrien", "weight", 10L, "age", 11L), - Map.of("pseudo", "Nico", "weight", 11L, "age", 10L), - Map.of("pseudo", "Franck", "weight", 12L, "age", 9L) + Java8Helpers.mapOf("pseudo", "Hadrien", "weight", 10L, "age", 11L), + Java8Helpers.mapOf("pseudo", "Nico", "weight", 11L, "age", 10L), + Java8Helpers.mapOf("pseudo", "Franck", "weight", 12L, "age", 9L) ); assertThat(ds.getDataStructure()).containsValues( new Structured.Component("pseudo", String.class, Dataset.Role.IDENTIFIER), diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ServiceLoaderTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ServiceLoaderTest.java index f8503c5b7..4af77f452 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ServiceLoaderTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ServiceLoaderTest.java @@ -3,9 +3,9 @@ import fr.insee.vtl.model.ProcessingEngineFactory; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; -import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @@ -13,10 +13,12 @@ public class ServiceLoaderTest { @Test public void testServiceLoader() { - List processingEngines = ServiceLoader.load(ProcessingEngineFactory.class).stream() - .map(ServiceLoader.Provider::get) - .map(ProcessingEngineFactory::getName) - .collect(Collectors.toList()); + List processingEngines = new ArrayList<>(); + ServiceLoader factories = ServiceLoader.load(ProcessingEngineFactory.class); + for (ProcessingEngineFactory factory : factories) { + processingEngines.add(factory.getName()); + } + assertThat(processingEngines).containsExactlyInAnyOrder( "memory", "spark" ); diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/UnionTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/UnionTest.java index 3ebda67bb..a99bb5dba 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/UnionTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/UnionTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -14,65 +15,59 @@ import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public class UnionTest { private final InMemoryDataset unionDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) ); private final InMemoryDataset unionDS2 = new InMemoryDataset( - List.of( - Map.of("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), - Map.of("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) ); private final InMemoryDataset unionDS3 = new InMemoryDataset( - List.of( - Map.of("Id_1", "2012", "Id_2", "L", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "M", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "L", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "M", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) ); private final InMemoryDataset unionDS4 = new InMemoryDataset( - List.of( - Map.of("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Id_4", String.class, "Me_1", Long.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Id_4", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) ); private final InMemoryDataset unionDS5 = new InMemoryDataset( - List.of( - Map.of("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Me_1", 3L) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Me_1", 3L) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Me_1", Long.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Id_3", String.class, "Me_1", Long.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Id_3", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE) ); private final String DEFAULT_NULL_STR = "null"; private SparkSession spark; @@ -192,11 +187,11 @@ public void testUnionTwoWithoutDup() throws ScriptException { for (Map map : actualWithNull) { actual.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - List> expected = List.of(Map.of("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), - Map.of("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), - Map.of("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L) + List> expected = Java8Helpers.listOf(Java8Helpers.mapOf("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L) ); assertEquals(new HashSet<>(actual), new HashSet<>(expected)); } @@ -258,15 +253,15 @@ public void testUnionThreeWithoutDup() throws ScriptException { for (Map map : actualWithNull) { actual.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - List> expected = List.of( - Map.of("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), - Map.of("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), - Map.of("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), - Map.of("Id_1", "2012", "Id_2", "M", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "L", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L) + List> expected = Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "M", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "L", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L) ); assertEquals(new HashSet<>(actual), new HashSet<>(expected)); } @@ -330,13 +325,13 @@ public void testUnionThreeWithDup() throws ScriptException { for (Map map : actualWithNull) { actual.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - List> expected = List.of( - Map.of("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), - Map.of("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), - Map.of("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), - Map.of("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), - Map.of("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) + List> expected = Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "G", "Id_3", "Total", "Id_4", "Total", "Me_1", 2L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "F", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "B", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "S", "Id_3", "Total", "Id_4", "Total", "Me_1", 5L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "N", "Id_3", "Total", "Id_4", "Total", "Me_1", 23L), + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "X", "Id_3", "Total", "Id_4", "Total", "Me_1", 3L) ); assertEquals(new HashSet<>(actual), new HashSet<>(expected)); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java index b210523aa..f6f20c2e4 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing.engine/ValidationTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; @@ -25,13 +26,13 @@ public class ValidationTest { private final String DEFAULT_NULL_STR = "null"; private final InMemoryDataset dataset = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", 10L), - List.of("2011", "I", "DEBIT", -2L), - List.of("2012", "I", "CREDIT", 10L), - List.of("2012", "I", "DEBIT", 2L) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", 10L), + Java8Helpers.listOf("2011", "I", "DEBIT", -2L), + Java8Helpers.listOf("2012", "I", "CREDIT", 10L), + Java8Helpers.listOf("2012", "I", "DEBIT", 2L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER, null, "vd_id_3"), @@ -39,55 +40,55 @@ public class ValidationTest { ) ); private final InMemoryDataset ds_1_check = new InMemoryDataset( - List.of( - List.of("2010", "I", 1L), - List.of("2011", "I", 2L), - List.of("2012", "I", 10L), - List.of("2013", "I", 4L), - List.of("2014", "I", 5L), - List.of("2015", "I", 6L), - List.of("2010", "D", 25L), - List.of("2011", "D", 35L), - List.of("2012", "D", 45L), - List.of("2013", "D", 55L), - List.of("2014", "D", 50L), - List.of("2015", "D", 75L) + Java8Helpers.listOf( + Java8Helpers.listOf("2010", "I", 1L), + Java8Helpers.listOf("2011", "I", 2L), + Java8Helpers.listOf("2012", "I", 10L), + Java8Helpers.listOf("2013", "I", 4L), + Java8Helpers.listOf("2014", "I", 5L), + Java8Helpers.listOf("2015", "I", 6L), + Java8Helpers.listOf("2010", "D", 25L), + Java8Helpers.listOf("2011", "D", 35L), + Java8Helpers.listOf("2012", "D", 45L), + Java8Helpers.listOf("2013", "D", 55L), + Java8Helpers.listOf("2014", "D", 50L), + Java8Helpers.listOf("2015", "D", 75L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Me_1", Long.class, Dataset.Role.MEASURE) ) ); private final InMemoryDataset ds_2_check = new InMemoryDataset( - List.of( - List.of("2010", "I", 9L), - List.of("2011", "I", 2L), - List.of("2012", "I", 10L), - List.of("2013", "I", 7L), - List.of("2014", "I", 5L), - List.of("2015", "I", 6L), - List.of("2010", "D", 50L), - List.of("2011", "D", 35L), - List.of("2012", "D", 40L), - List.of("2013", "D", 55L), - List.of("2014", "D", 65L), - List.of("2015", "D", 75L) + Java8Helpers.listOf( + Java8Helpers.listOf("2010", "I", 9L), + Java8Helpers.listOf("2011", "I", 2L), + Java8Helpers.listOf("2012", "I", 10L), + Java8Helpers.listOf("2013", "I", 7L), + Java8Helpers.listOf("2014", "I", 5L), + Java8Helpers.listOf("2015", "I", 6L), + Java8Helpers.listOf("2010", "D", 50L), + Java8Helpers.listOf("2011", "D", 35L), + Java8Helpers.listOf("2012", "D", 40L), + Java8Helpers.listOf("2013", "D", 55L), + Java8Helpers.listOf("2014", "D", 65L), + Java8Helpers.listOf("2015", "D", 75L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Me_1", Long.class, Dataset.Role.MEASURE) ) ); private final Dataset dsExpr = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", true), - List.of("2011", "I", "DEBIT", false), - List.of("2012", "I", "CREDIT", false), - List.of("2012", "I", "DEBIT", true) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", true), + Java8Helpers.listOf("2011", "I", "DEBIT", false), + Java8Helpers.listOf("2012", "I", "CREDIT", false), + Java8Helpers.listOf("2012", "I", "DEBIT", true) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), @@ -95,13 +96,13 @@ public class ValidationTest { ) ); private final Dataset dsImbalance = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", 1L), - List.of("2011", "I", "DEBIT", 2L), - List.of("2012", "I", "CREDIT", 2L), - List.of("2012", "I", "DEBIT", 3L) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", 1L), + Java8Helpers.listOf("2011", "I", "DEBIT", 2L), + Java8Helpers.listOf("2012", "I", "CREDIT", 2L), + Java8Helpers.listOf("2012", "I", "DEBIT", 3L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), @@ -109,13 +110,13 @@ public class ValidationTest { ) ); private final Dataset dsImbalanceToRename = new InMemoryDataset( - List.of( - List.of("2011", "I", "CREDIT", 1L), - List.of("2011", "I", "DEBIT", 2L), - List.of("2012", "I", "CREDIT", 2L), - List.of("2012", "I", "DEBIT", 3L) + Java8Helpers.listOf( + Java8Helpers.listOf("2011", "I", "CREDIT", 1L), + Java8Helpers.listOf("2011", "I", "DEBIT", 2L), + Java8Helpers.listOf("2012", "I", "CREDIT", 2L), + Java8Helpers.listOf("2012", "I", "DEBIT", 3L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_3", String.class, Dataset.Role.IDENTIFIER), @@ -136,24 +137,24 @@ public class ValidationTest { "R110 : M <= G errorlevel 5\n" + "end hierarchical ruleset; \n"; private final Dataset DS_1_HR = new InMemoryDataset( - List.of( - List.of("2010", "A", 5L), - List.of("2010", "B", 11L), - List.of("2010", "C", 0L), - List.of("2010", "G", 19L), + Java8Helpers.listOf( + Java8Helpers.listOf("2010", "A", 5L), + Java8Helpers.listOf("2010", "B", 11L), + Java8Helpers.listOf("2010", "C", 0L), + Java8Helpers.listOf("2010", "G", 19L), Stream.of("2010", "H", null).collect(Collectors.toList()), - List.of("2010", "I", 14L), - List.of("2010", "M", 2L), - List.of("2010", "N", 5L), - List.of("2010", "O", 4L), - List.of("2010", "P", 7L), - List.of("2010", "Q", -7L), - List.of("2010", "S", 3L), - List.of("2010", "T", 9L), + Java8Helpers.listOf("2010", "I", 14L), + Java8Helpers.listOf("2010", "M", 2L), + Java8Helpers.listOf("2010", "N", 5L), + Java8Helpers.listOf("2010", "O", 4L), + Java8Helpers.listOf("2010", "P", 7L), + Java8Helpers.listOf("2010", "Q", -7L), + Java8Helpers.listOf("2010", "S", 3L), + Java8Helpers.listOf("2010", "T", 9L), Stream.of("2010", "U", null).collect(Collectors.toList()), - List.of("2010", "V", 6L) + Java8Helpers.listOf("2010", "V", 6L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Me_1", Long.class, Dataset.Role.MEASURE) @@ -246,34 +247,34 @@ public void testValidateDPruleset() throws ScriptException { } assertThat(DS_rWithoutNull).containsExactlyInAnyOrder( - Map.of("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", "Me_1", -2L, "ruleid", "dpr1_2", "errorcode", "Bad debit", "errorlevel", 1L) ).containsExactlyInAnyOrderElementsOf(DS_r_invalidWithoutNull); assertThat(DS_r_allWithoutNull).containsExactlyInAnyOrder( - Map.of("Id_1", "2011", "Id_2", "I", "Id_3", "CREDIT", + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "I", "Id_3", "CREDIT", "Me_1", 10L, "ruleid", "ruleA", "bool_var", true, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2011", "Id_2", "I", "Id_3", "CREDIT", + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "I", "Id_3", "CREDIT", "Me_1", 10L, "ruleid", "dpr1_2", "bool_var", true, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", "Me_1", -2L, "ruleid", "ruleA", "bool_var", true, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", "Me_1", -2L, "ruleid", "dpr1_2", "bool_var", false, "errorcode", "Bad debit", "errorlevel", 1L), - Map.of("Id_1", "2012", "Id_2", "I", "Id_3", "CREDIT", + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "I", "Id_3", "CREDIT", "Me_1", 10L, "ruleid", "ruleA", "bool_var", true, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2012", "Id_2", "I", "Id_3", "CREDIT", + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "I", "Id_3", "CREDIT", "Me_1", 10L, "ruleid", "dpr1_2", "bool_var", true, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2012", "Id_2", "I", "Id_3", "DEBIT", + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "I", "Id_3", "DEBIT", "Me_1", 2L, "ruleid", "ruleA", "bool_var", true, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2012", "Id_2", "I", "Id_3", "DEBIT", + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "I", "Id_3", "DEBIT", "Me_1", 2L, "ruleid", "dpr1_2", "bool_var", true, "errorcode", "null", "errorlevel", "null") ).containsExactlyInAnyOrderElementsOf(DS_r_all_measuresWithoutNull); @@ -317,7 +318,7 @@ public void testValidateDPrulesetWithAlias() throws ScriptException { } assertThat(DS_rWithoutNull).containsExactlyInAnyOrder( - Map.of("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "I", "Id_3", "DEBIT", "Me_1", -2L, "ruleid", "dpr1_2", "errorcode", "Bad debit", "errorlevel", "null") ); @@ -332,7 +333,7 @@ public void testCheck() throws ScriptException { engine.eval("ds := check(DS1 >= DS2 errorcode \"err\" errorlevel 1 imbalance DS1 - DS2);" + "ds1 := check(DS1 >= DS2 errorcode \"err\" errorlevel 1 imbalance DS1 - DS2 invalid);"); - var ds = (Dataset) engine.getContext().getAttribute("ds"); + Dataset ds = (Dataset) engine.getContext().getAttribute("ds"); assertThat(ds).isInstanceOf(Dataset.class); List> dsWithNull = ds.getDataAsMap(); @@ -341,30 +342,30 @@ public void testCheck() throws ScriptException { dsWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "I", "bool_var", false, + assertThat(dsWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "I", "bool_var", false, "imbalance", -8L, "errorcode", "err", "errorlevel", 1L), - Map.of("Id_1", "2011", "Id_2", "I", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "I", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2012", "Id_2", "I", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "I", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2013", "Id_2", "I", "bool_var", false, + Java8Helpers.mapOf("Id_1", "2013", "Id_2", "I", "bool_var", false, "imbalance", -3L, "errorcode", "err", "errorlevel", 1L), - Map.of("Id_1", "2014", "Id_2", "I", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2014", "Id_2", "I", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2015", "Id_2", "I", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2015", "Id_2", "I", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "D", "bool_var", false, + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "D", "bool_var", false, "imbalance", -25L, "errorcode", "err", "errorlevel", 1L), - Map.of("Id_1", "2011", "Id_2", "D", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2011", "Id_2", "D", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2012", "Id_2", "D", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2012", "Id_2", "D", "bool_var", true, "imbalance", 5L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2013", "Id_2", "D", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2013", "Id_2", "D", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2014", "Id_2", "D", "bool_var", false, + Java8Helpers.mapOf("Id_1", "2014", "Id_2", "D", "bool_var", false, "imbalance", -15L, "errorcode", "err", "errorlevel", 1L), - Map.of("Id_1", "2015", "Id_2", "D", "bool_var", true, + Java8Helpers.mapOf("Id_1", "2015", "Id_2", "D", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"))); assertThat(ds.getDataStructure()).containsValues( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), @@ -375,7 +376,7 @@ public void testCheck() throws ScriptException { new Structured.Component("errorlevel", Long.class, Dataset.Role.MEASURE) ); - var ds1 = (Dataset) engine.getContext().getAttribute("ds1"); + Dataset ds1 = (Dataset) engine.getContext().getAttribute("ds1"); assertThat(ds1).isInstanceOf(Dataset.class); List> ds1WithNull = ds1.getDataAsMap(); @@ -384,14 +385,14 @@ public void testCheck() throws ScriptException { ds1WithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(ds1WithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "I", + assertThat(ds1WithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "I", "imbalance", -8L, "errorcode", "err", "errorlevel", 1L), - Map.of("Id_1", "2013", "Id_2", "I", + Java8Helpers.mapOf("Id_1", "2013", "Id_2", "I", "imbalance", -3L, "errorcode", "err", "errorlevel", 1L), - Map.of("Id_1", "2010", "Id_2", "D", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "D", "imbalance", -25L, "errorcode", "err", "errorlevel", 1L), - Map.of("Id_1", "2014", "Id_2", "D", + Java8Helpers.mapOf("Id_1", "2014", "Id_2", "D", "imbalance", -15L, "errorcode", "err", "errorlevel", 1L))); } @@ -468,11 +469,11 @@ public void checkHierarchyOutputMode() throws ScriptException { for (Map map : dsRWithNull) { dsRWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + assertThat(dsRWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "Me_1", 19L, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "Me_1", 2L, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L) )); @@ -492,20 +493,20 @@ public void checkHierarchyOutputMode() throws ScriptException { for (Map map : dsRAllWithNull) { dsRAllWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRAllWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + assertThat(dsRAllWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -525,20 +526,20 @@ public void checkHierarchyOutputMode() throws ScriptException { for (Map map : dsRAllMeasuresWithNull) { dsRAllMeasuresWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRAllMeasuresWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + assertThat(dsRAllMeasuresWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "Me_1", 11L, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "Me_1", 0L, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "Me_1", 19L, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "Me_1", 2L, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "Me_1", 2L, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -574,20 +575,20 @@ public void checkHierarchyValidationMode() throws ScriptException { for (Map map : dsRNonNullWithNull) { dsRNonNullWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRNonNullWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + assertThat(dsRNonNullWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -598,35 +599,35 @@ public void checkHierarchyValidationMode() throws ScriptException { for (Map map : dsRNonZeroWithNull) { dsRNonZeroWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRNonZeroWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "A", "ruleid", "R010", + assertThat(dsRNonZeroWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "A", "ruleid", "R010", "bool_var", false, "imbalance", 5L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "D", "ruleid", "R040", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "D", "ruleid", "R040", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 1L), - Map.of("Id_1", "2010", "Id_2", "E", "ruleid", "R050", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "E", "ruleid", "R050", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "H", "ruleid", "R080", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "H", "ruleid", "R080", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "I", "ruleid", "R090", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "I", "ruleid", "R090", "bool_var", false, "imbalance", -5L, "errorcode", "YY", "errorlevel", 0L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -638,32 +639,32 @@ public void checkHierarchyValidationMode() throws ScriptException { dsRPartialNullWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRPartialNullWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "A", "ruleid", "R010", + assertThat(dsRPartialNullWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "A", "ruleid", "R010", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "D", "ruleid", "R040", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "D", "ruleid", "R040", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "E", "ruleid", "R050", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "E", "ruleid", "R050", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "I", "ruleid", "R090", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "I", "ruleid", "R090", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -675,32 +676,32 @@ public void checkHierarchyValidationMode() throws ScriptException { dsRPartialZeroWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRPartialZeroWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "A", "ruleid", "R010", + assertThat(dsRPartialZeroWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "A", "ruleid", "R010", "bool_var", false, "imbalance", 5L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "D", "ruleid", "R040", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "D", "ruleid", "R040", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 1L), - Map.of("Id_1", "2010", "Id_2", "E", "ruleid", "R050", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "E", "ruleid", "R050", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "I", "ruleid", "R090", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "I", "ruleid", "R090", "bool_var", false, "imbalance", -5L, "errorcode", "YY", "errorlevel", 0L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -712,38 +713,38 @@ public void checkHierarchyValidationMode() throws ScriptException { dsRAlwaysNullWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRAlwaysNullWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "A", "ruleid", "R010", + assertThat(dsRAlwaysNullWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "A", "ruleid", "R010", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "D", "ruleid", "R040", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "D", "ruleid", "R040", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "E", "ruleid", "R050", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "E", "ruleid", "R050", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "F", "ruleid", "R060", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "F", "ruleid", "R060", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "H", "ruleid", "R080", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "H", "ruleid", "R080", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "I", "ruleid", "R090", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "I", "ruleid", "R090", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -755,38 +756,38 @@ public void checkHierarchyValidationMode() throws ScriptException { dsRAlwaysZeroWithoutNull.add(replaceNullValues(map, DEFAULT_NULL_STR)); } - assertThat(dsRAlwaysZeroWithoutNull).isEqualTo(List.of( - Map.of("Id_1", "2010", "Id_2", "A", "ruleid", "R010", + assertThat(dsRAlwaysZeroWithoutNull).isEqualTo(Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "A", "ruleid", "R010", "bool_var", false, "imbalance", 5L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "B", "ruleid", "R020", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "B", "ruleid", "R020", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "C", "ruleid", "R030", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "C", "ruleid", "R030", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "D", "ruleid", "R040", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "D", "ruleid", "R040", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 1L), - Map.of("Id_1", "2010", "Id_2", "E", "ruleid", "R050", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "E", "ruleid", "R050", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "F", "ruleid", "R060", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "F", "ruleid", "R060", "bool_var", true, "imbalance", 0L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "G", "ruleid", "R070", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "G", "ruleid", "R070", "bool_var", false, "imbalance", 8L, "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "H", "ruleid", "R080", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "H", "ruleid", "R080", "bool_var", "null", "imbalance", "null", "errorcode", "null", "errorlevel", "null"), - Map.of("Id_1", "2010", "Id_2", "I", "ruleid", "R090", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "I", "ruleid", "R090", "bool_var", false, "imbalance", -5L, "errorcode", "YY", "errorlevel", 0L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R100", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R100", "bool_var", false, "imbalance", -3L, "errorcode", "null", "errorlevel", 5L), - Map.of("Id_1", "2010", "Id_2", "M", "ruleid", "R110", + Java8Helpers.mapOf("Id_1", "2010", "Id_2", "M", "ruleid", "R110", "bool_var", true, "imbalance", -17L, "errorcode", "null", "errorlevel", "null") )); @@ -795,10 +796,10 @@ public void checkHierarchyValidationMode() throws ScriptException { @Test public void checkHierarchyException() { Dataset DS_2_HR = new InMemoryDataset( - List.of( - List.of("2010", "A", 5L, 5L) + Java8Helpers.listOf( + Java8Helpers.listOf("2010", "A", 5L, 5L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Me_1", Long.class, Dataset.Role.MEASURE), @@ -807,10 +808,10 @@ public void checkHierarchyException() { ); Dataset DS_3_HR = new InMemoryDataset( - List.of( - List.of("2010", "A", "5") + Java8Helpers.listOf( + Java8Helpers.listOf("2010", "A", "5") ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Me_1", String.class, Dataset.Role.MEASURE) @@ -818,10 +819,10 @@ public void checkHierarchyException() { ); Dataset DS_4_HR = new InMemoryDataset( - List.of( - List.of("2010", "A", 5L) + Java8Helpers.listOf( + Java8Helpers.listOf("2010", "A", 5L) ), - List.of( + Java8Helpers.listOf( new Structured.Component("Id_1", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Id_2", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("Me_1", Long.class, Dataset.Role.MEASURE) diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/AggregateTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/AggregateTest.java index d27bc8e66..9123958a9 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/AggregateTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/AggregateTest.java @@ -1,36 +1,32 @@ package fr.insee.vtl.spark.processing.engine; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; -import fr.insee.vtl.model.Structured; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.threeten.extra.Interval; import javax.script.ScriptContext; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; public class AggregateTest { InMemoryDataset dataset = new InMemoryDataset( - List.of( - Map.of("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), - Map.of("name", "Nico", "country", "france", "age", 11L, "weight", 10D), - Map.of("name", "Franck", "country", "france", "age", 12L, "weight", 9D), - Map.of("name", "pengfei", "country", "france", "age", 13L, "weight", 11D) + Java8Helpers.listOf( + Java8Helpers.mapOf("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), + Java8Helpers.mapOf("name", "Nico", "country", "france", "age", 11L, "weight", 10D), + Java8Helpers.mapOf("name", "Franck", "country", "france", "age", 12L, "weight", 9D), + Java8Helpers.mapOf("name", "pengfei", "country", "france", "age", 13L, "weight", 11D) ), - Map.of("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), - Map.of("name", Dataset.Role.IDENTIFIER, "country", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) + Java8Helpers.mapOf("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), + Java8Helpers.mapOf("name", Dataset.Role.IDENTIFIER, "country", Dataset.Role.IDENTIFIER, "age", Dataset.Role.MEASURE, "weight", Dataset.Role.MEASURE) ); private SparkSession spark; private ScriptEngine engine; @@ -61,12 +57,12 @@ void testAggregateGroupAll() throws ScriptException { engine.put("ds1", dataset); engine.eval("res := ds1[aggr test := sum(age) group all length(name)];"); - var actual = ((Dataset) engine.get("res")); + Dataset actual = ((Dataset) engine.get("res")); assertThat(actual.getDataAsMap()).containsExactly( - Map.of("test", 23L, "time", 7L), - Map.of("test", 12L, "time", 6L), - Map.of("test", 11L, "time", 4L) + Java8Helpers.mapOf("test", 23L, "time", 7L), + Java8Helpers.mapOf("test", 12L, "time", 6L), + Java8Helpers.mapOf("test", 11L, "time", 4L) ); } @@ -87,26 +83,26 @@ public void testAggregateClause() throws ScriptException { " group by country];"); assertThat(engine.getContext().getAttribute("res")).isInstanceOf(Dataset.class); assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("country", "france", "sumAge", 72L, "avgWeight", 10.0D, + Java8Helpers.mapOf("country", "france", "sumAge", 72L, "avgWeight", 10.0D, "countVal", 3L, "maxAge", 13L, "maxWeight", 11.0D, "minAge", 11L, "minWeight", 9D, "medianAge", 12L, "medianWeight", 10.0D), - Map.of("country", "norway", "sumAge", 20L, "avgWeight", 11.0, + Java8Helpers.mapOf("country", "norway", "sumAge", 20L, "avgWeight", 11.0, "countVal", 1L, "maxAge", 10L, "maxWeight", 11.0D, "minAge", 10L, "minWeight", 11D, "medianAge", 10L, "medianWeight", 11D) ); // InMemoryDataset dataset2 = new InMemoryDataset( -// List.of( -// Map.of("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), -// Map.of("name", "Nico", "country", "france", "age", 9L, "weight", 5D), -// Map.of("name", "Franck", "country", "france", "age", 10L, "weight", 15D), -// Map.of("name", "Nico1", "country", "france", "age", 11L, "weight", 10D), -// Map.of("name", "Franck1", "country", "france", "age", 12L, "weight", 8D) +// Java8Helpers.listOf( +// Java8Helpers.mapOf("name", "Hadrien", "country", "norway", "age", 10L, "weight", 11D), +// Java8Helpers.mapOf("name", "Nico", "country", "france", "age", 9L, "weight", 5D), +// Java8Helpers.mapOf("name", "Franck", "country", "france", "age", 10L, "weight", 15D), +// Java8Helpers.mapOf("name", "Nico1", "country", "france", "age", 11L, "weight", 10D), +// Java8Helpers.mapOf("name", "Franck1", "country", "france", "age", 12L, "weight", 8D) // ), -// Map.of("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), -// Map.of("name", Role.IDENTIFIER, "country", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) +// Java8Helpers.mapOf("name", String.class, "country", String.class, "age", Long.class, "weight", Double.class), +// Java8Helpers.mapOf("name", Role.IDENTIFIER, "country", Role.IDENTIFIER, "age", Role.MEASURE, "weight", Role.MEASURE) // ); // // context.setAttribute("ds2", dataset2, ScriptContext.ENGINE_SCOPE); diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticAvgTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticAvgTest.java index f46c1249d..7b6130464 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticAvgTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticAvgTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.sql.SparkSession; @@ -20,19 +21,19 @@ public class AnalyticAvgTest { private final InMemoryDataset anCountDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); private static SparkSession spark; @@ -101,14 +102,14 @@ public void testAnAvgWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "avg_Me_1", 3.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "avg_Me_1", 3.5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "avg_Me_1", 4.666666666666667D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "avg_Me_1", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "avg_Me_1", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "avg_Me_1", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "avg_Me_1", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "avg_Me_1", 7.25D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "avg_Me_1", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "avg_Me_1", 3.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "avg_Me_1", 4.666666666666667D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "avg_Me_1", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "avg_Me_1", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "avg_Me_1", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "avg_Me_1", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "avg_Me_1", 7.25D) ); } @@ -154,14 +155,14 @@ public void testAnAvgWithPartitionClause() throws ScriptException { +----+----+----+----+----+--------+--------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 5.0D, "Me_2", 5.75D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 5.0D, "Me_2", 5.75D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.0D, "Me_2", 5.75D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5.0D, "Me_2", 5.75D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 7.25D, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.25D, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 7.25D, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 7.25D, "Me_2", 4.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 5.0D, "Me_2", 5.75D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 5.0D, "Me_2", 5.75D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.0D, "Me_2", 5.75D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5.0D, "Me_2", 5.75D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 7.25D, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.25D, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 7.25D, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 7.25D, "Me_2", 4.0D) ); } @@ -209,14 +210,14 @@ public void testAnAvgWithPartitionOrderByClause() throws ScriptException { List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"), AnalyticTest.DEFAULT_PRECISION); assertThat(res).contains( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3.0D, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.5D, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.67D, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5.0D, "Me_2", 5.75D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9.0D, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.0D, "Me_2", 3.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 8.0D, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 7.25D, "Me_2", 4.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3.0D, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.5D, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.67D, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5.0D, "Me_2", 5.75D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9.0D, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.0D, "Me_2", 3.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 8.0D, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 7.25D, "Me_2", 4.0D) ); } @@ -265,14 +266,14 @@ public void testAnAvgWithPartitionOrderByDPClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"), AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4.67D, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 5.0D, "Me_2", 5.75D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.8D, "Me_2", 5.2D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6.2D, "Me_2", 5.8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 7.4D, "Me_2", 4.4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.0D, "Me_2", 4.8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 7.25D, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6.67D, "Me_2", 4.33D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4.67D, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 5.0D, "Me_2", 5.75D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.8D, "Me_2", 5.2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6.2D, "Me_2", 5.8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 7.4D, "Me_2", 4.4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.0D, "Me_2", 4.8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 7.25D, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6.67D, "Me_2", 4.33D) ); } @@ -324,14 +325,14 @@ public void testAnAvgWithPartitionOrderByRangeClause() throws ScriptException { AnalyticTest.DEFAULT_PRECISION ); assertThat(res).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 5.25D, "Me_2", 4.25D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 6.33D, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 6.17D, "Me_2", 5.83D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 7D, "Me_2", 5.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5.25D, "Me_2", 4.25D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6.33D, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6.17D, "Me_2", 5.83D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 7.0D, "Me_2", 5.5D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 5.25D, "Me_2", 4.25D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 6.33D, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 6.17D, "Me_2", 5.83D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 7D, "Me_2", 5.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5.25D, "Me_2", 4.25D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6.33D, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6.17D, "Me_2", 5.83D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 7.0D, "Me_2", 5.5D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticCountTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticCountTest.java index a089024e8..596e20c4a 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticCountTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticCountTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.sql.SparkSession; @@ -20,19 +21,19 @@ public class AnalyticCountTest { private final InMemoryDataset anCountDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); private static SparkSession spark; @@ -103,14 +104,14 @@ public void testAnCountWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "count_Me_1", 1L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "count_Me_1", 2L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "count_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "count_Me_1", 1L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "count_Me_1", 2L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "count_Me_1", 4L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "count_Me_1", 1L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "count_Me_1", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "count_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "count_Me_1", 1L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "count_Me_1", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "count_Me_1", 4L) ); } @@ -157,14 +158,14 @@ public void testAnCountDPWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "count_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "count_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "count_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "count_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "count_Me_1", 3L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "count_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "count_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "count_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "count_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "count_Me_1", 3L) ); } @@ -211,14 +212,14 @@ public void testAnCountRangeWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "count_Me_1", 2L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "count_Me_1", 2L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "count_Me_1", 2L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "count_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "count_Me_1", 2L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "count_Me_1", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "count_Me_1", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "count_Me_1", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "count_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "count_Me_1", 2L) ); } @@ -267,14 +268,14 @@ public void testAnCountWithPartitionClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 8L, "Me_2", 8L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 8L, "Me_2", 8L) ); } @@ -321,14 +322,14 @@ public void testAnCountWithPartitionOrderByClause() throws ScriptException { +----+----+----+----+----+----------+----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 8L, "Me_2", 8L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 8L, "Me_2", 8L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 8L, "Me_2", 8L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 8L, "Me_2", 8L) ); } @@ -375,14 +376,14 @@ public void testAnCountWithPartitionOrderByDPClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5L, "Me_2", 5L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5L, "Me_2", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 3L, "Me_2", 3L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5L, "Me_2", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5L, "Me_2", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 3L, "Me_2", 3L) ); } @@ -431,14 +432,14 @@ public void testAnCountWithPartitionOrderByRange() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 6L, "Me_2", 6L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6L, "Me_2", 6L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 6L, "Me_2", 6L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 6L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 4L, "Me_2", 4L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 6L, "Me_2", 6L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6L, "Me_2", 6L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 6L, "Me_2", 6L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 6L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 4L, "Me_2", 4L) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticFirstTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticFirstTest.java index 2121653ff..40609aaeb 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticFirstTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticFirstTest.java @@ -1,11 +1,11 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; import javax.script.ScriptContext; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -53,14 +53,14 @@ public void testAnFirstWithPartitionClause() throws ScriptException { +----+----+----+----+----+----------+----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 9L, "Me_2", 3.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 9L, "Me_2", 3.0D) ); } @@ -106,14 +106,14 @@ public void testAnFirstPartitionOrderByDesc() throws ScriptException { +----+----+----+----+----+----------+----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 2L, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 2L, "Me_2", 7.0D) ); } @@ -160,14 +160,14 @@ public void testAnFirstWithPartitionOrderByDPClause() throws ScriptException { +----+----+----+----+----+----------+----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 7L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 5L, "Me_2", 4.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 7L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 5L, "Me_2", 4.0D) ); } @@ -213,14 +213,14 @@ public void testAnFirstPartitionOrderByRangeClause() throws ScriptException { +----+----+----+----+----+----------+----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 7L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 7L, "Me_2", 5.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 7L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 7L, "Me_2", 5.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLagTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLagTest.java index e06811b1b..503676710 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLagTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLagTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; @@ -60,14 +61,14 @@ public void testAnLagWithCalcClause() throws ScriptException { actual.add(replaceNullValues(map, DEFAULT_NULL_STR)); } assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "lag_Me_1", "null"), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "lag_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "lag_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "lag_Me_1", 7L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "lag_Me_1", "null"), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "lag_Me_1", 9L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "lag_Me_1", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "lag_Me_1", 10L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "lag_Me_1", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "lag_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "lag_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "lag_Me_1", 7L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "lag_Me_1", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "lag_Me_1", 9L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "lag_Me_1", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "lag_Me_1", 10L) ); } @@ -122,17 +123,17 @@ public void testAnLag() throws ScriptException { +----+----+----+----+----+---------+---------+ * */ List> actualWithNull = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); - var actual = actualWithNull.stream().map(map -> replaceNullValues(map, DEFAULT_NULL_STR)).collect(Collectors.toList()); + List> actual = actualWithNull.stream().map(map -> replaceNullValues(map, DEFAULT_NULL_STR)).collect(Collectors.toList()); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", "null", "Me_2", "null"), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 7L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", "null", "Me_2", "null"), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 5L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 10L, "Me_2", 2.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", "null", "Me_2", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 7L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", "null", "Me_2", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 5L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 10L, "Me_2", 2.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLastTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLastTest.java index f7b1908f3..014577215 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLastTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLastTest.java @@ -1,11 +1,11 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; import javax.script.ScriptContext; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -57,14 +57,14 @@ public void testAnLastWithPartitionClause() throws ScriptException { +----+----+----+----+----+---------+---------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D) ); } @@ -110,14 +110,14 @@ public void testAnLastPartitionOrderByDesc() throws ScriptException { +----+----+----+----+----+---------+---------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D) ); } @@ -164,14 +164,14 @@ public void testAnLastWithPartitionOrderByDPClause() throws ScriptException { +----+----+----+----+----+---------+---------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 7L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 5L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 10L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 7L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 5L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 10L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D) ); } @@ -217,14 +217,14 @@ public void testAnLastPartitionOrderByRangeClause() throws ScriptException { +----+----+----+----+----+---------+---------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 7L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 5L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 10L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 7L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 5L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 10L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLeadTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLeadTest.java index 5b2fa380f..45f6eca04 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLeadTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticLeadTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; @@ -61,14 +62,14 @@ public void testAnLeadWithCalcClause() throws ScriptException { } assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "lead_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "lead_Me_1", 7L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "lead_Me_1", 6L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "lead_Me_1", "null"), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "lead_Me_1", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "lead_Me_1", 10L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "lead_Me_1", 2L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "lead_Me_1", "null") + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "lead_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "lead_Me_1", 7L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "lead_Me_1", 6L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "lead_Me_1", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "lead_Me_1", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "lead_Me_1", 10L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "lead_Me_1", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "lead_Me_1", "null") ); } @@ -123,18 +124,18 @@ public void testAnLead() throws ScriptException { | A| YY|1996| 2| 7.0| null| null| +----+----+----+----+----+---------+---------+ * */ - var actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().stream() + List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().stream() .map(map -> replaceNullValues(map, DEFAULT_NULL_STR)) .collect(Collectors.toList()); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 7L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", "null", "Me_2", "null"), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 5L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 10L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", "null", "Me_2", "null") + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 7L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 6L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", "null", "Me_2", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 5L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 10L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 2L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", "null", "Me_2", "null") ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMaxTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMaxTest.java index 96e5ec70b..08e2426fa 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMaxTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMaxTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.sql.SparkSession; @@ -20,19 +21,19 @@ public class AnalyticMaxTest { private final InMemoryDataset anCountDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); private static SparkSession spark; @@ -101,14 +102,14 @@ public void testAnMaxWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "max_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "max_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "max_Me_1", 7L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "max_Me_1", 7L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "max_Me_1", 9L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "max_Me_1", 9L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "max_Me_1", 10L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "max_Me_1", 10L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "max_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "max_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "max_Me_1", 7L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "max_Me_1", 7L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "max_Me_1", 9L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "max_Me_1", 9L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "max_Me_1", 10L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "max_Me_1", 10L) ); } @@ -154,14 +155,14 @@ public void testAnMaxWithPartitionClause() throws ScriptException { +----+----+----+----+----+--------+--------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 10L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 10L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 10L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 10L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 7.0D) ); } @@ -207,14 +208,14 @@ public void testAnMaxWithPartitionOrderByClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 9L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 9L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 7.0D) ); } @@ -262,14 +263,14 @@ public void testAnMaxWithPartitionOrderByDPClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 9L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 9L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 10L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 10L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 9L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 9L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 10L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 10L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 7.0D) ); } @@ -317,14 +318,14 @@ public void testAnMaxWithPartitionOrderByRangeClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 9L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 10L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 10L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 10L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 9.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 10L, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 8.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 9L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 10L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 10L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 10L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 9.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 10L, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 10L, "Me_2", 8.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMedianTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMedianTest.java index 61fbb152a..85ce42d0a 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMedianTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMedianTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.sql.SparkSession; @@ -20,19 +21,19 @@ public class AnalyticMedianTest { private final InMemoryDataset anCountDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); private static SparkSession spark; @@ -101,14 +102,14 @@ public void testAnMedianWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "median_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "median_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "median_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "median_Me_1", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "median_Me_1", 9L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "median_Me_1", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "median_Me_1", 9L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "median_Me_1", 5L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "median_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "median_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "median_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "median_Me_1", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "median_Me_1", 9L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "median_Me_1", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "median_Me_1", 9L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "median_Me_1", 5L) ); } @@ -155,14 +156,14 @@ public void testAnMedianWithPartitionClause() throws ScriptException { +----+----+----+----+----+-----------+-----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 3.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 3.0D) ); } @@ -208,14 +209,14 @@ public void testAnMedianWithPartitionOrderByClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 3.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 3.0D) ); } @@ -263,14 +264,14 @@ public void testAnMedianWithPartitionOrderByDPClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 6L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 7L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6L, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 4.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 6L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 7L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6L, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 4.0D) ); } @@ -318,14 +319,14 @@ public void testAnMedianWithPartitionOrderByRangeClause() throws ScriptException * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 4L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 5.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6L, "Me_2", 5.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 4L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 5.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6L, "Me_2", 5.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMinTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMinTest.java index b95db2eee..a021c65d6 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMinTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticMinTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.sql.SparkSession; @@ -20,19 +21,19 @@ public class AnalyticMinTest { private final InMemoryDataset anCountDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); private static SparkSession spark; @@ -101,14 +102,14 @@ public void testAnMinWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "min_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "min_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "min_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "min_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "min_Me_1", 9L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "min_Me_1", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "min_Me_1", 5L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "min_Me_1", 5L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "min_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "min_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "min_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "min_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "min_Me_1", 9L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "min_Me_1", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "min_Me_1", 5L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "min_Me_1", 5L) ); } @@ -154,14 +155,14 @@ public void testAnMinWithPartitionClause() throws ScriptException { +----+----+----+----+----+--------+--------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) ); } @@ -207,14 +208,14 @@ public void testAnMinWithPartitionOrderByClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) ); } @@ -261,14 +262,14 @@ public void testAnMinWithOrderByClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 3L, "Me_2", 1.0D) ); } @@ -315,14 +316,14 @@ public void testAnMinWithPartitionOrderByDPClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 3.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4L, "Me_2", 3.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) ); } @@ -370,14 +371,14 @@ public void testAnMinWithPartitionOrderByRangeClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 2.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRankTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRankTest.java index 793dc35f0..364c1785b 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRankTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRankTest.java @@ -1,11 +1,11 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; import javax.script.ScriptContext; import javax.script.ScriptException; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -59,14 +59,14 @@ public void testAnRankAscClause() throws ScriptException { +----+----+----+----+----+--------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "rank_col", 1L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "rank_col", 2L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "rank_col", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "rank_col", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "rank_col", 1L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "rank_col", 2L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "rank_col", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "rank_col", 4L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "rank_col", 1L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "rank_col", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "rank_col", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "rank_col", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "rank_col", 1L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "rank_col", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "rank_col", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "rank_col", 4L) ); } @@ -113,14 +113,14 @@ public void testAnRankDesc() throws ScriptException { +----+----+----+----+----+--------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "rank_col", 1L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "rank_col", 2L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "rank_col", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "rank_col", 4L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "rank_col", 1L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "rank_col", 2L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "rank_col", 3L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "rank_col", 4L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8.0D, "rank_col", 1L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5.0D, "rank_col", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9.0D, "rank_col", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1.0D, "rank_col", 4L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7.0D, "rank_col", 1L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2.0D, "rank_col", 2L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4.0D, "rank_col", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3.0D, "rank_col", 4L) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRatioToReportTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRatioToReportTest.java index 21317bdab..b1de6563f 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRatioToReportTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticRatioToReportTest.java @@ -1,13 +1,12 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.junit.jupiter.api.Test; import javax.script.ScriptContext; import javax.script.ScriptException; -import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -17,19 +16,19 @@ public class AnalyticRatioToReportTest extends AnalyticTest { public void testAnRatioToReportWithCalcClause() throws ScriptException { InMemoryDataset anDS = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 12L, "Me_2", 0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 14L, "Me_2", -3D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 12L, "Me_2", 0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 14L, "Me_2", -3D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); /* Input dataset +----+----+----+----+----+ @@ -67,14 +66,14 @@ public void testAnRatioToReportWithCalcClause() throws ScriptException { +----+----+----+----+----+----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "ratio_Me_1", 0.15D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 3.0D, "ratio_Me_1", 0.2D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "ratio_Me_1", 0.35D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 1.0D, "ratio_Me_1", 0.3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 12L, "Me_2", 0.0D, "ratio_Me_1", 0.3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8.0D, "ratio_Me_1", 0.2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 5.0D, "ratio_Me_1", 0.15D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 14L, "Me_2", -3.0D, "ratio_Me_1", 0.35D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "ratio_Me_1", 0.15D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 3.0D, "ratio_Me_1", 0.2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "ratio_Me_1", 0.35D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 1.0D, "ratio_Me_1", 0.3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 12L, "Me_2", 0.0D, "ratio_Me_1", 0.3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8.0D, "ratio_Me_1", 0.2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 5.0D, "ratio_Me_1", 0.15D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 14L, "Me_2", -3.0D, "ratio_Me_1", 0.35D) ); } @@ -89,19 +88,19 @@ public void testAnRatioToReportWithCalcClause() throws ScriptException { public void testAnRatioToReport() throws ScriptException { InMemoryDataset anDS = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 12L, "Me_2", 0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 14L, "Me_2", -3D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 12L, "Me_2", 0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 14L, "Me_2", -3D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); /* Input dataset +----+----+----+----+----+ @@ -140,14 +139,14 @@ public void testAnRatioToReport() throws ScriptException { +----+----+----+----+----+----------+----------+----------+----------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.15D, "Me_2", 0.1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.2D, "Me_2", 0.3D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 0.35D, "Me_2", 0.5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 0.3D, "Me_2", 0.1D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 0.3D, "Me_2", 0.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 0.2D, "Me_2", 0.8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 0.15D, "Me_2", 0.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 0.35D, "Me_2", -0.3D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.15D, "Me_2", 0.1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.2D, "Me_2", 0.3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 0.35D, "Me_2", 0.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 0.3D, "Me_2", 0.1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 0.3D, "Me_2", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 0.2D, "Me_2", 0.8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 0.15D, "Me_2", 0.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 0.35D, "Me_2", -0.3D) ); diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdPopTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdPopTest.java index a15a6f139..fbb8c346d 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdPopTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdPopTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; @@ -54,14 +55,14 @@ public void testAnStdPopWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "stddev_pop_Me_1", 0.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "stddev_pop_Me_1", 0.5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "stddev_pop_Me_1", 1.699673171197595D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "stddev_pop_Me_1", 1.5811388300841895D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "stddev_pop_Me_1", 0.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "stddev_pop_Me_1", 2.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "stddev_pop_Me_1", 2.160246899469287D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "stddev_pop_Me_1", 2.277608394786075D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "stddev_pop_Me_1", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "stddev_pop_Me_1", 0.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "stddev_pop_Me_1", 1.699673171197595D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "stddev_pop_Me_1", 1.5811388300841895D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "stddev_pop_Me_1", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "stddev_pop_Me_1", 2.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "stddev_pop_Me_1", 2.160246899469287D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "stddev_pop_Me_1", 2.277608394786075D) ); } @@ -109,14 +110,14 @@ public void testAnStdPopWithPartitionClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"),AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.58D, "Me_2", 3.11D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.58D, "Me_2", 3.11D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.58D, "Me_2", 3.11D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.58D, "Me_2", 3.11D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.28D, "Me_2", 1.87D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.28D, "Me_2", 1.87D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.28D, "Me_2", 1.87D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.28D, "Me_2", 1.87D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.58D, "Me_2", 3.11D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.58D, "Me_2", 3.11D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.58D, "Me_2", 3.11D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.58D, "Me_2", 3.11D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.28D, "Me_2", 1.87D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.28D, "Me_2", 1.87D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.28D, "Me_2", 1.87D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.28D, "Me_2", 1.87D) ); } @@ -163,14 +164,14 @@ public void testAnStdPopWithPartitionOrderByClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"),AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.5D, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.70D, "Me_2", 3.27D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.58D, "Me_2", 3.11D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.0D, "Me_2", 0.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.16D, "Me_2", 0.82D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.28D, "Me_2", 1.87D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.5D, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.70D, "Me_2", 3.27D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.58D, "Me_2", 3.11D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.0D, "Me_2", 0.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.16D, "Me_2", 0.82D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.28D, "Me_2", 1.87D) ); } @@ -219,14 +220,14 @@ public void testAnStdPopWithPartitionOrderByDPClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"),AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.70D, "Me_2", 3.27D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.58D, "Me_2", 3.11D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.14D, "Me_2", 2.99D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.72D, "Me_2", 2.32D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 1.85D, "Me_2", 2.06D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.1D, "Me_2", 2.32D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.28D, "Me_2", 1.87D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.36D, "Me_2", 2.05D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.70D, "Me_2", 3.27D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.58D, "Me_2", 3.11D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.14D, "Me_2", 2.99D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.72D, "Me_2", 2.32D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 1.85D, "Me_2", 2.06D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.1D, "Me_2", 2.32D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.28D, "Me_2", 1.87D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.36D, "Me_2", 2.05D) ); } @@ -278,14 +279,14 @@ public void testAnStdPopWithPartitionOrderByRangeClause() throws ScriptException AnalyticTest.DEFAULT_PRECISION ); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.28D, "Me_2", 2.95D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.28D, "Me_2", 2.95D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.56D, "Me_2", 2.58D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.56D, "Me_2", 2.58D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.95D, "Me_2", 2.41D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 1.95D, "Me_2", 2.41D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.87D, "Me_2", 2.29D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 1.87D, "Me_2", 2.29D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.28D, "Me_2", 2.95D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.28D, "Me_2", 2.95D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.56D, "Me_2", 2.58D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.56D, "Me_2", 2.58D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.95D, "Me_2", 2.41D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 1.95D, "Me_2", 2.41D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.87D, "Me_2", 2.29D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 1.87D, "Me_2", 2.29D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdSampTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdSampTest.java index f3181ca95..32756833f 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdSampTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticStdSampTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; @@ -63,14 +64,14 @@ public void testAnStdSampWithCalcClause() throws ScriptException { assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "stddev_samp_Me_1", "null"), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "stddev_samp_Me_1", 0.7071067811865476D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "stddev_samp_Me_1", 2.0816659994661326D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "stddev_samp_Me_1", 1.8257418583505536D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "stddev_samp_Me_1", "null"), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "stddev_samp_Me_1", 2.8284271247461903D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "stddev_samp_Me_1", 2.6457513110645907D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "stddev_samp_Me_1", 2.6299556396765835D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "stddev_samp_Me_1", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "stddev_samp_Me_1", 0.7071067811865476D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "stddev_samp_Me_1", 2.0816659994661326D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "stddev_samp_Me_1", 1.8257418583505536D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "stddev_samp_Me_1", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "stddev_samp_Me_1", 2.8284271247461903D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "stddev_samp_Me_1", 2.6457513110645907D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "stddev_samp_Me_1", 2.6299556396765835D) ); } @@ -118,14 +119,14 @@ public void testAnStdSampWithPartitionClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"), AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.83D, "Me_2", 3.59D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.83D, "Me_2", 3.59D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.83D, "Me_2", 3.59D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.83D, "Me_2", 3.59D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.63D, "Me_2", 2.16D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.63D, "Me_2", 2.16D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.63D, "Me_2", 2.16D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.63D, "Me_2", 2.16D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 1.83D, "Me_2", 3.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.83D, "Me_2", 3.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 1.83D, "Me_2", 3.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.83D, "Me_2", 3.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.63D, "Me_2", 2.16D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.63D, "Me_2", 2.16D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.63D, "Me_2", 2.16D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.63D, "Me_2", 2.16D) ); } @@ -172,18 +173,18 @@ public void testAnStdSampWithPartitionOrderByClause() throws ScriptException { * */ - var actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().stream() + List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().stream() .map(map -> replaceNullValues(map, DEFAULT_NULL_STR)) .collect(Collectors.toList()); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", "null", "Me_2", "null"), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.71D, "Me_2", 5.66D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.08D, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.83D, "Me_2", 3.59D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", "null", "Me_2", "null"), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.83D, "Me_2", 0.71D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.65D, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.63D, "Me_2", 2.16D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", "null", "Me_2", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.71D, "Me_2", 5.66D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.08D, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.83D, "Me_2", 3.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", "null", "Me_2", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.83D, "Me_2", 0.71D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.65D, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.63D, "Me_2", 2.16D) ); } @@ -235,14 +236,14 @@ public void testAnStdSampWithPartitionOrderByDPClause() throws ScriptException { List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"), AnalyticTest.DEFAULT_PRECISION); assertThat(res).contains( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.08D, "Me_2", 4.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.83D, "Me_2", 3.59D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.39D, "Me_2", 3.35D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.92D, "Me_2", 2.59D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.07D, "Me_2", 2.30D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.35D, "Me_2", 2.59D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.63D, "Me_2", 2.16D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.89D, "Me_2", 2.52D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.08D, "Me_2", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 1.83D, "Me_2", 3.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.39D, "Me_2", 3.35D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 1.92D, "Me_2", 2.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.07D, "Me_2", 2.30D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.35D, "Me_2", 2.59D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.63D, "Me_2", 2.16D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.89D, "Me_2", 2.52D) ); } @@ -294,14 +295,14 @@ public void testAnStdSampWithPartitionOrderByRangeClause() throws ScriptExceptio AnalyticTest.DEFAULT_PRECISION ); assertThat(res).contains( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.63D, "Me_2", 3.40D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.63D, "Me_2", 3.40D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.80D, "Me_2", 2.83D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.80D, "Me_2", 2.83D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.14D, "Me_2", 2.64D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.14D, "Me_2", 2.64D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.16D, "Me_2", 2.65D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.16D, "Me_2", 2.65D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.63D, "Me_2", 3.40D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 2.63D, "Me_2", 3.40D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.80D, "Me_2", 2.83D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 2.80D, "Me_2", 2.83D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.14D, "Me_2", 2.64D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 2.14D, "Me_2", 2.64D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.16D, "Me_2", 2.65D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 2.16D, "Me_2", 2.65D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticSumTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticSumTest.java index 48ea154af..3c7453e97 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticSumTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticSumTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.sql.SparkSession; @@ -20,19 +21,19 @@ public class AnalyticSumTest { private final InMemoryDataset anCountDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); private static SparkSession spark; @@ -101,14 +102,14 @@ public void testAnSumWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "sum_Me_1", 3L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "sum_Me_1", 7L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "sum_Me_1", 14L), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "sum_Me_1", 20L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "sum_Me_1", 9L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "sum_Me_1", 14L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "sum_Me_1", 24L), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "sum_Me_1", 29L) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "sum_Me_1", 3L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "sum_Me_1", 7L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "sum_Me_1", 14L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "sum_Me_1", 20L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "sum_Me_1", 9L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "sum_Me_1", 14L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "sum_Me_1", 24L), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "sum_Me_1", 29L) ); } @@ -155,14 +156,14 @@ public void testAnSumWithPartitionClause() throws ScriptException { +----+----+----+----+----+--------+--------+ * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 29L, "Me_2", 16.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 29L, "Me_2", 16.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 29L, "Me_2", 16.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 29L, "Me_2", 16.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 29L, "Me_2", 16.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 29L, "Me_2", 16.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 29L, "Me_2", 16.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 29L, "Me_2", 16.0D) ); } @@ -210,14 +211,14 @@ public void testAnSumWithPartitionOrderByClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 49L, "Me_2", 39.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 49L, "Me_2", 39.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 49L, "Me_2", 39.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 49L, "Me_2", 39.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 49L, "Me_2", 39.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 49L, "Me_2", 39.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 49L, "Me_2", 39.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 49L, "Me_2", 39.0D) ); } @@ -265,14 +266,14 @@ public void testAnSumWithOrderByClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7L, "Me_2", 10.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 14L, "Me_2", 15.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 29L, "Me_2", 26.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 34L, "Me_2", 30.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 44L, "Me_2", 32.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 49L, "Me_2", 39.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7L, "Me_2", 10.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 14L, "Me_2", 15.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 29L, "Me_2", 26.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 34L, "Me_2", 30.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 44L, "Me_2", 32.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 49L, "Me_2", 39.0D) ); } @@ -320,14 +321,14 @@ public void testAnSumWithPartitionOrderByDPClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 14L, "Me_2", 15.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 20L, "Me_2", 23.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 29L, "Me_2", 26.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 31L, "Me_2", 29.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 37L, "Me_2", 22.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 35L, "Me_2", 24.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 29L, "Me_2", 16.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 20L, "Me_2", 13.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 14L, "Me_2", 15.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 20L, "Me_2", 23.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 29L, "Me_2", 26.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 31L, "Me_2", 29.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 37L, "Me_2", 22.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 35L, "Me_2", 24.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 29L, "Me_2", 16.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 20L, "Me_2", 13.0D) ); } @@ -375,14 +376,14 @@ public void testAnSumWithPartitionOrderByRangeClause() throws ScriptException { * */ assertThat(((Dataset) engine.getContext().getAttribute("res")).getDataAsMap()).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 21L, "Me_2", 17.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 21L, "Me_2", 17.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 38L, "Me_2", 24.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 38L, "Me_2", 24.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 37L, "Me_2", 35.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 37L, "Me_2", 35.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 28L, "Me_2", 22.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 28L, "Me_2", 22.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 21L, "Me_2", 17.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 21L, "Me_2", 17.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 38L, "Me_2", 24.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 38L, "Me_2", 24.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 37L, "Me_2", 35.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 37L, "Me_2", 35.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 28L, "Me_2", 22.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 28L, "Me_2", 22.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticTest.java index f5f9759b9..510c8c0d3 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.Dataset.Role; import fr.insee.vtl.model.InMemoryDataset; @@ -16,38 +17,39 @@ import java.util.Map; import java.util.stream.Collectors; + public abstract class AnalyticTest { public final InMemoryDataset ds1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Role.IDENTIFIER, "Id_2", Role.IDENTIFIER, "Year", Role.IDENTIFIER, "Me_1", Role.MEASURE, "Me_2", Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Role.IDENTIFIER, "Id_2", Role.IDENTIFIER, "Year", Role.IDENTIFIER, "Me_1", Role.MEASURE, "Me_2", Role.MEASURE) ); public final InMemoryDataset ds2 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1993L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1994L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1995L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 1996L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1993L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1994L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1995L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 1996L, "Me_1", 2L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); public final static int DEFAULT_PRECISION = 2; diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarPopTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarPopTest.java index 30feff601..c519d2180 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarPopTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarPopTest.java @@ -1,6 +1,7 @@ package fr.insee.vtl.spark.processing.engine.analytic; import fr.insee.vtl.engine.VtlScriptEngine; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import org.apache.spark.sql.SparkSession; @@ -20,19 +21,19 @@ public class AnalyticVarPopTest { private final InMemoryDataset anCountDS1 = new InMemoryDataset( - List.of( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) + Java8Helpers.listOf( + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7D) ), - Map.of("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), - Map.of("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) + Java8Helpers.mapOf("Id_1", String.class, "Id_2", String.class, "Year", Long.class, "Me_1", Long.class, "Me_2", Double.class), + Java8Helpers.mapOf("Id_1", Dataset.Role.IDENTIFIER, "Id_2", Dataset.Role.IDENTIFIER, "Year", Dataset.Role.IDENTIFIER, "Me_1", Dataset.Role.MEASURE, "Me_2", Dataset.Role.MEASURE) ); private static SparkSession spark; @@ -101,14 +102,14 @@ public void testAnVarPopWithCalcClause() throws ScriptException { List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap(); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "var_pop_Me_1", 0.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "var_pop_Me_1", 0.25D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "var_pop_Me_1", 2.888888888888889D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "var_pop_Me_1", 2.4999999999999996D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "var_pop_Me_1", 0.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "var_pop_Me_1", 4.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "var_pop_Me_1", 4.666666666666667D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "var_pop_Me_1", 5.1875D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3L, "Me_2", 1.0D, "var_pop_Me_1", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 4L, "Me_2", 9.0D, "var_pop_Me_1", 0.25D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 7L, "Me_2", 5.0D, "var_pop_Me_1", 2.888888888888889D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 6L, "Me_2", 8.0D, "var_pop_Me_1", 2.4999999999999996D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 9L, "Me_2", 3.0D, "var_pop_Me_1", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5L, "Me_2", 4.0D, "var_pop_Me_1", 4.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 10L, "Me_2", 2.0D, "var_pop_Me_1", 4.666666666666667D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5L, "Me_2", 7.0D, "var_pop_Me_1", 5.1875D) ); } @@ -160,14 +161,14 @@ public void testAnVarPopWithPartitionClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"), AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.50D, "Me_2", 9.69D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.50D, "Me_2", 9.69D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.50D, "Me_2", 9.69D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.50D, "Me_2", 9.69D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5.19D, "Me_2", 3.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5.19D, "Me_2", 3.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5.19D, "Me_2", 3.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5.19D, "Me_2", 3.5D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.50D, "Me_2", 9.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.50D, "Me_2", 9.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.50D, "Me_2", 9.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.50D, "Me_2", 9.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5.19D, "Me_2", 3.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5.19D, "Me_2", 3.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5.19D, "Me_2", 3.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5.19D, "Me_2", 3.5D) ); } @@ -217,14 +218,14 @@ public void testAnVarPopWithPartitionOrderByClause() throws ScriptException { assertThat(res).contains( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.25D, "Me_2", 16.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.89D, "Me_2", 10.67D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.50D, "Me_2", 9.69D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 4.0D, "Me_2", 0.25D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4.67D, "Me_2", 0.67D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5.19D, "Me_2", 3.5D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.25D, "Me_2", 16.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 2.89D, "Me_2", 10.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.50D, "Me_2", 9.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 0.0D, "Me_2", 0.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 4.0D, "Me_2", 0.25D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4.67D, "Me_2", 0.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5.19D, "Me_2", 3.5D) ); } @@ -274,14 +275,14 @@ public void testAnVarPopWithPartitionOrderByDPClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"), AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.89D, "Me_2", 10.67D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.5D, "Me_2", 9.69D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.56D, "Me_2", 8.96D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.96D, "Me_2", 5.36D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 3.44D, "Me_2", 4.24D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 4.4D, "Me_2", 5.36D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5.19D, "Me_2", 3.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5.56D, "Me_2", 4.22D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 2.89D, "Me_2", 10.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 2.5D, "Me_2", 9.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.56D, "Me_2", 8.96D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 2.96D, "Me_2", 5.36D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 3.44D, "Me_2", 4.24D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 4.4D, "Me_2", 5.36D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 5.19D, "Me_2", 3.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 5.56D, "Me_2", 4.22D) ); } @@ -333,14 +334,14 @@ public void testAnVarPopWithPartitionOrderByRangeClause() throws ScriptException AnalyticTest.DEFAULT_PRECISION ); assertThat(res).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 5.19D, "Me_2", 8.69D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5.19D, "Me_2", 8.69D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 6.56D, "Me_2", 6.67D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6.56D, "Me_2", 6.67D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3.81D, "Me_2", 5.81D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 3.81D, "Me_2", 5.81D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.5D, "Me_2", 5.25D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 3.5D, "Me_2", 5.25D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 5.19D, "Me_2", 8.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 5.19D, "Me_2", 8.69D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 6.56D, "Me_2", 6.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6.56D, "Me_2", 6.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3.81D, "Me_2", 5.81D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 3.81D, "Me_2", 5.81D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.5D, "Me_2", 5.25D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 3.5D, "Me_2", 5.25D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarSampTest.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarSampTest.java index bbf675c21..7cdaceda8 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarSampTest.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/processing/engine/analytic/AnalyticVarSampTest.java @@ -1,5 +1,6 @@ package fr.insee.vtl.spark.processing.engine.analytic; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import org.junit.jupiter.api.Test; @@ -56,14 +57,14 @@ public void testAnVarSampWithPartitionClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"),AnalyticTest.DEFAULT_PRECISION); assertThat(res).contains( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3.33D, "Me_2", 12.92D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.33D, "Me_2", 12.92D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3.33D, "Me_2", 12.92D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.33D, "Me_2", 12.92D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 6.92D, "Me_2", 4.67D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6.92D, "Me_2", 4.67D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6.92D, "Me_2", 4.67D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6.92D, "Me_2", 4.67D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 3.33D, "Me_2", 12.92D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.33D, "Me_2", 12.92D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 3.33D, "Me_2", 12.92D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.33D, "Me_2", 12.92D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 6.92D, "Me_2", 4.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 6.92D, "Me_2", 4.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6.92D, "Me_2", 4.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6.92D, "Me_2", 4.67D) ); } @@ -109,18 +110,18 @@ public void testAnVarSampWithPartitionOrderByClause() throws ScriptException { +----+----+----+----+----+------------------+------------------+ * */ - var actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().stream() + List> actual = ((Dataset) engine.getContext().getAttribute("res")).getDataAsMap().stream() .map(map -> replaceNullValues(map, DEFAULT_NULL_STR)) .collect(Collectors.toList()); assertThat(actual).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", "null", "Me_2", "null"), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.5D, "Me_2", 32.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.33D, "Me_2", 16.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.33D, "Me_2", 12.92D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", "null", "Me_2", "null"), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8.0D, "Me_2", 0.5D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 7.0D, "Me_2", 1.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6.92D, "Me_2", 4.67D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", "null", "Me_2", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 0.5D, "Me_2", 32.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.33D, "Me_2", 16.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.33D, "Me_2", 12.92D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", "null", "Me_2", "null"), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 8.0D, "Me_2", 0.5D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 7.0D, "Me_2", 1.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 6.92D, "Me_2", 4.67D) ); } @@ -171,14 +172,14 @@ public void testAnVarSampWithPartitionOrderByDPClause() throws ScriptException { * */ List> res = AnalyticTest.roundDecimalInDataset((Dataset) engine.getContext().getAttribute("res"),AnalyticTest.DEFAULT_PRECISION); assertThat(res).containsExactly( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4.33D, "Me_2", 16.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.33D, "Me_2", 12.92D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.70D, "Me_2", 11.2D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.7D, "Me_2", 6.7D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 4.3D, "Me_2", 5.3D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5.5D, "Me_2", 6.7D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6.92D, "Me_2", 4.67D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 8.33D, "Me_2", 6.33D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 4.33D, "Me_2", 16.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 3.33D, "Me_2", 12.92D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 5.70D, "Me_2", 11.2D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 3.7D, "Me_2", 6.7D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 4.3D, "Me_2", 5.3D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 5.5D, "Me_2", 6.7D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 6.92D, "Me_2", 4.67D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 8.33D, "Me_2", 6.33D) ); } @@ -230,14 +231,14 @@ public void testAnVarSampWithPartitionOrderByRangeClause() throws ScriptExceptio AnalyticTest.DEFAULT_PRECISION ); assertThat(res).containsExactlyInAnyOrder( - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 6.92D, "Me_2", 11.58D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 6.92D, "Me_2", 11.58D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7.87D, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.87D, "Me_2", 8.0D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.57D, "Me_2", 6.97D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4.57D, "Me_2", 6.97D), - Map.of("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4.67D, "Me_2", 7.0D), - Map.of("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 4.67D, "Me_2", 7.0D) + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2000L, "Me_1", 6.92D, "Me_2", 11.58D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2000L, "Me_1", 6.92D, "Me_2", 11.58D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2001L, "Me_1", 7.87D, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2001L, "Me_1", 7.87D, "Me_2", 8.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2002L, "Me_1", 4.57D, "Me_2", 6.97D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2002L, "Me_1", 4.57D, "Me_2", 6.97D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "XX", "Year", 2003L, "Me_1", 4.67D, "Me_2", 7.0D), + Java8Helpers.mapOf("Id_1", "A", "Id_2", "YY", "Year", 2003L, "Me_1", 4.67D, "Me_2", 7.0D) ); } diff --git a/vtl-spark/src/test/java/fr/insee/vtl/spark/samples/DatasetSamples.java b/vtl-spark/src/test/java/fr/insee/vtl/spark/samples/DatasetSamples.java index 3b08d3fc8..ea8eea146 100644 --- a/vtl-spark/src/test/java/fr/insee/vtl/spark/samples/DatasetSamples.java +++ b/vtl-spark/src/test/java/fr/insee/vtl/spark/samples/DatasetSamples.java @@ -1,16 +1,16 @@ package fr.insee.vtl.spark.samples; +import fr.insee.vtl.model.utils.Java8Helpers; import fr.insee.vtl.model.Dataset; import fr.insee.vtl.model.InMemoryDataset; import fr.insee.vtl.model.Structured; import java.util.Arrays; -import java.util.List; public class DatasetSamples { public static InMemoryDataset ds1 = new InMemoryDataset( - List.of( + Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("long1", Long.class, Dataset.Role.MEASURE), new Structured.Component("long2", Long.class, Dataset.Role.MEASURE), @@ -28,7 +28,7 @@ public class DatasetSamples { ); public static InMemoryDataset ds2 = new InMemoryDataset( - List.of( + Java8Helpers.listOf( new Structured.Component("id", String.class, Dataset.Role.IDENTIFIER), new Structured.Component("long1", Long.class, Dataset.Role.MEASURE), new Structured.Component("double1", Double.class, Dataset.Role.MEASURE),