From a6e595861c2e0e1731bea3e2ecff655a708d82c5 Mon Sep 17 00:00:00 2001 From: Philipp Zehnder Date: Thu, 4 Jul 2024 22:57:11 +0200 Subject: [PATCH] refactor(#2964): Fix unix timestamp conversion for integer values (#2996) * refactor(#2964): Fix unix timestamp conversion for integer values * refactor(#2964): Fix linting --- .../connect/shared/DatatypeUtils.java | 20 ++- .../transform/DatatypeUtilsTest.java | 148 ++++++++++++++---- .../tests/adapter/fileStream.smoke.spec.ts | 41 ++--- 3 files changed, 151 insertions(+), 58 deletions(-) diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java index cdfaea25ce..8c95afb2f5 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java @@ -28,6 +28,19 @@ public class DatatypeUtils { private static final Logger LOG = LoggerFactory.getLogger(DatatypeUtils.class); + /** + * Converts the given value to a specified XSD datatype. + * This method attempts to convert the input value to the target datatype specified by the XSD string. + * It supports conversion to string, double, float, boolean, integer, and long types. + * If the conversion is not possible due to a format mismatch, the original value is returned. + * A number format exception during conversion is logged as an error. + * + * @param value The value to be converted. It can be of any type. + * @param targetDatatypeXsd The target XSD datatype as a string. Supported types are XSD.STRING, + * XSD.DOUBLE, XSD.FLOAT, XSD.BOOLEAN, XSD.INTEGER, and XSD.LONG. + * @return The converted value as an Object. If conversion fails, the original value is returned. + * @throws NumberFormatException if the string does not contain a parsable number for numeric conversions. + */ public static Object convertValue(Object value, String targetDatatypeXsd) { var stringValue = String.valueOf(value); @@ -42,7 +55,7 @@ public static Object convertValue(Object value, } else if (XSD.BOOLEAN.toString().equals(targetDatatypeXsd)) { return Boolean.parseBoolean(stringValue); } else if (XSD.INTEGER.toString().equals(targetDatatypeXsd)) { - var floatingNumber = Float.parseFloat(stringValue); + var floatingNumber = Double.parseDouble(stringValue); return Integer.parseInt(String.valueOf(Math.round(floatingNumber))); } else if (XSD.LONG.toString().equals(targetDatatypeXsd)) { var floatingNumber = Double.parseDouble(stringValue); @@ -57,11 +70,6 @@ public static Object convertValue(Object value, return value; } - public static String getCanonicalTypeClassName(String value, - boolean preferFloat) { - return getTypeClass(value, preferFloat).getCanonicalName(); - } - public static String getXsdDatatype(String value, boolean preferFloat) { var clazz = getTypeClass(value, preferFloat); diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java index ef1235b4b8..7937d6875c 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java @@ -21,110 +21,192 @@ import org.apache.streampipes.connect.shared.DatatypeUtils; import org.apache.streampipes.vocabulary.XSD; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Locale; +import static org.junit.jupiter.api.Assertions.assertEquals; + public class DatatypeUtilsTest { - @Test /** - * This test ensures that timestamps represented as strings are correctly parsed. + * The following tests ensure that timestamps represented as strings are correctly parsed. * Often they are first parsed into floating point number before transformed back to long. * The data type for those values should be Double and not Float, because the transformation to Float might change * the value */ - public void convertTimestampValue() { - var inputValue = "1667904471000"; + @Test + public void convertValue_StringToStringValue() { + var inputValue = "testString"; + var actualValue = DatatypeUtils.convertValue(inputValue, XSD.STRING.toString()); + + assertEquals(inputValue, actualValue); + } + + @Test + public void convertValue_StringToDoubleValue() { + var actualValue = DatatypeUtils.convertValue("1667904471000", XSD.DOUBLE.toString()); + + assertEquals(1.667904471E12, actualValue); + } + + @Test + public void convertValue_StringToFloatValue() { + var actualValue = DatatypeUtils.convertValue("123.45", XSD.FLOAT.toString()); + + assertEquals(123.45f, actualValue); + } + + @Test + public void convertValue_StringToInteger() { + var actualValue = DatatypeUtils.convertValue("1623871500", XSD.INTEGER.toString()); + + assertEquals(1623871500, actualValue); + } + + @Test + public void convertValue_StringToIntegerValue() { + var actualValue = DatatypeUtils.convertValue("123", XSD.INTEGER.toString()); + + assertEquals(123, actualValue); + } + + @Test + public void convertValue_StringToLongValue() { + var actualValue = DatatypeUtils.convertValue("1623871500000", XSD.LONG.toString()); + + assertEquals(1623871500000L, actualValue); + } + + @Test + public void convertValue_StringToBooleanTrueValue() { + var actualValue = DatatypeUtils.convertValue("true", XSD.BOOLEAN.toString()); - var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString()); - var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString()); + assertEquals(true, actualValue); + } + + @Test + public void convertValue_StringToBooleanFalseValue() { + var actualValue = DatatypeUtils.convertValue("false", XSD.BOOLEAN.toString()); + + assertEquals(false, actualValue); + } + + @Test + public void convertValue_FloatToIntegerValue_Rounding() { + var actualValue = DatatypeUtils.convertValue(123.45f, XSD.INTEGER.toString()); - Assertions.assertEquals(Long.parseLong(inputValue), longValue); + assertEquals(123, actualValue); } + @Test + public void convertValue_DoubleToLongValue_Rounding1() { + var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString()); + + assertEquals(1234567890L, actualValue); + } + + @Test + public void convertValue_DoubleToLongValue() { + var actualValue = DatatypeUtils.convertValue(1.667904471E12, XSD.LONG.toString()); + + assertEquals(1667904471000L, actualValue); + } + + @Test + public void convertValue_DoubleToLongValue_Rounding() { + var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString()); + + assertEquals(1234567890L, actualValue); + } + + String booleanInputValue = "true"; + @Test - public void getTypeClassNoPrefereFloatingPointBoolean() { + public void getTypeClass_NoPrefereFloatingPointBoolean() { var result = DatatypeUtils.getTypeClass(booleanInputValue, false); - Assertions.assertEquals(Boolean.class, result); + assertEquals(Boolean.class, result); } @Test - public void getTypeClassWithPrefereFloatingPointBoolean() { + public void getTypeClass_WithPrefereFloatingPointBoolean() { var result = DatatypeUtils.getTypeClass(booleanInputValue, true); - Assertions.assertEquals(Boolean.class, result); + assertEquals(Boolean.class, result); } String integerInputValue = "1"; + @Test - public void getTypeClassNoPrefereFloatingPointInteger() { + public void getTypeClass_NoPrefereFloatingPointInteger() { var result = DatatypeUtils.getTypeClass(integerInputValue, false); - Assertions.assertEquals(Integer.class, result); + assertEquals(Integer.class, result); } @Test - public void getTypeClassWithPrefereFloatingPointInteger() { + public void getTypeClass_WithPrefereFloatingPointInteger() { var result = DatatypeUtils.getTypeClass(integerInputValue, true); - Assertions.assertEquals(Float.class, result); + assertEquals(Float.class, result); } String floatInputValue = "1.0"; + @Test - public void getTypeClassNoPrefereFloatingPointFloat() { + public void getTypeClass_NoPrefereFloatingPointFloat() { var result = DatatypeUtils.getTypeClass(floatInputValue, false); - Assertions.assertEquals(Float.class, result); + assertEquals(Float.class, result); } @Test - public void getTypeClassWithPrefereFloatingPointFloat() { + public void getTypeClass_WithPrefereFloatingPointFloat() { var result = DatatypeUtils.getTypeClass(floatInputValue, true); - Assertions.assertEquals(Float.class, result); + assertEquals(Float.class, result); } String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE); + @Test - public void getTypeClassNoPrefereFloatingPointDouble() { + public void getTypeClass_NoPrefereFloatingPointDouble() { var result = DatatypeUtils.getTypeClass(doubleInputValue, false); - Assertions.assertEquals(Double.class, result); + assertEquals(Double.class, result); } @Test - public void getTypeClassWithPrefereFloatingPointDouble() { + public void getTypeClass_WithPrefereFloatingPointDouble() { var result = DatatypeUtils.getTypeClass(doubleInputValue, true); - Assertions.assertEquals(Double.class, result); + assertEquals(Double.class, result); } String longInputValue = "1667904471000"; + @Test - public void getTypeClassNoPrefereFloatingPointLong() { + public void getTypeClass_NoPrefereFloatingPointLong() { var result = DatatypeUtils.getTypeClass(longInputValue, false); - Assertions.assertEquals(Long.class, result); + assertEquals(Long.class, result); } @Test - public void getTypeClassWithPrefereFloatingPointLong() { + public void getTypeClass_WithPrefereFloatingPointLong() { var result = DatatypeUtils.getTypeClass(longInputValue, true); - Assertions.assertEquals(Double.class, result); + assertEquals(Double.class, result); } String stringInputValue = "one"; + @Test - public void getTypeClassNoPrefereFloatingPointString() { + public void getTypeClass_NoPrefereFloatingPointString() { var result = DatatypeUtils.getTypeClass(stringInputValue, false); - Assertions.assertEquals(String.class, result); + assertEquals(String.class, result); } @Test - public void getTypeClassWithPrefereFloatingPointString() { + public void getTypeClass_WithPrefereFloatingPointString() { var result = DatatypeUtils.getTypeClass(stringInputValue, true); - Assertions.assertEquals(String.class, result); + assertEquals(String.class, result); } - } diff --git a/ui/cypress/tests/adapter/fileStream.smoke.spec.ts b/ui/cypress/tests/adapter/fileStream.smoke.spec.ts index 7024335b05..c93bbb6ba5 100644 --- a/ui/cypress/tests/adapter/fileStream.smoke.spec.ts +++ b/ui/cypress/tests/adapter/fileStream.smoke.spec.ts @@ -20,6 +20,7 @@ import { ConnectUtils } from '../../support/utils/connect/ConnectUtils'; import { FileManagementUtils } from '../../support/utils/FileManagementUtils'; import { AdapterBuilder } from '../../support/builder/AdapterBuilder'; import { ConnectBtns } from '../../support/utils/connect/ConnectBtns'; +import { ConnectEventSchemaUtils } from '../../support/utils/connect/ConnectEventSchemaUtils'; describe( 'Test File Replay Adapter', @@ -62,25 +63,27 @@ describe( ConnectUtils.testAdapter(adapterInput, true); }); - // it('File Stream adapter with unix timestamp in seconds', () => { - // FileManagementUtils.addFile('connect/fileReplay/timestampInSeconds/input.csv'); - // const adapterConfiguration = - // ConnectUtils.setUpPreprocessingRuleTest(false); - // - // // Edit timestamp property - // ConnectEventSchemaUtils.editTimestampPropertyWithNumber( - // 'timestamp', - // 'Seconds', - // ); - // - // ConnectEventSchemaUtils.finishEventSchemaConfiguration(); - // ConnectUtils.tearDownPreprocessingRuleTest( - // adapterConfiguration, - // 'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv', - // false, - // 2000, - // ); - // }); + it('File Stream adapter with unix timestamp in seconds', () => { + FileManagementUtils.addFile( + 'connect/fileReplay/timestampInSeconds/input.csv', + ); + const adapterConfiguration = + ConnectUtils.setUpPreprocessingRuleTest(false); + + // Edit timestamp property + ConnectEventSchemaUtils.editTimestampPropertyWithNumber( + 'timestamp', + 'Seconds', + ); + + ConnectEventSchemaUtils.finishEventSchemaConfiguration(); + ConnectUtils.tearDownPreprocessingRuleTest( + adapterConfiguration, + 'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv', + false, + 2000, + ); + }); // it('File Stream adapter with unix timestamp in milliseconds', () => { // FileManagementUtils.addFile('connect/fileReplay/timestampInMilliseconds/input.csv');