Skip to content

Commit

Permalink
refactor(#2964): Fix unix timestamp conversion for integer values (#2996
Browse files Browse the repository at this point in the history 
)

* refactor(#2964): Fix unix timestamp conversion for integer values

* refactor(#2964): Fix linting
  • Loading branch information
@tenthe
tenthe authored Jul 4, 2024
1 parent 8d35cf0 commit a6e5958
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 58 deletions.
20 changes: 14 additions & 6 deletions ...pes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,19 @@ public class DatatypeUtils {

private static final Logger LOG = LoggerFactory.getLogger(DatatypeUtils.class);

/**
* Converts the given value to a specified XSD datatype.
* This method attempts to convert the input value to the target datatype specified by the XSD string.
* It supports conversion to string, double, float, boolean, integer, and long types.
* If the conversion is not possible due to a format mismatch, the original value is returned.
* A number format exception during conversion is logged as an error.
*
* @param value The value to be converted. It can be of any type.
* @param targetDatatypeXsd The target XSD datatype as a string. Supported types are XSD.STRING,
* XSD.DOUBLE, XSD.FLOAT, XSD.BOOLEAN, XSD.INTEGER, and XSD.LONG.
* @return The converted value as an Object. If conversion fails, the original value is returned.
* @throws NumberFormatException if the string does not contain a parsable number for numeric conversions.
*/
public static Object convertValue(Object value,
String targetDatatypeXsd) {
var stringValue = String.valueOf(value);
Expand All @@ -42,7 +55,7 @@ public static Object convertValue(Object value,
} else if (XSD.BOOLEAN.toString().equals(targetDatatypeXsd)) {
return Boolean.parseBoolean(stringValue);
} else if (XSD.INTEGER.toString().equals(targetDatatypeXsd)) {
var floatingNumber = Float.parseFloat(stringValue);
var floatingNumber = Double.parseDouble(stringValue);
return Integer.parseInt(String.valueOf(Math.round(floatingNumber)));
} else if (XSD.LONG.toString().equals(targetDatatypeXsd)) {
var floatingNumber = Double.parseDouble(stringValue);
Expand All @@ -57,11 +70,6 @@ public static Object convertValue(Object value,
return value;
}

public static String getCanonicalTypeClassName(String value,
boolean preferFloat) {
return getTypeClass(value, preferFloat).getCanonicalName();
}

public static String getXsdDatatype(String value,
boolean preferFloat) {
var clazz = getTypeClass(value, preferFloat);
Expand Down
148 changes: 115 additions & 33 deletions ...java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,110 +21,192 @@
import org.apache.streampipes.connect.shared.DatatypeUtils;
import org.apache.streampipes.vocabulary.XSD;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.Locale;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class DatatypeUtilsTest {

@Test
/**
* This test ensures that timestamps represented as strings are correctly parsed.
* The following tests ensure that timestamps represented as strings are correctly parsed.
* Often they are first parsed into floating point number before transformed back to long.
* The data type for those values should be Double and not Float, because the transformation to Float might change
* the value
*/
public void convertTimestampValue() {
var inputValue = "1667904471000";
@Test
public void convertValue_StringToStringValue() {
var inputValue = "testString";
var actualValue = DatatypeUtils.convertValue(inputValue, XSD.STRING.toString());

assertEquals(inputValue, actualValue);
}

@Test
public void convertValue_StringToDoubleValue() {
var actualValue = DatatypeUtils.convertValue("1667904471000", XSD.DOUBLE.toString());

assertEquals(1.667904471E12, actualValue);
}

@Test
public void convertValue_StringToFloatValue() {
var actualValue = DatatypeUtils.convertValue("123.45", XSD.FLOAT.toString());

assertEquals(123.45f, actualValue);
}

@Test
public void convertValue_StringToInteger() {
var actualValue = DatatypeUtils.convertValue("1623871500", XSD.INTEGER.toString());

assertEquals(1623871500, actualValue);
}

@Test
public void convertValue_StringToIntegerValue() {
var actualValue = DatatypeUtils.convertValue("123", XSD.INTEGER.toString());

assertEquals(123, actualValue);
}

@Test
public void convertValue_StringToLongValue() {
var actualValue = DatatypeUtils.convertValue("1623871500000", XSD.LONG.toString());

assertEquals(1623871500000L, actualValue);
}

@Test
public void convertValue_StringToBooleanTrueValue() {
var actualValue = DatatypeUtils.convertValue("true", XSD.BOOLEAN.toString());

var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString());
var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString());
assertEquals(true, actualValue);
}

@Test
public void convertValue_StringToBooleanFalseValue() {
var actualValue = DatatypeUtils.convertValue("false", XSD.BOOLEAN.toString());

assertEquals(false, actualValue);
}

@Test
public void convertValue_FloatToIntegerValue_Rounding() {
var actualValue = DatatypeUtils.convertValue(123.45f, XSD.INTEGER.toString());

Assertions.assertEquals(Long.parseLong(inputValue), longValue);
assertEquals(123, actualValue);
}

@Test
public void convertValue_DoubleToLongValue_Rounding1() {
var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString());

assertEquals(1234567890L, actualValue);
}

@Test
public void convertValue_DoubleToLongValue() {
var actualValue = DatatypeUtils.convertValue(1.667904471E12, XSD.LONG.toString());

assertEquals(1667904471000L, actualValue);
}

@Test
public void convertValue_DoubleToLongValue_Rounding() {
var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString());

assertEquals(1234567890L, actualValue);
}


String booleanInputValue = "true";

@Test
public void getTypeClassNoPrefereFloatingPointBoolean() {
public void getTypeClass_NoPrefereFloatingPointBoolean() {
var result = DatatypeUtils.getTypeClass(booleanInputValue, false);
Assertions.assertEquals(Boolean.class, result);
assertEquals(Boolean.class, result);
}

@Test
public void getTypeClassWithPrefereFloatingPointBoolean() {
public void getTypeClass_WithPrefereFloatingPointBoolean() {
var result = DatatypeUtils.getTypeClass(booleanInputValue, true);
Assertions.assertEquals(Boolean.class, result);
assertEquals(Boolean.class, result);
}

String integerInputValue = "1";

@Test
public void getTypeClassNoPrefereFloatingPointInteger() {
public void getTypeClass_NoPrefereFloatingPointInteger() {
var result = DatatypeUtils.getTypeClass(integerInputValue, false);
Assertions.assertEquals(Integer.class, result);
assertEquals(Integer.class, result);
}

@Test
public void getTypeClassWithPrefereFloatingPointInteger() {
public void getTypeClass_WithPrefereFloatingPointInteger() {
var result = DatatypeUtils.getTypeClass(integerInputValue, true);
Assertions.assertEquals(Float.class, result);
assertEquals(Float.class, result);
}

String floatInputValue = "1.0";

@Test
public void getTypeClassNoPrefereFloatingPointFloat() {
public void getTypeClass_NoPrefereFloatingPointFloat() {
var result = DatatypeUtils.getTypeClass(floatInputValue, false);
Assertions.assertEquals(Float.class, result);
assertEquals(Float.class, result);
}

@Test
public void getTypeClassWithPrefereFloatingPointFloat() {
public void getTypeClass_WithPrefereFloatingPointFloat() {
var result = DatatypeUtils.getTypeClass(floatInputValue, true);
Assertions.assertEquals(Float.class, result);
assertEquals(Float.class, result);
}


String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE);

@Test
public void getTypeClassNoPrefereFloatingPointDouble() {
public void getTypeClass_NoPrefereFloatingPointDouble() {
var result = DatatypeUtils.getTypeClass(doubleInputValue, false);
Assertions.assertEquals(Double.class, result);
assertEquals(Double.class, result);
}

@Test
public void getTypeClassWithPrefereFloatingPointDouble() {
public void getTypeClass_WithPrefereFloatingPointDouble() {
var result = DatatypeUtils.getTypeClass(doubleInputValue, true);
Assertions.assertEquals(Double.class, result);
assertEquals(Double.class, result);
}


String longInputValue = "1667904471000";

@Test
public void getTypeClassNoPrefereFloatingPointLong() {
public void getTypeClass_NoPrefereFloatingPointLong() {
var result = DatatypeUtils.getTypeClass(longInputValue, false);
Assertions.assertEquals(Long.class, result);
assertEquals(Long.class, result);
}

@Test
public void getTypeClassWithPrefereFloatingPointLong() {
public void getTypeClass_WithPrefereFloatingPointLong() {
var result = DatatypeUtils.getTypeClass(longInputValue, true);
Assertions.assertEquals(Double.class, result);
assertEquals(Double.class, result);
}

String stringInputValue = "one";

@Test
public void getTypeClassNoPrefereFloatingPointString() {
public void getTypeClass_NoPrefereFloatingPointString() {
var result = DatatypeUtils.getTypeClass(stringInputValue, false);

Assertions.assertEquals(String.class, result);
assertEquals(String.class, result);
}

@Test
public void getTypeClassWithPrefereFloatingPointString() {
public void getTypeClass_WithPrefereFloatingPointString() {
var result = DatatypeUtils.getTypeClass(stringInputValue, true);

Assertions.assertEquals(String.class, result);
assertEquals(String.class, result);
}


}
41 changes: 22 additions & 19 deletions ui/cypress/tests/adapter/fileStream.smoke.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
import { ConnectEventSchemaUtils } from '../../support/utils/connect/ConnectEventSchemaUtils';

describe(
'Test File Replay Adapter',
Expand Down Expand Up @@ -62,25 +63,27 @@ describe(
ConnectUtils.testAdapter(adapterInput, true);
});

// it('File Stream adapter with unix timestamp in seconds', () => {
// FileManagementUtils.addFile('connect/fileReplay/timestampInSeconds/input.csv');
// const adapterConfiguration =
// ConnectUtils.setUpPreprocessingRuleTest(false);
//
// // Edit timestamp property
// ConnectEventSchemaUtils.editTimestampPropertyWithNumber(
// 'timestamp',
// 'Seconds',
// );
//
// ConnectEventSchemaUtils.finishEventSchemaConfiguration();
// ConnectUtils.tearDownPreprocessingRuleTest(
// adapterConfiguration,
// 'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv',
// false,
// 2000,
// );
// });
it('File Stream adapter with unix timestamp in seconds', () => {
FileManagementUtils.addFile(
'connect/fileReplay/timestampInSeconds/input.csv',
);
const adapterConfiguration =
ConnectUtils.setUpPreprocessingRuleTest(false);

// Edit timestamp property
ConnectEventSchemaUtils.editTimestampPropertyWithNumber(
'timestamp',
'Seconds',
);

ConnectEventSchemaUtils.finishEventSchemaConfiguration();
ConnectUtils.tearDownPreprocessingRuleTest(
adapterConfiguration,
'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv',
false,
2000,
);
});

// it('File Stream adapter with unix timestamp in milliseconds', () => {
// FileManagementUtils.addFile('connect/fileReplay/timestampInMilliseconds/input.csv');
Expand Down

0 comments on commit a6e5958

Please sign in to comment.