Repository: apache/streampipes
Pull request #2996 (2024-07-04 15:43:28)

refactor(#2964): Fix unix timestamp conversion for integer values
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  ~
  -->

  <!--
Thanks for contributing! Here are some tips you can follow to help us incorporate your contribution quickly and easily:
1. If this is your first time, please read our contributor guidelines:
    - https://streampipes.apache.org/community/get-involved/
    - https://cwiki.apache.org/confluence/display/STREAMPIPES/Getting+Started
2. Make sure the PR title is formatted like: `[#<GitHub issue id>] PR title ...`
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., `[WIP][#<GitHub issue id>] PR title ...`.
4. Please write your PR title to summarize what this PR proposes/fixes.
5. Link the PR to the corresponding GitHub issue (if present) in the `Development` section in the right menu bar.
6. Be sure to keep the PR description updated to reflect all changes.
7. If possible, provide a concise example to reproduce the issue for a faster review.
8. Make sure tests pass via `mvn clean install`.
9. (Optional) If the contribution is large, please file an Apache ICLA
    - http://apache.org/licenses/icla.pdf
-->

### Purpose
<!--
Please clarify what changes you are proposing and describe how those changes will address the issue.
Furthermore, describe potential consequences the changes might have.
-->
This PR fixes the unit timestamp conversion rule within adapters.
Therefore, it changed the implementation of `DatatypeUtils` convertValue.

### Remarks
<!--
Is there anything left we need to pay attention on?
Are there some references that might be important? E.g. links to Confluence, or discussions
on the mailing list or GitHub.
-->
PR introduces (a) breaking change(s): no

PR introduces (a) deprecation(s): no



Patch:
diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
index cdfaea25ce..8c95afb2f5 100644
--- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
+++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
@@ -28,6 +28,19 @@ public class DatatypeUtils {
 
   private static final Logger LOG = LoggerFactory.getLogger(DatatypeUtils.class);
 
+  /**
+   * Converts the given value to a specified XSD datatype.
+   * This method attempts to convert the input value to the target datatype specified by the XSD string.
+   * It supports conversion to string, double, float, boolean, integer, and long types.
+   * If the conversion is not possible due to a format mismatch, the original value is returned.
+   * A number format exception during conversion is logged as an error.
+   *
+   * @param value The value to be converted. It can be of any type.
+   * @param targetDatatypeXsd The target XSD datatype as a string. Supported types are XSD.STRING,
+   *                          XSD.DOUBLE, XSD.FLOAT, XSD.BOOLEAN, XSD.INTEGER, and XSD.LONG.
+   * @return The converted value as an Object. If conversion fails, the original value is returned.
+   * @throws NumberFormatException if the string does not contain a parsable number for numeric conversions.
+   */
   public static Object convertValue(Object value,
                                     String targetDatatypeXsd) {
     var stringValue = String.valueOf(value);
@@ -42,7 +55,7 @@ public class DatatypeUtils {
         } else if (XSD.BOOLEAN.toString().equals(targetDatatypeXsd)) {
           return Boolean.parseBoolean(stringValue);
         } else if (XSD.INTEGER.toString().equals(targetDatatypeXsd)) {
-          var floatingNumber = Float.parseFloat(stringValue);
+          var floatingNumber = Double.parseDouble(stringValue);
           return Integer.parseInt(String.valueOf(Math.round(floatingNumber)));
         } else if (XSD.LONG.toString().equals(targetDatatypeXsd)) {
           var floatingNumber = Double.parseDouble(stringValue);
@@ -57,11 +70,6 @@ public class DatatypeUtils {
     return value;
   }
 
-  public static String getCanonicalTypeClassName(String value,
-                                                 boolean preferFloat) {
-    return getTypeClass(value, preferFloat).getCanonicalName();
-  }
-
   public static String getXsdDatatype(String value,
                                       boolean preferFloat) {
     var clazz = getTypeClass(value, preferFloat);


Test patch:
diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
index ef1235b4b8..7937d6875c 100644
--- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
+++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
@@ -21,110 +21,192 @@ package org.apache.streampipes.connect.shared.preprocessing.transform;
 import org.apache.streampipes.connect.shared.DatatypeUtils;
 import org.apache.streampipes.vocabulary.XSD;
 
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.util.Locale;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 public class DatatypeUtilsTest {
 
-  @Test
   /**
-   * This test ensures that timestamps represented as strings are correctly parsed.
+   * The following tests ensure that timestamps represented as strings are correctly parsed.
    * Often they are first parsed into floating point number before transformed back to long.
    * The data type for those values should be Double and not Float, because the transformation to Float might change
    * the value
    */
-  public void convertTimestampValue() {
-    var inputValue = "1667904471000";
+  @Test
+  public void convertValue_StringToStringValue() {
+    var inputValue = "testString";
+    var actualValue = DatatypeUtils.convertValue(inputValue, XSD.STRING.toString());
+
+    assertEquals(inputValue, actualValue);
+  }
+
+  @Test
+  public void convertValue_StringToDoubleValue() {
+    var actualValue = DatatypeUtils.convertValue("1667904471000", XSD.DOUBLE.toString());
+
+    assertEquals(1.667904471E12, actualValue);
+  }
+
+  @Test
+  public void convertValue_StringToFloatValue() {
+    var actualValue = DatatypeUtils.convertValue("123.45", XSD.FLOAT.toString());
+
+    assertEquals(123.45f, actualValue);
+  }
+
+  @Test
+  public void convertValue_StringToInteger() {
+    var actualValue = DatatypeUtils.convertValue("1623871500", XSD.INTEGER.toString());
+
+    assertEquals(1623871500, actualValue);
+  }
+
+  @Test
+  public void convertValue_StringToIntegerValue() {
+    var actualValue = DatatypeUtils.convertValue("123", XSD.INTEGER.toString());
+
+    assertEquals(123, actualValue);
+  }
+
+  @Test
+  public void convertValue_StringToLongValue() {
+    var actualValue = DatatypeUtils.convertValue("1623871500000", XSD.LONG.toString());
+
+    assertEquals(1623871500000L, actualValue);
+  }
+
+  @Test
+  public void convertValue_StringToBooleanTrueValue() {
+    var actualValue = DatatypeUtils.convertValue("true", XSD.BOOLEAN.toString());
 
-    var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString());
-    var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString());
+    assertEquals(true, actualValue);
+  }
+
+  @Test
+  public void convertValue_StringToBooleanFalseValue() {
+    var actualValue = DatatypeUtils.convertValue("false", XSD.BOOLEAN.toString());
+
+    assertEquals(false, actualValue);
+  }
+
+  @Test
+  public void convertValue_FloatToIntegerValue_Rounding() {
+    var actualValue = DatatypeUtils.convertValue(123.45f, XSD.INTEGER.toString());
 
-    Assertions.assertEquals(Long.parseLong(inputValue), longValue);
+    assertEquals(123, actualValue);
   }
 
+  @Test
+  public void convertValue_DoubleToLongValue_Rounding1() {
+    var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString());
+
+    assertEquals(1234567890L, actualValue);
+  }
+
+  @Test
+  public void convertValue_DoubleToLongValue() {
+    var actualValue = DatatypeUtils.convertValue(1.667904471E12, XSD.LONG.toString());
+
+    assertEquals(1667904471000L, actualValue);
+  }
+
+  @Test
+  public void convertValue_DoubleToLongValue_Rounding() {
+    var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString());
+
+    assertEquals(1234567890L, actualValue);
+  }
+
+
   String booleanInputValue = "true";
+
   @Test
-  public void getTypeClassNoPrefereFloatingPointBoolean() {
+  public void getTypeClass_NoPrefereFloatingPointBoolean() {
     var result = DatatypeUtils.getTypeClass(booleanInputValue, false);
-    Assertions.assertEquals(Boolean.class, result);
+    assertEquals(Boolean.class, result);
   }
 
   @Test
-  public void getTypeClassWithPrefereFloatingPointBoolean() {
+  public void getTypeClass_WithPrefereFloatingPointBoolean() {
     var result = DatatypeUtils.getTypeClass(booleanInputValue, true);
-    Assertions.assertEquals(Boolean.class, result);
+    assertEquals(Boolean.class, result);
   }
 
   String integerInputValue = "1";
+
   @Test
-  public void getTypeClassNoPrefereFloatingPointInteger() {
+  public void getTypeClass_NoPrefereFloatingPointInteger() {
     var result = DatatypeUtils.getTypeClass(integerInputValue, false);
-    Assertions.assertEquals(Integer.class, result);
+    assertEquals(Integer.class, result);
   }
 
   @Test
-  public void getTypeClassWithPrefereFloatingPointInteger() {
+  public void getTypeClass_WithPrefereFloatingPointInteger() {
     var result = DatatypeUtils.getTypeClass(integerInputValue, true);
-    Assertions.assertEquals(Float.class, result);
+    assertEquals(Float.class, result);
   }
 
   String floatInputValue = "1.0";
+
   @Test
-  public void getTypeClassNoPrefereFloatingPointFloat() {
+  public void getTypeClass_NoPrefereFloatingPointFloat() {
     var result = DatatypeUtils.getTypeClass(floatInputValue, false);
-    Assertions.assertEquals(Float.class, result);
+    assertEquals(Float.class, result);
   }
 
   @Test
-  public void getTypeClassWithPrefereFloatingPointFloat() {
+  public void getTypeClass_WithPrefereFloatingPointFloat() {
     var result = DatatypeUtils.getTypeClass(floatInputValue, true);
-    Assertions.assertEquals(Float.class, result);
+    assertEquals(Float.class, result);
   }
 
 
   String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE);
+
   @Test
-  public void getTypeClassNoPrefereFloatingPointDouble() {
+  public void getTypeClass_NoPrefereFloatingPointDouble() {
     var result = DatatypeUtils.getTypeClass(doubleInputValue, false);
-    Assertions.assertEquals(Double.class, result);
+    assertEquals(Double.class, result);
   }
 
   @Test
-  public void getTypeClassWithPrefereFloatingPointDouble() {
+  public void getTypeClass_WithPrefereFloatingPointDouble() {
     var result = DatatypeUtils.getTypeClass(doubleInputValue, true);
-    Assertions.assertEquals(Double.class, result);
+    assertEquals(Double.class, result);
   }
 
 
   String longInputValue = "1667904471000";
+
   @Test
-  public void getTypeClassNoPrefereFloatingPointLong() {
+  public void getTypeClass_NoPrefereFloatingPointLong() {
     var result = DatatypeUtils.getTypeClass(longInputValue, false);
-    Assertions.assertEquals(Long.class, result);
+    assertEquals(Long.class, result);
   }
 
   @Test
-  public void getTypeClassWithPrefereFloatingPointLong() {
+  public void getTypeClass_WithPrefereFloatingPointLong() {
     var result = DatatypeUtils.getTypeClass(longInputValue, true);
-    Assertions.assertEquals(Double.class, result);
+    assertEquals(Double.class, result);
   }
 
   String stringInputValue = "one";
+
   @Test
-  public void getTypeClassNoPrefereFloatingPointString() {
+  public void getTypeClass_NoPrefereFloatingPointString() {
     var result = DatatypeUtils.getTypeClass(stringInputValue, false);
 
-    Assertions.assertEquals(String.class, result);
+    assertEquals(String.class, result);
   }
 
   @Test
-  public void getTypeClassWithPrefereFloatingPointString() {
+  public void getTypeClass_WithPrefereFloatingPointString() {
     var result = DatatypeUtils.getTypeClass(stringInputValue, true);
 
-    Assertions.assertEquals(String.class, result);
+    assertEquals(String.class, result);
   }
 
-
 }
diff --git a/ui/cypress/tests/adapter/fileStream.smoke.spec.ts b/ui/cypress/tests/adapter/fileStream.smoke.spec.ts
index 7024335b05..c93bbb6ba5 100644
--- a/ui/cypress/tests/adapter/fileStream.smoke.spec.ts
+++ b/ui/cypress/tests/adapter/fileStream.smoke.spec.ts
@@ -20,6 +20,7 @@ import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
 import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
 import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
 import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
+import { ConnectEventSchemaUtils } from '../../support/utils/connect/ConnectEventSchemaUtils';
 
 describe(
     'Test File Replay Adapter',
@@ -62,25 +63,27 @@ describe(
             ConnectUtils.testAdapter(adapterInput, true);
         });
 
-        // it('File Stream adapter with unix timestamp in seconds', () => {
-        //     FileManagementUtils.addFile('connect/fileReplay/timestampInSeconds/input.csv');
-        //     const adapterConfiguration =
-        //         ConnectUtils.setUpPreprocessingRuleTest(false);
-        //
-        //     // Edit timestamp property
-        //     ConnectEventSchemaUtils.editTimestampPropertyWithNumber(
-        //         'timestamp',
-        //         'Seconds',
-        //     );
-        //
-        //     ConnectEventSchemaUtils.finishEventSchemaConfiguration();
-        //     ConnectUtils.tearDownPreprocessingRuleTest(
-        //         adapterConfiguration,
-        //         'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv',
-        //         false,
-        //         2000,
-        //     );
-        // });
+        it('File Stream adapter with unix timestamp in seconds', () => {
+            FileManagementUtils.addFile(
+                'connect/fileReplay/timestampInSeconds/input.csv',
+            );
+            const adapterConfiguration =
+                ConnectUtils.setUpPreprocessingRuleTest(false);
+
+            // Edit timestamp property
+            ConnectEventSchemaUtils.editTimestampPropertyWithNumber(
+                'timestamp',
+                'Seconds',
+            );
+
+            ConnectEventSchemaUtils.finishEventSchemaConfiguration();
+            ConnectUtils.tearDownPreprocessingRuleTest(
+                adapterConfiguration,
+                'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv',
+                false,
+                2000,
+            );
+        });
 
         // it('File Stream adapter with unix timestamp in milliseconds', () => {
         //     FileManagementUtils.addFile('connect/fileReplay/timestampInMilliseconds/input.csv');