Repository: apache/streampipes
Pull request #1099 (2023-01-16 16:31:08)

[hotfix] Fix failing e2e tests
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  ~
  -->
  
  <!--
Thanks for contributing! Here are some tips you can follow to help us incorporate your contribution quickly and easily:
1. If this is your first time, please read our contributor guidelines:
    - https://streampipes.apache.org/getinvolved.html
    - https://cwiki.apache.org/confluence/display/STREAMPIPES/Getting+Started
2. Make sure the PR title is formatted like: `[#<GitHub issue id>] PR title ...`
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., `[WIP][#<GitHub issue id>] PR title ...`.
4. Please write your PR title to summarize what this PR proposes/fixes.
5. Link the PR to the corresponding GitHub issue (if present) in the `Development` section in the right menu bar. 
6. Be sure to keep the PR description updated to reflect all changes.
7. If possible, provide a concise example to reproduce the issue for a faster review.
8. Make sure tests pass via `mvn clean install`.
9. (Optional) If the contribution is large, please file an Apache ICLA
    - http://apache.org/licenses/icla.pdf
-->

### Purpose
<!--
Please clarify what changes you are proposing and describe how those changes will address the issue.
Furthermore, describe potential consequences the changes might have.
-->
This PR fixes the failing e2e tests. 
There were two problems:
- The data-cy tag `data-cy="schema-preview-original-event"` was missing for the format tests.
- The behavior of the CSV parser has been changed. Now it uses Double as default type to ensure that the values do not change during parsing.

### Remarks
<!--
Is there anything left we need to pay attention on?
Are there some references that might be important? E.g. links to Confluence, or discussions
on the mailing list or GitHub.
-->
PR introduces (a) breaking change(s): no

PR introduces (a) deprecation(s): no



Patch:
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
index a0a68d9632..12724b38a4 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
@@ -108,11 +108,11 @@ public class CsvParser extends Parser {
     EventSchema resultSchema = new EventSchema();
     for (int i = 0; i < keys.length; i++) {
       EventPropertyPrimitive p = new EventPropertyPrimitive();
-      var runtimeType = DatatypeUtils.getXsdDatatype(data[i], false);
+      var runtimeType = DatatypeUtils.getXsdDatatype(data[i], true);
       var convertedValue = DatatypeUtils.convertValue(data[i], runtimeType);
       p.setRuntimeName(keys[i]);
       p.setRuntimeType(runtimeType);
-      sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], false), convertedValue));
+      sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], true), convertedValue));
       resultSchema.addEventProperty(p);
     }
 
@@ -165,11 +165,6 @@ public class CsvParser extends Parser {
 
           inQuotes = true;
 
-          //Fixed : allow "" in empty quote enclosed
-//                    if (chars[0] != '"' && customQuote == '\"') {
-//                        curVal.append('"');
-//                    }
-
           //double quotes in column will hit this!
           if (startCollectChar) {
             curVal.append('"');
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
index 20655f112d..4e1933ee0f 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
@@ -81,17 +81,29 @@ public class DatatypeUtils {
   }
 
   public static Class<?> getTypeClass(String value,
-                                      boolean preferFloat) {
+                                      boolean preferFloatingPointNumber) {
+    var targetClass = String.class;
     if (NumberUtils.isParsable(value)) {
+      Class<?> numberClass;
       try {
-        Integer.parseInt(value);
-        return preferFloat ? Float.class : Integer.class;
+        long longValue = Long.parseLong(value);
+        numberClass = longValue > Integer.MAX_VALUE ? Long.class : Integer.class;
+        if (preferFloatingPointNumber) {
+          return numberClass == Long.class ? Double.class : Float.class;
+        } else {
+          return numberClass;
+        }
       } catch (NumberFormatException ignored) {
       }
 
       try {
-        Long.parseLong(value);
-        return preferFloat ? Float.class : Long.class;
+        double doubleValue = Double.parseDouble(value);
+        numberClass = doubleValue > Float.MAX_VALUE ? Double.class : Float.class;
+        if (preferFloatingPointNumber) {
+          return numberClass == Double.class ? Double.class : Float.class;
+        } else {
+          return numberClass;
+        }
       } catch (NumberFormatException ignored) {
       }
 
@@ -107,13 +119,7 @@ public class DatatypeUtils {
       return Boolean.class;
     }
 
-    return String.class;
+    return targetClass;
   }
 
-  public static void main(String[] args) {
-    long max = Long.MAX_VALUE;
-    String className = getCanonicalTypeClassName(String.valueOf(max), true);
-    System.out.println(className);
-    System.out.println(convertValue(max, className));
-  }
 }
diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
index a55ad96cec..6bb8b51e36 100644
--- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
+++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
@@ -22,6 +22,7 @@
             <pre
                 [innerHTML]="originalField | jsonpretty"
                 class="preview-text"
+                data-cy="schema-preview-original-event"
             ></pre>
         </sp-basic-inner-panel>
     </div>


Test patch:
diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java
new file mode 100644
index 0000000000..f525319ad9
--- /dev/null
+++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.management.connect.adapter.format.csv;
+
+
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class CsvParserTest {
+
+  @Test
+  public void getSchemaAndSampleTimestamp() {
+    var csvValue = List.of(
+        "timestamp,value".getBytes(StandardCharsets.UTF_8),
+        "1667904471000,1".getBytes(StandardCharsets.UTF_8)
+    );
+
+    var csvParser = new CsvParser(",", true);
+    var result = csvParser.getSchemaAndSample(csvValue);
+
+    assertEquals(result.getEventPreview().size(), 1);
+    assertEquals(1667904471000.0, result.getEventPreview().get(0).get("timestamp").getValue());
+  }
+
+}
\ No newline at end of file
diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java
new file mode 100644
index 0000000000..c5d273e071
--- /dev/null
+++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.management.connect.adapter.util;
+
+import org.apache.streampipes.vocabulary.XSD;
+
+import org.junit.Test;
+
+import java.util.Locale;
+
+import static org.junit.Assert.assertEquals;
+
+public class DatatypeUtilsTest {
+
+  @Test
+  /**
+   * This test ensures that timestamps represented as strings are correctly parsed.
+   * Often they are first parsed into floating point number before transformed back to long.
+   * The data type for those values should be Double and not Float, because the transformation to Float might change
+   * the value
+   */
+  public void convertTimestampValue() {
+    var inputValue = "1667904471000";
+
+    var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString());
+    var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString());
+
+    assertEquals(Long.parseLong(inputValue), longValue);
+  }
+
+  String booleanInputValue = "true";
+  @Test
+  public void getTypeClassNoPrefereFloatingPointBoolean() {
+    var result = DatatypeUtils.getTypeClass(booleanInputValue, false);
+    assertEquals(Boolean.class, result);
+  }
+
+  @Test
+  public void getTypeClassWithPrefereFloatingPointBoolean() {
+    var result = DatatypeUtils.getTypeClass(booleanInputValue, true);
+    assertEquals(Boolean.class, result);
+  }
+
+  String integerInputValue = "1";
+  @Test
+  public void getTypeClassNoPrefereFloatingPointInteger() {
+    var result = DatatypeUtils.getTypeClass(integerInputValue, false);
+    assertEquals(Integer.class, result);
+  }
+
+  @Test
+  public void getTypeClassWithPrefereFloatingPointInteger() {
+    var result = DatatypeUtils.getTypeClass(integerInputValue, true);
+    assertEquals(Float.class, result);
+  }
+
+  String floatInputValue = "1.0";
+  @Test
+  public void getTypeClassNoPrefereFloatingPointFloat() {
+    var result = DatatypeUtils.getTypeClass(floatInputValue, false);
+    assertEquals(Float.class, result);
+  }
+
+  @Test
+  public void getTypeClassWithPrefereFloatingPointFloat() {
+    var result = DatatypeUtils.getTypeClass(floatInputValue, true);
+    assertEquals(Float.class, result);
+  }
+
+
+  String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE);
+  @Test
+  public void getTypeClassNoPrefereFloatingPointDouble() {
+    var result = DatatypeUtils.getTypeClass(doubleInputValue, false);
+    assertEquals(Double.class, result);
+  }
+
+  @Test
+  public void getTypeClassWithPrefereFloatingPointDouble() {
+    var result = DatatypeUtils.getTypeClass(doubleInputValue, true);
+    assertEquals(Double.class, result);
+  }
+
+
+  String longInputValue = "1667904471000";
+  @Test
+  public void getTypeClassNoPrefereFloatingPointLong() {
+    var result = DatatypeUtils.getTypeClass(longInputValue, false);
+    assertEquals(Long.class, result);
+  }
+
+  @Test
+  public void getTypeClassWithPrefereFloatingPointLong() {
+    var result = DatatypeUtils.getTypeClass(longInputValue, true);
+    assertEquals(Double.class, result);
+  }
+
+  String stringInputValue = "one";
+  @Test
+  public void getTypeClassNoPrefereFloatingPointString() {
+    var result = DatatypeUtils.getTypeClass(stringInputValue, false);
+
+    assertEquals(String.class, result);
+  }
+
+  @Test
+  public void getTypeClassWithPrefereFloatingPointString() {
+    var result = DatatypeUtils.getTypeClass(stringInputValue, true);
+
+    assertEquals(String.class, result);
+  }
+
+
+}
\ No newline at end of file