Repository: apache/streampipes
Pull request #1774 (2023-07-15 21:14:32)

Replace forbidden keywords in data lake measure (#1769)
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  ~
  -->
  
  <!--
Thanks for contributing! Here are some tips you can follow to help us incorporate your contribution quickly and easily:
1. If this is your first time, please read our contributor guidelines:
    - https://streampipes.apache.org/getinvolved.html
    - https://cwiki.apache.org/confluence/display/STREAMPIPES/Getting+Started
2. Make sure the PR title is formatted like: `[#<GitHub issue id>] PR title ...`
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., `[WIP][#<GitHub issue id>] PR title ...`.
4. Please write your PR title to summarize what this PR proposes/fixes.
5. Link the PR to the corresponding GitHub issue (if present) in the `Development` section in the right menu bar. 
6. Be sure to keep the PR description updated to reflect all changes.
7. If possible, provide a concise example to reproduce the issue for a faster review.
8. Make sure tests pass via `mvn clean install`.
9. (Optional) If the contribution is large, please file an Apache ICLA
    - http://apache.org/licenses/icla.pdf
-->

### Purpose
<!--
Please clarify what changes you are proposing and describe how those changes will address the issue.
Furthermore, describe potential consequences the changes might have.
-->

See #1769 

### Remarks
<!--
Is there anything left we need to pay attention on?
Are there some references that might be important? E.g. links to Confluence, or discussions
on the mailing list or GitHub.
-->
PR introduces (a) breaking change(s): no

PR introduces (a) deprecation(s): no


Patch:
diff --git a/streampipes-data-explorer-commons/pom.xml b/streampipes-data-explorer-commons/pom.xml
index 579a6fe50f..3035df2733 100644
--- a/streampipes-data-explorer-commons/pom.xml
+++ b/streampipes-data-explorer-commons/pom.xml
@@ -48,6 +48,12 @@
             <version>0.93.0-SNAPSHOT</version>
         </dependency>
 
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
 
         <!-- Others -->
         <dependency>
diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java
index 85259ad653..6c28f9c145 100644
--- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java
+++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.dataexplorer.commons;
 import org.apache.streampipes.client.api.IStreamPipesClient;
 import org.apache.streampipes.commons.exceptions.SpRuntimeException;
 import org.apache.streampipes.dataexplorer.commons.influx.InfluxNameSanitizer;
+import org.apache.streampipes.dataexplorer.commons.sanitizer.MeasureNameSanitizer;
 import org.apache.streampipes.model.datalake.DataLakeMeasure;
 import org.apache.streampipes.model.schema.EventProperty;
 
@@ -64,6 +65,9 @@ public class DataExplorerUtils {
     // Removes selected timestamp from event schema
     removeTimestampsFromEventSchema(measure);
 
+    // Sanitize the data lake measure name
+    measure.setMeasureName(new MeasureNameSanitizer().sanitize(measure.getMeasureName()));
+
     // Removes all spaces with _ and validates that no special terms are used as runtime names
     measure.getEventSchema()
         .getEventProperties()
diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java
new file mode 100644
index 0000000000..1ffbd02fef
--- /dev/null
+++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.dataexplorer.commons.sanitizer;
+
+import java.util.List;
+
+public class MeasureNameSanitizer {
+
+  private final List<String> forbiddenChars = List.of(
+      "/",
+      "?",
+      "=",
+      "\""
+  );
+
+  private final String replacement = "_";
+
+  public String sanitize(String measureName) {
+    return forbiddenChars.stream()
+        .reduce(measureName, (str, forbiddenChar) -> str.replace(forbiddenChar, replacement));
+  }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java
index 361fe09093..a7588f66ba 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java
@@ -17,6 +17,7 @@
  */
 package org.apache.streampipes.rest.impl.datalake;
 
+import org.apache.streampipes.dataexplorer.commons.sanitizer.MeasureNameSanitizer;
 import org.apache.streampipes.model.datalake.DataLakeMeasure;
 import org.apache.streampipes.model.graph.DataSinkInvocation;
 import org.apache.streampipes.model.pipeline.Pipeline;
@@ -66,11 +67,15 @@ public class PersistedDataStreamResource extends AbstractPipelineExtractionResou
 
   @Override
   protected DataLakeMeasure convert(Pipeline pipeline, DataSinkInvocation sink) {
+
+    var measureName = extractFieldValue(sink, MeasureFieldInternalName);
+    var sanitizedMeasureName = new MeasureNameSanitizer().sanitize(measureName);
+
     DataLakeMeasure measure = new DataLakeMeasure();
     measure.setEventSchema(sink.getInputStreams().get(0).getEventSchema());
     measure.setPipelineId(pipeline.getPipelineId());
     measure.setPipelineName(pipeline.getName());
-    measure.setMeasureName(extractFieldValue(sink, MeasureFieldInternalName));
+    measure.setMeasureName(sanitizedMeasureName);
     measure.setPipelineIsRunning(pipeline.isRunning());
 
     return measure;
diff --git a/ui/package-lock.json b/ui/package-lock.json
index ee9ab62ec4..6b7f4b9426 100644
--- a/ui/package-lock.json
+++ b/ui/package-lock.json
@@ -88,7 +88,7 @@
         "@types/rx": "^4.1.2",
         "@types/showdown": "1.9.4",
         "@typescript-eslint/eslint-plugin": "5.57.1",
-        "@typescript-eslint/parser": "5.59.11",
+        "@typescript-eslint/parser": "5.62.0",
         "ajv-keywords": "^5.1.0",
         "assert": "^2.0.0",
         "compression-webpack-plugin": "^10.0.0",
@@ -7103,14 +7103,14 @@
       }
     },
     "node_modules/@typescript-eslint/parser": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.59.11.tgz",
-      "integrity": "sha512-s9ZF3M+Nym6CAZEkJJeO2TFHHDsKAM3ecNkLuH4i4s8/RCPnF5JRip2GyviYkeEAcwGMJxkqG9h2dAsnA1nZpA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.62.0.tgz",
+      "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/scope-manager": "5.59.11",
-        "@typescript-eslint/types": "5.59.11",
-        "@typescript-eslint/typescript-estree": "5.59.11",
+        "@typescript-eslint/scope-manager": "5.62.0",
+        "@typescript-eslint/types": "5.62.0",
+        "@typescript-eslint/typescript-estree": "5.62.0",
         "debug": "^4.3.4"
       },
       "engines": {
@@ -7130,13 +7130,13 @@
       }
     },
     "node_modules/@typescript-eslint/scope-manager": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.59.11.tgz",
-      "integrity": "sha512-dHFOsxoLFtrIcSj5h0QoBT/89hxQONwmn3FOQ0GOQcLOOXm+MIrS8zEAhs4tWl5MraxCY3ZJpaXQQdFMc2Tu+Q==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.62.0.tgz",
+      "integrity": "sha512-VXuvVvZeQCQb5Zgf4HAxc04q5j+WrNAtNh9OwCsCgpKqESMTu3tF/jhZ3xG6T4NZwWl65Bg8KuS2uEvhSfLl0w==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/types": "5.59.11",
-        "@typescript-eslint/visitor-keys": "5.59.11"
+        "@typescript-eslint/types": "5.62.0",
+        "@typescript-eslint/visitor-keys": "5.62.0"
       },
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@@ -7231,9 +7231,9 @@
       }
     },
     "node_modules/@typescript-eslint/types": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.59.11.tgz",
-      "integrity": "sha512-epoN6R6tkvBYSc+cllrz+c2sOFWkbisJZWkOE+y3xHtvYaOE6Wk6B8e114McRJwFRjGvYdJwLXQH5c9osME/AA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.62.0.tgz",
+      "integrity": "sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ==",
       "dev": true,
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@@ -7244,13 +7244,13 @@
       }
     },
     "node_modules/@typescript-eslint/typescript-estree": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.59.11.tgz",
-      "integrity": "sha512-YupOpot5hJO0maupJXixi6l5ETdrITxeo5eBOeuV7RSKgYdU3G5cxO49/9WRnJq9EMrB7AuTSLH/bqOsXi7wPA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.62.0.tgz",
+      "integrity": "sha512-CmcQ6uY7b9y694lKdRB8FEel7JbU/40iSAPomu++SjLMntB+2Leay2LO6i8VnJk58MtE9/nQSFIH6jpyRWyYzA==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/types": "5.59.11",
-        "@typescript-eslint/visitor-keys": "5.59.11",
+        "@typescript-eslint/types": "5.62.0",
+        "@typescript-eslint/visitor-keys": "5.62.0",
         "debug": "^4.3.4",
         "globby": "^11.1.0",
         "is-glob": "^4.0.3",
@@ -7393,12 +7393,12 @@
       }
     },
     "node_modules/@typescript-eslint/visitor-keys": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.59.11.tgz",
-      "integrity": "sha512-KGYniTGG3AMTuKF9QBD7EIrvufkB6O6uX3knP73xbKLMpH+QRPcgnCxjWXSHjMRuOxFLovljqQgQpR0c7GvjoA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.62.0.tgz",
+      "integrity": "sha512-07ny+LHRzQXepkGg6w0mFY41fVUNBrL2Roj/++7V1txKugfjm/Ci/qSND03r2RhlJhJYMcTn9AhhSSqQp0Ysyw==",
       "dev": true,
       "dependencies": {
-        "@typescript-eslint/types": "5.59.11",
+        "@typescript-eslint/types": "5.62.0",
         "eslint-visitor-keys": "^3.3.0"
       },
       "engines": {
@@ -32654,25 +32654,25 @@
       }
     },
     "@typescript-eslint/parser": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.59.11.tgz",
-      "integrity": "sha512-s9ZF3M+Nym6CAZEkJJeO2TFHHDsKAM3ecNkLuH4i4s8/RCPnF5JRip2GyviYkeEAcwGMJxkqG9h2dAsnA1nZpA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.62.0.tgz",
+      "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
       "dev": true,
       "requires": {
-        "@typescript-eslint/scope-manager": "5.59.11",
-        "@typescript-eslint/types": "5.59.11",
-        "@typescript-eslint/typescript-estree": "5.59.11",
+        "@typescript-eslint/scope-manager": "5.62.0",
+        "@typescript-eslint/types": "5.62.0",
+        "@typescript-eslint/typescript-estree": "5.62.0",
         "debug": "^4.3.4"
       }
     },
     "@typescript-eslint/scope-manager": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.59.11.tgz",
-      "integrity": "sha512-dHFOsxoLFtrIcSj5h0QoBT/89hxQONwmn3FOQ0GOQcLOOXm+MIrS8zEAhs4tWl5MraxCY3ZJpaXQQdFMc2Tu+Q==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.62.0.tgz",
+      "integrity": "sha512-VXuvVvZeQCQb5Zgf4HAxc04q5j+WrNAtNh9OwCsCgpKqESMTu3tF/jhZ3xG6T4NZwWl65Bg8KuS2uEvhSfLl0w==",
       "dev": true,
       "requires": {
-        "@typescript-eslint/types": "5.59.11",
-        "@typescript-eslint/visitor-keys": "5.59.11"
+        "@typescript-eslint/types": "5.62.0",
+        "@typescript-eslint/visitor-keys": "5.62.0"
       }
     },
     "@typescript-eslint/type-utils": {
@@ -32721,19 +32721,19 @@
       }
     },
     "@typescript-eslint/types": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.59.11.tgz",
-      "integrity": "sha512-epoN6R6tkvBYSc+cllrz+c2sOFWkbisJZWkOE+y3xHtvYaOE6Wk6B8e114McRJwFRjGvYdJwLXQH5c9osME/AA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.62.0.tgz",
+      "integrity": "sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ==",
       "dev": true
     },
     "@typescript-eslint/typescript-estree": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.59.11.tgz",
-      "integrity": "sha512-YupOpot5hJO0maupJXixi6l5ETdrITxeo5eBOeuV7RSKgYdU3G5cxO49/9WRnJq9EMrB7AuTSLH/bqOsXi7wPA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.62.0.tgz",
+      "integrity": "sha512-CmcQ6uY7b9y694lKdRB8FEel7JbU/40iSAPomu++SjLMntB+2Leay2LO6i8VnJk58MtE9/nQSFIH6jpyRWyYzA==",
       "dev": true,
       "requires": {
-        "@typescript-eslint/types": "5.59.11",
-        "@typescript-eslint/visitor-keys": "5.59.11",
+        "@typescript-eslint/types": "5.62.0",
+        "@typescript-eslint/visitor-keys": "5.62.0",
         "debug": "^4.3.4",
         "globby": "^11.1.0",
         "is-glob": "^4.0.3",
@@ -32817,12 +32817,12 @@
       }
     },
     "@typescript-eslint/visitor-keys": {
-      "version": "5.59.11",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.59.11.tgz",
-      "integrity": "sha512-KGYniTGG3AMTuKF9QBD7EIrvufkB6O6uX3knP73xbKLMpH+QRPcgnCxjWXSHjMRuOxFLovljqQgQpR0c7GvjoA==",
+      "version": "5.62.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.62.0.tgz",
+      "integrity": "sha512-07ny+LHRzQXepkGg6w0mFY41fVUNBrL2Roj/++7V1txKugfjm/Ci/qSND03r2RhlJhJYMcTn9AhhSSqQp0Ysyw==",
       "dev": true,
       "requires": {
-        "@typescript-eslint/types": "5.59.11",
+        "@typescript-eslint/types": "5.62.0",
         "eslint-visitor-keys": "^3.3.0"
       }
     },
diff --git a/ui/package.json b/ui/package.json
index 10337b17fb..7787bee97a 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -110,7 +110,7 @@
     "@types/rx": "^4.1.2",
     "@types/showdown": "1.9.4",
     "@typescript-eslint/eslint-plugin": "5.57.1",
-    "@typescript-eslint/parser": "5.59.11",
+    "@typescript-eslint/parser": "5.62.0",
     "ajv-keywords": "^5.1.0",
     "assert": "^2.0.0",
     "compression-webpack-plugin": "^10.0.0",


Test patch:
diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java
new file mode 100644
index 0000000000..45d452dd0c
--- /dev/null
+++ b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.dataexplorer.commons.sanitizer;
+
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+public class MeasureNameSanitizerTest {
+
+  @Test
+  public void testSanitizedMeasure() {
+    String originalMeasureName = "test/001";
+    String expected = "test_001";
+    String sanitizedMeasureName = new MeasureNameSanitizer().sanitize(originalMeasureName);
+
+    assertEquals(expected, sanitizedMeasureName);
+  }
+
+  @Test
+  public void testOriginalMeasure() {
+    String originalMeasureName = "test_001";
+    String expected = "test_001";
+    String sanitizedMeasureName = new MeasureNameSanitizer().sanitize(originalMeasureName);
+
+    assertEquals(expected, sanitizedMeasureName);
+  }
+
+  @Test
+  public void testMultipleCharacters() {
+    String originalMeasureName = "t?\"est_001=";
+    String expected = "t__est_001_";
+    String sanitizedMeasureName = new MeasureNameSanitizer().sanitize(originalMeasureName);
+
+    assertEquals(expected, sanitizedMeasureName);
+  }
+}