Repository: apache/streampipes
Pull request #2888 (2024-05-21 15:00:20)

fix(#2830): Data adapter losing dimension selection
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  ~
  -->
  
  <!--
Thanks for contributing! Here are some tips you can follow to help us incorporate your contribution quickly and easily:
1. If this is your first time, please read our contributor guidelines:
    - https://streampipes.apache.org/getinvolved.html
    - https://cwiki.apache.org/confluence/display/STREAMPIPES/Getting+Started
2. Make sure the PR title is formatted like: `[#<GitHub issue id>] PR title ...`
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., `[WIP][#<GitHub issue id>] PR title ...`.
4. Please write your PR title to summarize what this PR proposes/fixes.
5. Link the PR to the corresponding GitHub issue (if present) in the `Development` section in the right menu bar. 
6. Be sure to keep the PR description updated to reflect all changes.
7. If possible, provide a concise example to reproduce the issue for a faster review.
8. Make sure tests pass via `mvn clean install`.
9. (Optional) If the contribution is large, please file an Apache ICLA
    - http://apache.org/licenses/icla.pdf
-->

### Purpose
<!--
Please clarify what changes you are proposing and describe how those changes will address the issue.
Furthermore, describe potential consequences the changes might have.
-->

### Remarks
<!--
Is there anything left we need to pay attention on?
Are there some references that might be important? E.g. links to Confluence, or discussions
on the mailing list or GitHub.
-->
PR introduces (a) breaking change(s): <yes/no>

PR introduces (a) deprecation(s): <yes/no>



Patch:
diff --git a/pom.xml b/pom.xml
index 6e1f33d663..9a713f93ea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,7 +74,6 @@
         <jakarta-activation-api.version>2.1.3</jakarta-activation-api.version>
         <jakarta-inject-api.version>2.0.1</jakarta-inject-api.version>
         <jakarta-servlet-api.version>6.0.0</jakarta-servlet-api.version>
-        <jakarta-validation.version>3.0.2</jakarta-validation.version>
         <jakarta-xml-bind-api.version>4.0.0</jakarta-xml-bind-api.version>
         <javax.xml.bind.version>2.4.0-b180725.0427</javax.xml.bind.version>
         <java-websocket.version>1.5.0</java-websocket.version>
@@ -328,11 +327,6 @@
                 <artifactId>jakarta.inject-api</artifactId>
                 <version>${jakarta-inject-api.version}</version>
             </dependency>
-            <dependency>
-                <groupId>jakarta.validation</groupId>
-                <artifactId>jakarta.validation-api</artifactId>
-                <version>${jakarta-validation.version}</version>
-            </dependency>
             <dependency>
                 <groupId>jakarta.servlet</groupId>
                 <artifactId>jakarta.servlet-api</artifactId>
diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java
index d332a79e42..a0494cff34 100644
--- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java
+++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java
@@ -42,7 +42,10 @@ public abstract class SupportsNestedTransformationRule implements Transformation
           applyNested((Map<String, Object>) event.get(eventKey.get(0)), newKeysTmpList);
 
       event.remove(key);
-      event.put(key, newSubEvent);
+
+      if (newSubEvent != null && !newSubEvent.isEmpty()) {
+        event.put(key, newSubEvent);
+      }
     }
     return event;
   }
diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java
index 60f282abd4..92b1674d0d 100644
--- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java
+++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java
@@ -109,6 +109,10 @@ public class ToTransformedSchemaConverter implements ITransformationRuleVisitor,
   public void visit(AddTimestampRuleDescription rule) {
     var timestampProperty = EpProperties.timestampProperty(rule.getRuntimeKey());
     timestampProperty.setElementId(TIMESTAMP_ID_PREFIX + UUIDGenerator.generateUuid());
+    // null check required for backwards compatibility
+    if (rule.getPropertyScope() != null) {
+      timestampProperty.setPropertyScope(rule.getPropertyScope().name());
+    }
     this.properties.add(timestampProperty);
   }
 
diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java
index ae2d1c70d9..7fe7805baa 100644
--- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java
+++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java
@@ -38,6 +38,8 @@ public class DeleteTransformationRule extends SupportsNestedTransformationRule {
 
   @Override
   protected void applyTransformation(Map<String, Object> event, List<String> eventKeys) {
+    if (event != null) {
       event.remove(eventKeys.get(0));
+    }
   }
 }
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java
index 379ac3b90b..1df522dbd0 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/AddTimestampRuleDescription.java
@@ -20,10 +20,12 @@ package org.apache.streampipes.model.connect.rules.value;
 
 import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor;
 import org.apache.streampipes.model.connect.rules.TransformationRulePriority;
+import org.apache.streampipes.model.schema.PropertyScope;
 
 public class AddTimestampRuleDescription extends ValueTransformationRuleDescription {
 
   private String runtimeKey;
+  private PropertyScope propertyScope;
 
   public AddTimestampRuleDescription() {
     super();
@@ -36,6 +38,7 @@ public class AddTimestampRuleDescription extends ValueTransformationRuleDescript
   public AddTimestampRuleDescription(AddTimestampRuleDescription other) {
     super(other);
     this.runtimeKey = other.getRuntimeKey();
+    this.propertyScope = other.getPropertyScope();
   }
 
   public String getRuntimeKey() {
@@ -46,6 +49,14 @@ public class AddTimestampRuleDescription extends ValueTransformationRuleDescript
     this.runtimeKey = runtimeKey;
   }
 
+  public PropertyScope getPropertyScope() {
+    return propertyScope;
+  }
+
+  public void setPropertyScope(PropertyScope propertyScope) {
+    this.propertyScope = propertyScope;
+  }
+
   @Override
   public void accept(ITransformationRuleVisitor visitor) {
     visitor.visit(this);
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
index b0de88590b..1cd4d5a729 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
@@ -20,7 +20,7 @@
 /* tslint:disable */
 /* eslint-disable */
 // @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2024-04-30 20:29:03.
+// Generated using typescript-generator version 3.2.1263 on 2024-05-21 13:45:10.
 
 export class NamedStreamPipesEntity {
     '@class':
@@ -325,6 +325,7 @@ export class ValueTransformationRuleDescription extends TransformationRuleDescri
 
 export class AddTimestampRuleDescription extends ValueTransformationRuleDescription {
     '@class': 'org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription';
+    'propertyScope': PropertyScope;
     'runtimeKey': string;
 
     static 'fromData'(
@@ -336,6 +337,7 @@ export class AddTimestampRuleDescription extends ValueTransformationRuleDescript
         }
         const instance = target || new AddTimestampRuleDescription();
         super.fromData(data, instance);
+        instance.propertyScope = data.propertyScope;
         instance.runtimeKey = data.runtimeKey;
         return instance;
     }
diff --git a/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.html b/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.html
index 5f9b8ab1e0..04493f10e3 100644
--- a/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.html
+++ b/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.html
@@ -30,13 +30,17 @@
                     <h5 *ngIf="messageTimestamp" class="mr-15">
                         {{ messageTimestamp | date: 'short' }}
                     </h5>
-                    <h5 fxFlex class="color-warn">
+                    <h5 fxFlex>
                         {{ message.title || 'Error' }}
                     </h5>
                 </div>
                 <span fxFlex></span>
                 <div fxLayoutAlign="end center" *ngIf="showDetails">
-                    <button mat-button (click)="openDetailsDialog()">
+                    <button
+                        mat-button
+                        color="accent"
+                        (click)="openDetailsDialog()"
+                    >
                         <i class="material-icons">visibility</i>&nbsp;Details
                     </button>
                 </div>
diff --git a/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.scss b/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.scss
index 26170c62ba..165ce8fb0f 100644
--- a/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.scss
+++ b/ui/projects/streampipes/shared-ui/src/lib/components/sp-exception-message/sp-exception-message.component.scss
@@ -18,7 +18,6 @@
 
 .error-panel {
     background: var(--color-bg-1);
-    border-radius: 5px;
 }
 
 .error {
diff --git a/ui/src/app/connect/services/transformation-rule.service.ts b/ui/src/app/connect/services/transformation-rule.service.ts
index b0fd79fa3d..443abebab9 100644
--- a/ui/src/app/connect/services/transformation-rule.service.ts
+++ b/ui/src/app/connect/services/transformation-rule.service.ts
@@ -30,6 +30,7 @@ import {
     EventPropertyUnion,
     EventSchema,
     MoveRuleDescription,
+    PropertyScope,
     RenameRuleDescription,
     SemanticType,
     TimestampTranfsformationRuleDescription,
@@ -55,22 +56,23 @@ export class TransformationRuleService {
         if (originalSchema == null || targetSchema == null) {
             console.log('Old and new schema must be defined');
         } else {
-            const addedTimestampProperties = this.getTimestampProperty(
+            const addedTimestampProperty = this.getTimestampProperty(
                 targetSchema.eventProperties,
             );
-            if (addedTimestampProperties) {
+            if (addedTimestampProperty) {
                 // add to old event schema for the case users moved the property to a nested property
-                originalSchema.eventProperties.push(addedTimestampProperties);
+                originalSchema.eventProperties.push(addedTimestampProperty);
 
                 const timestampRuleDescription: AddTimestampRuleDescription =
                     new AddTimestampRuleDescription();
                 timestampRuleDescription['@class'] =
                     'org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription';
                 timestampRuleDescription.runtimeKey =
-                    addedTimestampProperties.runtimeName;
+                    addedTimestampProperty.runtimeName;
+                timestampRuleDescription.propertyScope =
+                    addedTimestampProperty.propertyScope as PropertyScope;
                 transformationRuleDescriptions.push(timestampRuleDescription);
             }
-
             const staticValueProperties = this.getStaticValueProperties(
                 targetSchema.eventProperties,
             );
@@ -84,6 +86,7 @@ export class TransformationRuleService {
                 rule.datatype = ep.runtimeType;
                 rule.label = ep.label;
                 rule.description = ep.description;
+                rule.propertyScope = ep.propertyScope as PropertyScope;
                 if (ep.domainProperties.length > 0) {
                     rule.semanticType = ep.domainProperties[0];
                 }


Test patch:
diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java
index 7253b33bd5..e8d5e1feb8 100644
--- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java
+++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java
@@ -44,13 +44,39 @@ public class DeleteTransformationRuleTest {
   public void transformNested() {
     var child = new HashMap<String, Object>();
     child.put("child", "value");
+    child.put("secondChild", "value");
     var event = new HashMap<String, Object>();
     event.put("parent", child);
+    event.put("keepProperty", "test");
+
+    var deleteRule = new DeleteTransformationRule(List.of("parent", "child"));
+
+    var result = deleteRule.apply(event);
+
+    assertEquals(2, result.keySet().size());
+  }
+
+  @Test
+  // verifying that mathod applyTransformation method works when passed a null event.
+  public void applyTransformationWithNullParameter() {
+    new DeleteTransformationRule(List.of())
+        .applyTransformation(null, List.of());
+  }
+
+  @Test
+  public void deleteNestedChildWithParentProperty() {
+    var child = new HashMap<String, Object>();
+    child.put("child", "value");
+    var event = new HashMap<String, Object>();
+    event.put("parent", child);
+    event.put("keepProperty", "test");
 
     var deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child"));
 
     var result = deleteRule.apply(event);
 
     assertEquals(1, result.keySet().size());
+    assertEquals("test", result.get("keepProperty"));
   }
+
 }
diff --git a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
index debbc5473f..2ece7e9a38 100644
--- a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
+++ b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
@@ -19,6 +19,7 @@ import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
 import { FileManagementUtils } from '../../../support/utils/FileManagementUtils';
 import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils';
 import { AdapterBuilder } from '../../../support/builder/AdapterBuilder';
+import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns';
 
 describe('Connect delete rule transformation', () => {
     beforeEach('Setup Test', () => {
@@ -72,5 +73,15 @@ describe('Connect delete rule transformation', () => {
             'have.text',
             '{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
         );
+
+        ConnectBtns.refreshSchema().click();
+
+        // Test to delete the parent property
+        ConnectEventSchemaUtils.deleteProperty('parent');
+        cy.dataCy('schema-preview-result-event').should(
+            'have.text',
+
+            '{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}',
+        );
     });
 });