Repository: apache/streampipes
Pull request #3271 (2024-09-30 19:40:23)

fix(#3237): Remove selector prefix from output event
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  ~
  -->

  <!--
Thanks for contributing! Here are some tips you can follow to help us incorporate your contribution quickly and easily:
1. If this is your first time, please read our contributor guidelines:
    - https://streampipes.apache.org/community/get-involved/
    - https://cwiki.apache.org/confluence/display/STREAMPIPES/Getting+Started
2. Make sure the PR title is formatted like: `[#<GitHub issue id>] PR title ...`
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., `[WIP][#<GitHub issue id>] PR title ...`.
4. Please write your PR title to summarize what this PR proposes/fixes.
5. Link the PR to the corresponding GitHub issue (if present) in the `Development` section in the right menu bar.
6. Be sure to keep the PR description updated to reflect all changes.
7. If possible, provide a concise example to reproduce the issue for a faster review.
8. Make sure tests pass via `mvn clean install`.
9. (Optional) If the contribution is large, please file an Apache ICLA
    - http://apache.org/licenses/icla.pdf
-->

### Purpose
<!--
Please clarify what changes you are proposing and describe how those changes will address the issue.
Furthermore, describe potential consequences the changes might have.
-->

Ensure that the property selector prefix is removed before adding the output field to the `TransformBooleanProcessor`.

### Remarks
<!--
Is there anything left we need to pay attention on?
Are there some references that might be important? E.g. links to Confluence, or discussions
on the mailing list or GitHub.
-->
PR introduces (a) breaking change(s): no

PR introduces (a) deprecation(s): no



Patch:
diff --git a/pom.xml b/pom.xml
index 002b222bd9..c1285c0a95 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,7 +48,7 @@
         <commons-codec.version>1.17.0</commons-codec.version>
         <commons-collections.version>3.2.2</commons-collections.version>
         <commons-compress.version>1.27.1</commons-compress.version>
-        <commons-io.version>2.16.1</commons-io.version>
+        <commons-io.version>2.17.0</commons-io.version>
         <commons-lang3.version>3.14.0</commons-lang3.version>
         <commons-logging.version>1.3.2</commons-logging.version>
         <commons-pool2.version>2.12.0</commons-pool2.version>
diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java
index 316aa9c375..fff8f8283d 100644
--- a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java
+++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java
@@ -85,11 +85,7 @@ public class TransformToBooleanProcessor
     EventSchema eventSchema = new EventSchema();
     EventSchema oldEventSchema = processingElement.getInputStreams().get(0).getEventSchema();
     // Gotta remove the "s0::" in the beginning
-    Set<String> transformFields =
-        (parameterExtractor.mappingPropertyValues(TRANSFORM_FIELDS_ID))
-            .stream()
-            .map(s -> s.substring(4))
-            .collect(Collectors.toSet());
+    Set<String> transformFields = getTransformFieldNames(parameterExtractor);
 
     for (EventProperty eventProperty : oldEventSchema.getEventProperties()) {
       //TODO: Test, if eventProperty is a primitive type (string, number, ...)
@@ -122,13 +118,13 @@ public class TransformToBooleanProcessor
   @Override
   public void onEvent(Event inputEvent, SpOutputCollector collector) throws SpRuntimeException {
     for (String transformField : transformFields) {
-      AbstractField field = inputEvent.getFieldBySelector(transformField);
+      AbstractField<?> field = inputEvent.getFieldBySelector(transformField);
       // Is the field a primitive (and no list/nested field)?
       if (field.isPrimitive()) {
         // Yes. So remove the element and replace it with a boolean (if possible)
         inputEvent.removeFieldBySelector(transformField);
         try {
-          inputEvent.addField(transformField, toBoolean(field.getRawValue()));
+          inputEvent.addField(removeFieldPrefix(transformField), toBoolean(field.getRawValue()));
         } catch (SpRuntimeException e) {
           LOG.info(e.getMessage());
           return;
@@ -154,4 +150,15 @@ public class TransformToBooleanProcessor
       throw new SpRuntimeException("Value " + s + " not convertible to boolean");
     }
   }
+
+  private Set<String> getTransformFieldNames(ProcessingElementParameterExtractor extractor) {
+    return extractor.mappingPropertyValues(TRANSFORM_FIELDS_ID)
+        .stream()
+        .map(this::removeFieldPrefix)
+        .collect(Collectors.toSet());
+  }
+
+  private String removeFieldPrefix(String fieldName) {
+    return fieldName.substring(4);
+  }
 }
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
index 006f3f955d..c5ff418e82 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.model.message.Notifications;
 import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
 import org.apache.streampipes.rest.security.AuthConstants;
 import org.apache.streampipes.rest.shared.exception.SpMessageException;
+import org.apache.streampipes.sdk.helpers.Filetypes;
 
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -72,6 +73,13 @@ public class PipelineElementFile extends AbstractAuthGuardedRestResource {
               fileDetail.getInputStream()
           );
       return ok(metadata);
+    } catch (IllegalArgumentException e) {
+      return badRequest(Notifications.error(
+          String.format(
+              "This file type is not supported. Allowed file types are %s.",
+              Filetypes.getAllFileExtensions().toString()
+          )
+      ));
     } catch (Exception e) {
       return fail();
     }
@@ -132,10 +140,10 @@ public class PipelineElementFile extends AbstractAuthGuardedRestResource {
   @PreAuthorize(AuthConstants.HAS_READ_FILE_PRIVILEGE)
   public ResponseEntity<List<String>> getAllOriginalFilenames() {
     return ok(fileManager.getAllFiles()
-                         .stream()
-                         .map(fileMetadata -> fileMetadata.getFilename()
-                                                          .toLowerCase())
-                         .toList());
+        .stream()
+        .map(fileMetadata -> fileMetadata.getFilename()
+            .toLowerCase())
+        .toList());
   }
 
   @GetMapping(
diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java
index 681b180983..b8895a9b1f 100644
--- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java
+++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java
@@ -29,6 +29,7 @@ public enum Filetypes {
   XLS("xls"),
   XLSX("xlsx"),
   XML("xml"),
+  YAML("yaml", "yml"),
   ZIP("zip");
 
   private final List<String> fileExtensions;
diff --git a/ui/src/app/configuration/configuration.module.ts b/ui/src/app/configuration/configuration.module.ts
index f24da2f959..ff860383f5 100644
--- a/ui/src/app/configuration/configuration.module.ts
+++ b/ui/src/app/configuration/configuration.module.ts
@@ -96,6 +96,7 @@ import { FileOverviewComponent } from './files/file-overview/file-overview.compo
 import { FileUploadDialogComponent } from './dialog/file-upload/file-upload-dialog.component';
 import { FileRenameDialogComponent } from './dialog/file-rename/file-rename-dialog.component';
 import { MatDialogModule } from '@angular/material/dialog';
+import { MatProgressBarModule } from '@angular/material/progress-bar';
 
 @NgModule({
     imports: [
@@ -120,6 +121,7 @@ import { MatDialogModule } from '@angular/material/dialog';
         CoreUiModule,
         ReactiveFormsModule,
         PlatformServicesModule,
+        MatProgressBarModule,
         RouterModule.forChild([
             {
                 path: '',
diff --git a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html
index 6d83e5a5c6..ecec2cccdd 100644
--- a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html
+++ b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html
@@ -17,7 +17,10 @@
 -->
 
 <div class="sp-dialog-container">
-    <div class="sp-dialog-content p-15">
+    <div class="sp-dialog-content p-15" fxLayout="column">
+        <sp-warning-box *ngIf="uploadError" style="font-size: smaller">
+            {{ uploadErrorMessage }}
+        </sp-warning-box>
         <div fxFlex="100" *ngIf="this.duplicateFileNames.length === 0">
             <div fxFlex="100" style="margin: 5px; width: 100%">
                 <mat-form-field
@@ -57,7 +60,7 @@
                         color="accent"
                         matSuffix
                         mat-button
-                        style="min-width: 0px"
+                        style="min-width: 0"
                     >
                         <mat-icon *ngIf="uploadStatus < 99"
                             >insert_drive_file</mat-icon
diff --git a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts
index 328cfe8494..beee8317b9 100644
--- a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts
+++ b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts
@@ -39,6 +39,9 @@ export class FileUploadDialogComponent {
 
     uploadStatus = 0;
 
+    uploadError = false;
+    uploadErrorMessage = '';
+
     constructor(
         private dialogRef: DialogRef<FileUploadDialogComponent>,
         private filesService: FilesService,
@@ -73,6 +76,7 @@ export class FileUploadDialogComponent {
     }
 
     uploadFile(index: number): void {
+        this.uploadError = false;
         this.filesService
             .uploadFile(this.selectedUploadFiles.item(index))
             .subscribe(
@@ -90,7 +94,15 @@ export class FileUploadDialogComponent {
                         }
                     }
                 },
-                error => {},
+                error => {
+                    this.uploadError = true;
+                    if (error.error?.notifications?.length > 0) {
+                        this.uploadErrorMessage =
+                            error.error.notifications[0].title;
+                    } else {
+                        this.uploadErrorMessage = error.error;
+                    }
+                },
             );
     }
 
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
index 56c283b6c0..6ff8ea6a14 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
@@ -62,10 +62,18 @@
                     name="pipelineName"
                     (blur)="triggerTutorial()"
                 />
+                <mat-error
+                    >Pipeline name must have between 3 and 40
+                    characters.</mat-error
+                >
             </mat-form-field>
             <mat-form-field fxFlex color="accent">
                 <mat-label>Description</mat-label>
                 <input [formControlName]="'pipelineDescription'" matInput />
+                <mat-error
+                    >Pipeline description must not have more than 80
+                    characters.</mat-error
+                >
             </mat-form-field>
         </div>
     </form>
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
index 6f32859192..d75b2c1aed 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
@@ -51,6 +51,7 @@ export class SavePipelineSettingsComponent implements OnInit {
             'pipelineName',
             new UntypedFormControl(this.pipeline.name, [
                 Validators.required,
+                Validators.minLength(3),
                 Validators.maxLength(40),
             ]),
         );
diff --git a/ui/src/app/files/files.module.ts b/ui/src/app/files/files.module.ts
deleted file mode 100644
index 991569618d..0000000000
--- a/ui/src/app/files/files.module.ts
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-import { NgModule } from '@angular/core';
-import { CommonModule } from '@angular/common';
-import { FlexLayoutModule } from '@ngbracket/ngx-layout';
-import { MatButtonModule } from '@angular/material/button';
-import { MatGridListModule } from '@angular/material/grid-list';
-import { MatIconModule } from '@angular/material/icon';
-import { MatDividerModule } from '@angular/material/divider';
-import { MatListModule } from '@angular/material/list';
-import { FilesComponent } from '../configuration/files/files.component';
-import { MatTabsModule } from '@angular/material/tabs';
-import { CoreUiModule } from '../core-ui/core-ui.module';
-import { FormsModule, ReactiveFormsModule } from '@angular/forms';
-import { MatFormFieldModule } from '@angular/material/form-field';
-import { MatProgressBarModule } from '@angular/material/progress-bar';
-import { MatInputModule } from '@angular/material/input';
-import { ServicesModule } from '../services/services.module';
-import { MatTableModule } from '@angular/material/table';
-import { MatPaginatorModule } from '@angular/material/paginator';
-import { MatChipsModule } from '@angular/material/chips';
-import { MatTooltipModule } from '@angular/material/tooltip';
-import { PlatformServicesModule } from '@streampipes/platform-services';
-import { RouterModule } from '@angular/router';
-import { SharedUiModule } from '@streampipes/shared-ui';
-import { MatDialogModule } from '@angular/material/dialog';
-
-@NgModule({
-    imports: [
-        CommonModule,
-        CoreUiModule,
-        FlexLayoutModule,
-        FormsModule,
-        ReactiveFormsModule,
-        MatButtonModule,
-        MatChipsModule,
-        MatFormFieldModule,
-        MatGridListModule,
-        MatIconModule,
-        MatInputModule,
-        MatDividerModule,
-        MatListModule,
-        MatPaginatorModule,
-        MatProgressBarModule,
-        MatTableModule,
-        MatTabsModule,
-        MatTooltipModule,
-        PlatformServicesModule,
-        ServicesModule,
-        SharedUiModule,
-        RouterModule.forChild([
-            {
-                path: '',
-                children: [
-                    {
-                        path: '',
-                        component: FilesComponent,
-                    },
-                ],
-            },
-        ]),
-        MatDialogModule,
-    ],
-    declarations: [],
-    providers: [],
-})
-export class FilesModule {}
diff --git a/ui/src/app/files/files.routes.ts b/ui/src/app/files/files.routes.ts
deleted file mode 100644
index be69607a13..0000000000
--- a/ui/src/app/files/files.routes.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-import { SpBreadcrumbItem } from '@streampipes/shared-ui';
-
-export class SpFilesRoutes {
-    static FILES_BASE_LINK = 'files';
-    static BASE: SpBreadcrumbItem = {
-        label: 'File Management',
-        link: [SpFilesRoutes.FILES_BASE_LINK],
-    };
-}


Test patch:
diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/test/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessorTest.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/test/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessorTest.java
new file mode 100644
index 0000000000..b93127a352
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/test/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessorTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.processors.transformation.jvm.processor.transformtoboolean;
+
+import org.apache.streampipes.test.executors.PrefixStrategy;
+import org.apache.streampipes.test.executors.ProcessingElementTestExecutor;
+import org.apache.streampipes.test.executors.TestConfiguration;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+public class TransformToBooleanProcessorTest {
+
+  @Test
+  void onEvent() {
+
+    var processor = new TransformToBooleanProcessor();
+
+    List<Map<String, Object>> inputEvents = List.of(Map.of(
+        "numberBoolean", 0,
+        "stringBoolean", "true",
+        "stringBooleanFalse", "false",
+        "floatBoolean", 1.0
+    ));
+
+    List<Map<String, Object>> outputEvents = List.of(Map.of(
+        "numberBoolean", false,
+        "stringBoolean", true,
+        "stringBooleanFalse", false,
+        "floatBoolean", true
+    ));
+
+    var configuration = TestConfiguration
+        .builder()
+        .config(TransformToBooleanProcessor.TRANSFORM_FIELDS_ID, List.of(
+            "s0::numberBoolean", "s0::stringBoolean", "s0::stringBooleanFalse", "s0::floatBoolean"))
+        .prefixStrategy(PrefixStrategy.SAME_PREFIX)
+        .build();
+
+    var testExecutor = new ProcessingElementTestExecutor(processor, configuration);
+
+    testExecutor.run(inputEvents, outputEvents);
+  }
+}
diff --git a/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java b/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java
index 8c287f2cbc..2e0204fb53 100644
--- a/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java
+++ b/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java
@@ -29,9 +29,9 @@ class FiletypesTest {
   public void getAllFileExtensions_returnsAllExtensions() {
     var actualExtensions = Filetypes.getAllFileExtensions();
 
-    assertEquals(8, actualExtensions.size());
+    assertEquals(10, actualExtensions.size());
     assertTrue(actualExtensions.contains("jpg"));
     assertTrue(actualExtensions.contains("jpeg"));
     assertTrue(actualExtensions.contains("csv"));
   }
-}
\ No newline at end of file
+}