Repository: apache/streampipes
Pull request #3267 (2024-09-30 09:36:25)

feat(#3266): Add API to programmatically create pipelines
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  ~
  -->

  <!--
Thanks for contributing! Here are some tips you can follow to help us incorporate your contribution quickly and easily:
1. If this is your first time, please read our contributor guidelines:
    - https://streampipes.apache.org/community/get-involved/
    - https://cwiki.apache.org/confluence/display/STREAMPIPES/Getting+Started
2. Make sure the PR title is formatted like: `[#<GitHub issue id>] PR title ...`
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., `[WIP][#<GitHub issue id>] PR title ...`.
4. Please write your PR title to summarize what this PR proposes/fixes.
5. Link the PR to the corresponding GitHub issue (if present) in the `Development` section in the right menu bar.
6. Be sure to keep the PR description updated to reflect all changes.
7. If possible, provide a concise example to reproduce the issue for a faster review.
8. Make sure tests pass via `mvn clean install`.
9. (Optional) If the contribution is large, please file an Apache ICLA
    - http://apache.org/licenses/icla.pdf
-->

### Purpose
<!--
Please clarify what changes you are proposing and describe how those changes will address the issue.
Furthermore, describe potential consequences the changes might have.
-->

See #3266 

Example: 

```yaml
name: "My pipeline"
id: mypipeline
description: Description
pipelineElements:
  - type: stream
    ref: mds
    id: sp:spdatastream:iAPZYK
  - type: processor
    ref: filter
    id: org.apache.streampipes.processors.filters.jvm.numericalfilter
    configuration:
      - operation: ">"
        value: 12
    connectedTo:
      - mds
  - type: sink
    ref: notification
    id: org.apache.streampipes.sinks.internal.jvm.notification
    connectedTo:
      - filter
    configuration:
      - title: Alarm
        content: "<b>ABC</b>"
        silentPeriod: 10
createOptions:
  start: true
```

### Remarks
<!--
Is there anything left we need to pay attention on?
Are there some references that might be important? E.g. links to Confluence, or discussions
on the mailing list or GitHub.
-->
PR introduces (a) breaking change(s):no
PR introduces (a) deprecation(s): no



Patch:
diff --git a/pom.xml b/pom.xml
index 002b222bd9..c1285c0a95 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,7 +48,7 @@
         <commons-codec.version>1.17.0</commons-codec.version>
         <commons-collections.version>3.2.2</commons-collections.version>
         <commons-compress.version>1.27.1</commons-compress.version>
-        <commons-io.version>2.16.1</commons-io.version>
+        <commons-io.version>2.17.0</commons-io.version>
         <commons-lang3.version>3.14.0</commons-lang3.version>
         <commons-logging.version>1.3.2</commons-logging.version>
         <commons-pool2.version>2.12.0</commons-pool2.version>
diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java
index 9aae62888d..b64b661495 100644
--- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java
+++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java
@@ -84,7 +84,7 @@ public class AdapterUpdateManagement {
       try {
         var modificationMessage = new PipelineVerificationHandlerV2(pipeline).verifyPipeline();
         var updateInfo = makeUpdateInfo(modificationMessage, pipeline);
-        var modifiedPipeline = new PipelineVerificationHandlerV2(pipeline).makeModifiedPipeline();
+        var modifiedPipeline = new PipelineVerificationHandlerV2(pipeline).makeModifiedPipeline().pipeline();
         var canAutoMigrate = canAutoMigrate(modificationMessage);
         if (!canAutoMigrate) {
           modifiedPipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION);
diff --git a/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java b/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java
index f0b49c1831..d35b598427 100644
--- a/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java
+++ b/streampipes-extensions/streampipes-extensions-all-iiot/src/main/java/org/apache/streampipes/extensions/all/iiot/AllExtensionsIIoTInit.java
@@ -48,6 +48,9 @@ import org.apache.streampipes.sinks.internal.jvm.InternalSinksExtensionModuleExp
 import org.apache.streampipes.sinks.notifications.jvm.NotificationsExtensionModuleExport;
 import org.apache.streampipes.wrapper.standalone.runtime.StandaloneStreamPipesRuntimeProvider;
 
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
 public class AllExtensionsIIoTInit extends StreamPipesExtensionsServiceBase {
 
   public static void main(String[] args) {
diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java
index 316aa9c375..fff8f8283d 100644
--- a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java
+++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessor.java
@@ -85,11 +85,7 @@ public class TransformToBooleanProcessor
     EventSchema eventSchema = new EventSchema();
     EventSchema oldEventSchema = processingElement.getInputStreams().get(0).getEventSchema();
     // Gotta remove the "s0::" in the beginning
-    Set<String> transformFields =
-        (parameterExtractor.mappingPropertyValues(TRANSFORM_FIELDS_ID))
-            .stream()
-            .map(s -> s.substring(4))
-            .collect(Collectors.toSet());
+    Set<String> transformFields = getTransformFieldNames(parameterExtractor);
 
     for (EventProperty eventProperty : oldEventSchema.getEventProperties()) {
       //TODO: Test, if eventProperty is a primitive type (string, number, ...)
@@ -122,13 +118,13 @@ public class TransformToBooleanProcessor
   @Override
   public void onEvent(Event inputEvent, SpOutputCollector collector) throws SpRuntimeException {
     for (String transformField : transformFields) {
-      AbstractField field = inputEvent.getFieldBySelector(transformField);
+      AbstractField<?> field = inputEvent.getFieldBySelector(transformField);
       // Is the field a primitive (and no list/nested field)?
       if (field.isPrimitive()) {
         // Yes. So remove the element and replace it with a boolean (if possible)
         inputEvent.removeFieldBySelector(transformField);
         try {
-          inputEvent.addField(transformField, toBoolean(field.getRawValue()));
+          inputEvent.addField(removeFieldPrefix(transformField), toBoolean(field.getRawValue()));
         } catch (SpRuntimeException e) {
           LOG.info(e.getMessage());
           return;
@@ -154,4 +150,15 @@ public class TransformToBooleanProcessor
       throw new SpRuntimeException("Value " + s + " not convertible to boolean");
     }
   }
+
+  private Set<String> getTransformFieldNames(ProcessingElementParameterExtractor extractor) {
+    return extractor.mappingPropertyValues(TRANSFORM_FIELDS_ID)
+        .stream()
+        .map(this::removeFieldPrefix)
+        .collect(Collectors.toSet());
+  }
+
+  private String removeFieldPrefix(String fieldName) {
+    return fieldName.substring(4);
+  }
 }
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/ExtendedPipelineElementValidationInfo.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/ExtendedPipelineElementValidationInfo.java
new file mode 100644
index 0000000000..9289b3de12
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/ExtendedPipelineElementValidationInfo.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.pipeline;
+
+public class ExtendedPipelineElementValidationInfo extends PipelineElementValidationInfo {
+
+  private final String pipelineElementName;
+  private final String pipelineElementId;
+
+  public ExtendedPipelineElementValidationInfo(String pipelineElementName,
+                                               String pipelineElementId,
+                                               PipelineElementValidationInfo info) {
+    super(info.getLevel(), info.getMessage());
+    this.pipelineElementName = pipelineElementName;
+    this.pipelineElementId = pipelineElementId;
+  }
+
+  public String getPipelineElementName() {
+    return pipelineElementName;
+  }
+
+  public String getPipelineElementId() {
+    return pipelineElementId;
+  }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java
index fc2ce9c6d3..585598890a 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java
@@ -37,19 +37,8 @@ public class PipelineModification {
   private List<SpDataStream> inputStreams;
   private SpDataStream outputStream;
 
-  public PipelineModification(String domId, String elementId,
-                              List<StaticProperty> staticProperties) {
-    super();
-    this.domId = domId;
-    this.elementId = elementId;
-    this.staticProperties = staticProperties;
-    this.inputStreams = new ArrayList<>();
-    this.outputStrategies = new ArrayList<>();
-    this.validationInfos = new ArrayList<>();
-  }
-
   public PipelineModification() {
-
+    validationInfos = new ArrayList<>();
   }
 
   public String getDomId() {
@@ -92,10 +81,6 @@ public class PipelineModification {
     this.inputStreams = inputStreams;
   }
 
-  public void addInputStream(SpDataStream inputStream) {
-    this.inputStreams.add(inputStream);
-  }
-
   public boolean isPipelineElementValid() {
     return pipelineElementValid;
   }
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModificationResult.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModificationResult.java
new file mode 100644
index 0000000000..9e6a1d8c74
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModificationResult.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.pipeline;
+
+import java.util.List;
+
+public record PipelineModificationResult(Pipeline pipeline,
+                                         List<ExtendedPipelineElementValidationInfo> validationInfos) {
+
+  public boolean allPipelineElementsValid() {
+    return validationInfos.stream().noneMatch(v -> v.getLevel() == ValidationInfoLevel.ERROR);
+  }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineVerificationResult.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineVerificationResult.java
new file mode 100644
index 0000000000..7e730b2832
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineVerificationResult.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.pipeline;
+
+import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+
+import java.util.List;
+
+public record PipelineVerificationResult(List<ExtendedPipelineElementValidationInfo> validationInfos,
+                                         List<NamedStreamPipesEntity> modifiedPipelineElements) {
+}
diff --git a/ui/src/app/files/files.routes.ts b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipeline.java
similarity index 70%
rename from ui/src/app/files/files.routes.ts
rename to streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipeline.java
index be69607a13..59948aeb03 100644
--- a/ui/src/app/files/files.routes.ts
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipeline.java
@@ -16,12 +16,16 @@
  *
  */
 
-import { SpBreadcrumbItem } from '@streampipes/shared-ui';
+package org.apache.streampipes.model.pipeline.compact;
 
-export class SpFilesRoutes {
-    static FILES_BASE_LINK = 'files';
-    static BASE: SpBreadcrumbItem = {
-        label: 'File Management',
-        link: [SpFilesRoutes.FILES_BASE_LINK],
-    };
-}
+import org.apache.streampipes.model.connect.adapter.compact.CreateOptions;
+
+import java.util.List;
+
+public record CompactPipeline(
+    String id,
+    String name,
+    String description,
+    List<CompactPipelineElement> pipelineElements,
+    CreateOptions createOptions
+) {}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipelineElement.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipelineElement.java
new file mode 100644
index 0000000000..0f976c8145
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CompactPipelineElement.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.pipeline.compact;
+
+import java.util.List;
+import java.util.Map;
+
+public record CompactPipelineElement(String type,
+                                     String ref,
+                                     String id,
+                                     List<String> connectedTo,
+                                     List<Map<String, Object>> configuration) {
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CreateOptions.java
similarity index 84%
rename from streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactConfiguration.java
rename to streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CreateOptions.java
index 9285ae1878..bcf4952573 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/compact/CompactConfiguration.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/compact/CreateOptions.java
@@ -16,8 +16,7 @@
  *
  */
 
-package org.apache.streampipes.model.connect.adapter.compact;
+package org.apache.streampipes.model.pipeline.compact;
 
-import java.util.Map;
-
-public record CompactConfiguration(Map<String, Object> values) {}
+public record CreateOptions(boolean start) {
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java
index 5e898c36e8..7a9400b17a 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java
@@ -20,6 +20,16 @@ package org.apache.streampipes.model.staticproperty;
 
 public abstract class DefaultStaticPropertyVisitor implements StaticPropertyVisitor {
 
+  protected boolean ignoreValidation;
+
+  public DefaultStaticPropertyVisitor(boolean ignoreValidation) {
+    this.ignoreValidation = ignoreValidation;
+  }
+
+  public DefaultStaticPropertyVisitor() {
+    this(false);
+  }
+
   @Override
   public void visit(CollectionStaticProperty collectionStaticProperty) {
     collectionStaticProperty.getStaticPropertyTemplate().accept(this);
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineModificationGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineModificationGenerator.java
index 362833f842..823d7c7714 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineModificationGenerator.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineModificationGenerator.java
@@ -34,6 +34,7 @@ import org.apache.streampipes.model.message.PipelineEdgeValidation;
 import org.apache.streampipes.model.message.PipelineModificationMessage;
 import org.apache.streampipes.model.pipeline.PipelineElementValidationInfo;
 import org.apache.streampipes.model.pipeline.PipelineModification;
+import org.apache.streampipes.model.pipeline.ValidationInfoLevel;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -99,7 +100,7 @@ public class PipelineModificationGenerator {
       modification.setElementId(t.getElementId());
       try {
         pipelineValidator.apply(source, t, targets, validationInfos);
-        buildModification(modification, t, t.getInputStreams(), true);
+        buildModification(modification, t, t.getInputStreams(), !hasValidationError(modification));
         edgeValidations.put(makeKey(source, t), PipelineEdgeValidation.complete(source.getDom(), t.getDom()));
       } catch (SpValidationException e) {
         e.getErrorLog().forEach(log -> validationInfos.add(PipelineElementValidationInfo.error(log.toString())));
@@ -114,6 +115,10 @@ public class PipelineModificationGenerator {
     });
   }
 
+  private boolean hasValidationError(PipelineModification modification) {
+    return modification.getValidationInfos().stream().anyMatch(v -> v.getLevel() == ValidationInfoLevel.ERROR);
+  }
+
   private String makeKey(NamedStreamPipesEntity source,
                          InvocableStreamPipesEntity t) {
     return source.getDom() + "-" + t.getDom();
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java
index a00a5c43b0..3275f95b98 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/PipelineVerificationHandlerV2.java
@@ -27,8 +27,11 @@ import org.apache.streampipes.model.graph.DataProcessorInvocation;
 import org.apache.streampipes.model.graph.DataSinkInvocation;
 import org.apache.streampipes.model.grounding.EventGrounding;
 import org.apache.streampipes.model.message.PipelineModificationMessage;
+import org.apache.streampipes.model.pipeline.ExtendedPipelineElementValidationInfo;
 import org.apache.streampipes.model.pipeline.Pipeline;
 import org.apache.streampipes.model.pipeline.PipelineModification;
+import org.apache.streampipes.model.pipeline.PipelineModificationResult;
+import org.apache.streampipes.model.pipeline.PipelineVerificationResult;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -49,11 +52,12 @@ public class PipelineVerificationHandlerV2 {
     return new PipelineModificationGenerator(graph, steps).buildPipelineModificationMessage();
   }
 
-  public Pipeline makeModifiedPipeline() {
-    var allElements = verifyAndBuildGraphs(false);
+  public PipelineModificationResult makeModifiedPipeline() {
+    var result = verifyAndBuildGraphs(false);
+    var allElements = result.modifiedPipelineElements();
     pipeline.setSepas(filterAndConvert(allElements, DataProcessorInvocation.class));
     pipeline.setActions(filterAndConvert(allElements, DataSinkInvocation.class));
-    return pipeline;
+    return new PipelineModificationResult(pipeline, result.validationInfos());
   }
 
   private <T extends InvocableStreamPipesEntity> List<T> filterAndConvert(List<NamedStreamPipesEntity> elements,
@@ -65,10 +69,11 @@ public class PipelineVerificationHandlerV2 {
         .toList();
   }
 
-  public List<NamedStreamPipesEntity> verifyAndBuildGraphs(boolean ignoreUnconfigured) {
+  public PipelineVerificationResult verifyAndBuildGraphs(boolean ignoreUnconfigured) {
     var pipelineModifications = verifyPipeline().getPipelineModifications();
     var allElements = new AllElementsProvider(pipeline).getAllElements();
-    var result = new ArrayList<NamedStreamPipesEntity>();
+    var validationInfos = new ArrayList<ExtendedPipelineElementValidationInfo>();
+    var modifiedPipelineElements = new ArrayList<NamedStreamPipesEntity>();
     allElements.forEach(pipelineElement -> {
       var modificationOpt = getModification(pipelineElement.getDom(), pipelineModifications);
       if (modificationOpt.isPresent()) {
@@ -79,15 +84,26 @@ public class PipelineVerificationHandlerV2 {
             applyModificationsForDataProcessor((DataProcessorInvocation) pipelineElement, modification);
           }
         }
+        validationInfos.addAll(
+            modification.getValidationInfos()
+                .stream()
+                .map(v -> new ExtendedPipelineElementValidationInfo(
+                        pipelineElement.getName(),
+                        pipelineElement.getDom(),
+                        v
+                    )
+                )
+                .toList()
+        );
         if (!ignoreUnconfigured || modification.isPipelineElementValid()) {
-          result.add(pipelineElement);
+          modifiedPipelineElements.add(pipelineElement);
         }
       } else {
-        result.add(pipelineElement);
+        modifiedPipelineElements.add(pipelineElement);
       }
     });
 
-    return result;
+    return new PipelineVerificationResult(validationInfos, modifiedPipelineElements);
   }
 
   private void applyModificationsForDataProcessor(DataProcessorInvocation pipelineElement,
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/CheckCompletedVisitor.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/CheckCompletedVisitor.java
index 1083396fcb..fadbd19ecb 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/CheckCompletedVisitor.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/matching/v2/pipeline/CheckCompletedVisitor.java
@@ -29,23 +29,33 @@ import org.apache.streampipes.model.staticproperty.MappingPropertyNary;
 import org.apache.streampipes.model.staticproperty.MappingPropertyUnary;
 import org.apache.streampipes.model.staticproperty.MatchingStaticProperty;
 import org.apache.streampipes.model.staticproperty.OneOfStaticProperty;
+import org.apache.streampipes.model.staticproperty.Option;
 import org.apache.streampipes.model.staticproperty.RuntimeResolvableGroupStaticProperty;
 import org.apache.streampipes.model.staticproperty.RuntimeResolvableTreeInputStaticProperty;
 import org.apache.streampipes.model.staticproperty.SecretStaticProperty;
 import org.apache.streampipes.model.staticproperty.SlideToggleStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternative;
+import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 public class CheckCompletedVisitor extends DefaultStaticPropertyVisitor {
 
-  private List<PipelineElementValidationInfo> validationInfos;
+  private final List<PipelineElementValidationInfo> validationInfos;
 
   public CheckCompletedVisitor() {
     this.validationInfos = new ArrayList<>();
   }
 
+  public CheckCompletedVisitor(boolean ignoreValidation) {
+    this();
+    this.ignoreValidation = ignoreValidation;
+  }
+
   @Override
   public void visit(AnyStaticProperty property) {
 
@@ -53,21 +63,22 @@ public class CheckCompletedVisitor extends DefaultStaticPropertyVisitor {
 
   @Override
   public void visit(CodeInputStaticProperty codeInputStaticProperty) {
+    validateNull(codeInputStaticProperty, codeInputStaticProperty.getValue());
   }
 
   @Override
   public void visit(ColorPickerStaticProperty colorPickerStaticProperty) {
-
+    validateNull(colorPickerStaticProperty, colorPickerStaticProperty.getSelectedColor());
   }
 
   @Override
   public void visit(FileStaticProperty fileStaticProperty) {
-
+    validateNull(fileStaticProperty, fileStaticProperty.getLocationPath());
   }
 
   @Override
   public void visit(FreeTextStaticProperty freeTextStaticProperty) {
-
+    validateNull(freeTextStaticProperty, freeTextStaticProperty.getValue());
   }
 
   @Override
@@ -110,7 +121,7 @@ public class CheckCompletedVisitor extends DefaultStaticPropertyVisitor {
         }
       }
     } else {
-      if (!mappingPropertyUnary.getMapsFromOptions().isEmpty()){
+      if (!mappingPropertyUnary.getMapsFromOptions().isEmpty()) {
         String firstSelector = mappingPropertyUnary.getMapsFromOptions().get(0);
         mappingPropertyUnary.setSelectedProperty(firstSelector);
       }
@@ -124,12 +135,19 @@ public class CheckCompletedVisitor extends DefaultStaticPropertyVisitor {
 
   @Override
   public void visit(OneOfStaticProperty oneOfStaticProperty) {
-
+    if (!ignoreValidation && oneOfStaticProperty.getOptions().stream().noneMatch(Option::isSelected)) {
+      validationInfos.add(PipelineElementValidationInfo.error(
+          String.format(
+              "Configuration \"%s\" must have one selected option, but no option was selected.",
+              oneOfStaticProperty.getInternalName()
+          )
+      ));
+    }
   }
 
   @Override
   public void visit(SecretStaticProperty secretStaticProperty) {
-
+    validateNull(secretStaticProperty, secretStaticProperty.getValue());
   }
 
   @Override
@@ -139,18 +157,43 @@ public class CheckCompletedVisitor extends DefaultStaticPropertyVisitor {
 
   @Override
   public void visit(RuntimeResolvableTreeInputStaticProperty treeInputStaticProperty) {
-
+    if (!ignoreValidation
+        && !treeInputStaticProperty.isOptional()
+        && treeInputStaticProperty.getSelectedNodesInternalNames().isEmpty()) {
+      addMissingConfiguration(treeInputStaticProperty);
+    }
   }
 
   @Override
   public void visit(RuntimeResolvableGroupStaticProperty groupStaticProperty) {
+  }
 
+  @Override
+  public void visit(StaticPropertyAlternatives staticPropertyAlternatives) {
+    if (!ignoreValidation && !staticPropertyAlternatives.isOptional()
+        && staticPropertyAlternatives.getAlternatives().stream().noneMatch(StaticPropertyAlternative::getSelected)) {
+      validationInfos.add(PipelineElementValidationInfo.error(
+          String.format(
+              "No alternative of configuration \"%s\" was selected, but at least one alternative must be chosen",
+              staticPropertyAlternatives.getInternalName()
+          )
+      ));
+    }
+    var visitor = new CheckCompletedVisitor(true);
+    staticPropertyAlternatives.getAlternatives().forEach(alternative -> alternative.accept(visitor));
+    validationInfos.addAll(visitor.getValidationInfos());
   }
 
   public List<PipelineElementValidationInfo> getValidationInfos() {
     return this.validationInfos;
   }
 
+  private void validateNull(StaticProperty sp, Object value) {
+    if (!ignoreValidation && !sp.isOptional() && Objects.isNull(value)) {
+      addMissingConfiguration(sp);
+    }
+  }
+
   private boolean existsSelection(MappingPropertyUnary mappingProperty) {
     return !(mappingProperty.getSelectedProperty() == null || mappingProperty.getSelectedProperty().isEmpty());
   }
@@ -158,4 +201,15 @@ public class CheckCompletedVisitor extends DefaultStaticPropertyVisitor {
   private boolean existsSelection(MappingPropertyNary mappingProperty) {
     return !(mappingProperty.getSelectedProperties() == null || mappingProperty.getSelectedProperties().isEmpty());
   }
+
+  private void addMissingConfiguration(StaticProperty sp) {
+    validationInfos.add(
+        PipelineElementValidationInfo.error(
+            String.format(
+                "Configuration option \"%s\" as no value although it is marked as required",
+                sp.getInternalName()
+            )
+        )
+    );
+  }
 }
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java
new file mode 100644
index 0000000000..1a0a1cd44e
--- /dev/null
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/CompactPipelineManagement.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.manager.pipeline.compact;
+
+import org.apache.streampipes.manager.matching.PipelineVerificationHandlerV2;
+import org.apache.streampipes.manager.pipeline.compact.generation.PipelineElementConfigurationStep;
+import org.apache.streampipes.model.pipeline.Pipeline;
+import org.apache.streampipes.model.pipeline.PipelineModificationResult;
+import org.apache.streampipes.model.pipeline.compact.CompactPipeline;
+import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage;
+
+public class CompactPipelineManagement {
+
+  private final IPipelineElementDescriptionStorage storage;
+
+  public CompactPipelineManagement(IPipelineElementDescriptionStorage storage) {
+    this.storage = storage;
+  }
+
+  public PipelineModificationResult makePipeline(CompactPipeline compactPipeline) throws Exception {
+    var pipeline = new Pipeline();
+    applyPipelineBasics(compactPipeline, pipeline);
+
+    new PipelineElementConfigurationStep(storage).apply(pipeline, compactPipeline);
+
+    return new PipelineVerificationHandlerV2(pipeline).makeModifiedPipeline();
+  }
+
+  private void applyPipelineBasics(CompactPipeline compactPipeline,
+                                   Pipeline pipeline) {
+    pipeline.setElementId(compactPipeline.id());
+    pipeline.setName(compactPipeline.name());
+    pipeline.setDescription(compactPipeline.description());
+  }
+}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/CompactPipelineGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/CompactPipelineGenerator.java
new file mode 100644
index 0000000000..18b8efe9e1
--- /dev/null
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/CompactPipelineGenerator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.manager.pipeline.compact.generation;
+
+import org.apache.streampipes.model.pipeline.Pipeline;
+import org.apache.streampipes.model.pipeline.compact.CompactPipeline;
+
+public interface CompactPipelineGenerator {
+
+  void apply(Pipeline pipeline,
+             CompactPipeline compactPipeline) throws Exception;
+}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataProcessorPipelineElementGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataProcessorPipelineElementGenerator.java
new file mode 100644
index 0000000000..c736293c20
--- /dev/null
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataProcessorPipelineElementGenerator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.manager.pipeline.compact.generation;
+
+import org.apache.streampipes.manager.template.DataProcessorTemplateHandler;
+import org.apache.streampipes.model.graph.DataProcessorInvocation;
+import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement;
+
+public class DataProcessorPipelineElementGenerator {
+
+  private final InvocablePipelineElementGenerator<DataProcessorInvocation> basicGenerator;
+
+  public DataProcessorPipelineElementGenerator(
+      InvocablePipelineElementGenerator<DataProcessorInvocation> basicGenerator) {
+    this.basicGenerator = basicGenerator;
+  }
+
+  public DataProcessorInvocation generate(DataProcessorInvocation processor,
+                                          CompactPipelineElement pipelineElement) {
+    basicGenerator.apply(processor, pipelineElement);
+    var template = basicGenerator.makeTemplate(processor, pipelineElement);
+    return new DataProcessorTemplateHandler(template, processor, false)
+        .applyTemplateOnPipelineElement();
+  }
+}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataSinkPipelineElementGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataSinkPipelineElementGenerator.java
new file mode 100644
index 0000000000..2c1f940252
--- /dev/null
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataSinkPipelineElementGenerator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.manager.pipeline.compact.generation;
+
+import org.apache.streampipes.manager.template.DataSinkTemplateHandler;
+import org.apache.streampipes.model.graph.DataSinkInvocation;
+import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement;
+
+public class DataSinkPipelineElementGenerator {
+
+  private final InvocablePipelineElementGenerator<DataSinkInvocation> basicGenerator;
+
+  public DataSinkPipelineElementGenerator(InvocablePipelineElementGenerator<DataSinkInvocation> basicGenerator) {
+    this.basicGenerator = basicGenerator;
+  }
+
+  public DataSinkInvocation generate(DataSinkInvocation sink,
+                                          CompactPipelineElement pipelineElement) {
+    basicGenerator.apply(sink, pipelineElement);
+    var template = basicGenerator.makeTemplate(sink, pipelineElement);
+    return new DataSinkTemplateHandler(template, sink, false)
+        .applyTemplateOnPipelineElement();
+  }
+}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataStreamPipelineElementGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataStreamPipelineElementGenerator.java
new file mode 100644
index 0000000000..9d466062c9
--- /dev/null
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/DataStreamPipelineElementGenerator.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.manager.pipeline.compact.generation;
+
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement;
+
+public class DataStreamPipelineElementGenerator {
+
+  public SpDataStream generate(SpDataStream stream,
+                               CompactPipelineElement pipelineElement) {
+    stream.setDom("jsplumb_" + pipelineElement.ref());
+    return stream;
+  }
+}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/InvocablePipelineElementGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/InvocablePipelineElementGenerator.java
new file mode 100644
index 0000000000..0f8d135830
--- /dev/null
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/InvocablePipelineElementGenerator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.manager.pipeline.compact.generation;
+
+import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
+import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement;
+import org.apache.streampipes.model.template.PipelineElementTemplate;
+import org.apache.streampipes.model.util.Cloner;
+
+import java.util.ArrayList;
+
+public class InvocablePipelineElementGenerator<T extends InvocableStreamPipesEntity> {
+
+  private static final String ID_PREFIX = "jsplumb_";
+
+  public void apply(T element,
+                    CompactPipelineElement compatPipelineElement) {
+    element.setDom(ID_PREFIX + compatPipelineElement.ref());
+    element.setConnectedTo(compatPipelineElement.connectedTo().stream().map(c -> ID_PREFIX + c).toList());
+    element.setStreamRequirements(new Cloner().streams(element.getStreamRequirements()));
+  }
+
+  protected PipelineElementTemplate makeTemplate(T element,
+                                                 CompactPipelineElement compactPipelineElement) {
+    var configs = compactPipelineElement.configuration();
+    if (compactPipelineElement.configuration() == null) {
+      configs = new ArrayList<>();
+    }
+    var template = new PipelineElementTemplate();
+    template.setTemplateConfigs(configs);
+    template.setBasePipelineElementAppId(element.getAppId());
+
+    return template;
+  }
+}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/PipelineElementConfigurationStep.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/PipelineElementConfigurationStep.java
new file mode 100644
index 0000000000..ec5539414c
--- /dev/null
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/pipeline/compact/generation/PipelineElementConfigurationStep.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.manager.pipeline.compact.generation;
+
+import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.graph.DataProcessorInvocation;
+import org.apache.streampipes.model.graph.DataSinkInvocation;
+import org.apache.streampipes.model.pipeline.Pipeline;
+import org.apache.streampipes.model.pipeline.compact.CompactPipeline;
+import org.apache.streampipes.model.pipeline.compact.CompactPipelineElement;
+import org.apache.streampipes.storage.api.IPipelineElementDescriptionStorage;
+
+public class PipelineElementConfigurationStep implements CompactPipelineGenerator {
+
+  private static final String StreamType = "stream";
+  private static final String ProcessorType = "processor";
+  private static final String SinkType = "sink";
+
+  private final IPipelineElementDescriptionStorage storage;
+
+  public PipelineElementConfigurationStep(IPipelineElementDescriptionStorage storage) {
+    this.storage = storage;
+  }
+
+  @Override
+  public void apply(Pipeline pipeline,
+                    CompactPipeline compactPipeline) throws Exception {
+    compactPipeline.pipelineElements().forEach(pe -> {
+      if (pe.type().equalsIgnoreCase(StreamType)) {
+        pipeline.getStreams().add(makeStream(pe));
+      } else if (pe.type().equalsIgnoreCase(ProcessorType)) {
+        pipeline.getSepas().add(makeProcessor(pe));
+      } else if (pe.type().equalsIgnoreCase(SinkType)) {
+        pipeline.getActions().add(makeSink(pe));
+      }
+    });
+  }
+
+  public SpDataStream makeStream(CompactPipelineElement pipelineElement) {
+    var element = storage.getDataStreamById(pipelineElement.id());
+    return new DataStreamPipelineElementGenerator().generate(element, pipelineElement);
+  }
+
+  public DataProcessorInvocation makeProcessor(CompactPipelineElement pipelineElement) {
+    var element = storage.getDataProcessorByAppId(pipelineElement.id());
+    var invocation = new DataProcessorInvocation(element);
+    return new DataProcessorPipelineElementGenerator(new InvocablePipelineElementGenerator<>())
+        .generate(invocation, pipelineElement);
+  }
+
+  public DataSinkInvocation makeSink(CompactPipelineElement pipelineElement) {
+    var element = storage.getDataSinkByAppId(pipelineElement.id());
+    var invocation = new DataSinkInvocation(element);
+    return new DataSinkPipelineElementGenerator(new InvocablePipelineElementGenerator<>())
+        .generate(invocation, pipelineElement);
+  }
+
+}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java
index 0e6b22bdc6..3ec765196f 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/preview/PipelinePreview.java
@@ -53,6 +53,7 @@ public class PipelinePreview {
     List<NamedStreamPipesEntity> pipelineElements = new ArrayList<>(
         new PipelineVerificationHandlerV2(pipeline)
             .verifyAndBuildGraphs(true)
+            .modifiedPipelineElements()
     );
 
     rewriteElementIds(pipelineElements, elementIdMappings);
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CompactPipelineResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CompactPipelineResource.java
new file mode 100644
index 0000000000..2147dde19e
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/CompactPipelineResource.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl;
+
+import org.apache.streampipes.manager.pipeline.PipelineManager;
+import org.apache.streampipes.manager.pipeline.compact.CompactPipelineManagement;
+import org.apache.streampipes.model.message.Notification;
+import org.apache.streampipes.model.message.NotificationType;
+import org.apache.streampipes.model.message.Notifications;
+import org.apache.streampipes.model.pipeline.PipelineOperationStatus;
+import org.apache.streampipes.model.pipeline.compact.CompactPipeline;
+import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
+import org.apache.streampipes.rest.security.AuthConstants;
+
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/v2/compact-pipelines")
+public class CompactPipelineResource extends AbstractAuthGuardedRestResource {
+
+  private final CompactPipelineManagement compactPipelineManagement;
+
+  public CompactPipelineResource() {
+    this.compactPipelineManagement = new CompactPipelineManagement(
+        getPipelineElementStorage()
+    );
+  }
+
+  @PostMapping(
+      consumes = {
+          MediaType.APPLICATION_JSON_VALUE,
+          "application/yaml",
+          "application/yml"
+      }
+  )
+  @PreAuthorize(AuthConstants.HAS_WRITE_PIPELINE_PRIVILEGE)
+  public ResponseEntity<?> addPipelineCompact(
+      @RequestBody CompactPipeline compactPipeline
+  ) throws Exception {
+
+    var pipelineGenerationResult = compactPipelineManagement.makePipeline(compactPipeline);
+    if (pipelineGenerationResult.allPipelineElementsValid()) {
+      String pipelineId = PipelineManager.addPipeline(getAuthenticatedUserSid(), pipelineGenerationResult.pipeline());
+      if (compactPipeline.createOptions().start()) {
+        try {
+          PipelineOperationStatus status = PipelineManager.startPipeline(pipelineId);
+          return ok(status);
+        } catch (Exception e) {
+          return statusMessage(Notifications.error(NotificationType.UNKNOWN_ERROR));
+        }
+      }
+      var message = Notifications.success("Pipeline stored");
+      message.addNotification(new Notification("id", pipelineId));
+      return ok(message);
+    } else {
+      return ResponseEntity.status(400).body(pipelineGenerationResult.validationInfos());
+    }
+  }
+
+  @PutMapping(
+      path = "{id}",
+      consumes = {
+          MediaType.APPLICATION_JSON_VALUE,
+          "application/yaml",
+          "application/yml"
+      }
+  )
+  @PreAuthorize(AuthConstants.HAS_WRITE_PIPELINE_PRIVILEGE)
+  public ResponseEntity<?> updatePipelineCompact(
+      @PathVariable("id") String elementId,
+      @RequestBody CompactPipeline compactPipeline
+  ) throws Exception {
+
+    return null;
+  }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
index 006f3f955d..c5ff418e82 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
@@ -23,6 +23,7 @@ import org.apache.streampipes.model.message.Notifications;
 import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
 import org.apache.streampipes.rest.security.AuthConstants;
 import org.apache.streampipes.rest.shared.exception.SpMessageException;
+import org.apache.streampipes.sdk.helpers.Filetypes;
 
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -72,6 +73,13 @@ public class PipelineElementFile extends AbstractAuthGuardedRestResource {
               fileDetail.getInputStream()
           );
       return ok(metadata);
+    } catch (IllegalArgumentException e) {
+      return badRequest(Notifications.error(
+          String.format(
+              "This file type is not supported. Allowed file types are %s.",
+              Filetypes.getAllFileExtensions().toString()
+          )
+      ));
     } catch (Exception e) {
       return fail();
     }
@@ -132,10 +140,10 @@ public class PipelineElementFile extends AbstractAuthGuardedRestResource {
   @PreAuthorize(AuthConstants.HAS_READ_FILE_PRIVILEGE)
   public ResponseEntity<List<String>> getAllOriginalFilenames() {
     return ok(fileManager.getAllFiles()
-                         .stream()
-                         .map(fileMetadata -> fileMetadata.getFilename()
-                                                          .toLowerCase())
-                         .toList());
+        .stream()
+        .map(fileMetadata -> fileMetadata.getFilename()
+            .toLowerCase())
+        .toList());
   }
 
   @GetMapping(
diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java
index 681b180983..b8895a9b1f 100644
--- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java
+++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/helpers/Filetypes.java
@@ -29,6 +29,7 @@ public enum Filetypes {
   XLS("xls"),
   XLSX("xlsx"),
   XML("xml"),
+  YAML("yaml", "yml"),
   ZIP("zip");
 
   private final List<String> fileExtensions;
diff --git a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/FunctionRegistrationHandler.java b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/FunctionRegistrationHandler.java
index 04f1f370c4..8e17da40dd 100644
--- a/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/FunctionRegistrationHandler.java
+++ b/streampipes-service-extensions/src/main/java/org/apache/streampipes/service/extensions/function/FunctionRegistrationHandler.java
@@ -26,6 +26,16 @@ import org.slf4j.LoggerFactory;
 
 import java.util.List;
 
+/**
+ * The {@code FunctionRegistrationHandler} class is responsible for registering
+ * a list of functions with the StreamPipes backend system.
+ *
+ * <p>Note: This class only handles the registration of the functions in the backend.
+ * The actual execution of the registered functions occurs in the extension service.
+ *
+ * <p>Once the registration is successful, a log entry is created to indicate
+ * the completion of the process.
+ */
 public class FunctionRegistrationHandler extends RegistrationHandler {
 
   private static final Logger LOG = LoggerFactory.getLogger(FunctionRegistrationHandler.class);
@@ -36,7 +46,8 @@ public class FunctionRegistrationHandler extends RegistrationHandler {
 
   @Override
   protected void performRequest(StreamPipesClient client) {
-    client.adminApi().registerFunctions(functions);
+    client.adminApi()
+          .registerFunctions(functions);
   }
 
   @Override
diff --git a/ui/src/app/configuration/configuration.module.ts b/ui/src/app/configuration/configuration.module.ts
index f24da2f959..ff860383f5 100644
--- a/ui/src/app/configuration/configuration.module.ts
+++ b/ui/src/app/configuration/configuration.module.ts
@@ -96,6 +96,7 @@ import { FileOverviewComponent } from './files/file-overview/file-overview.compo
 import { FileUploadDialogComponent } from './dialog/file-upload/file-upload-dialog.component';
 import { FileRenameDialogComponent } from './dialog/file-rename/file-rename-dialog.component';
 import { MatDialogModule } from '@angular/material/dialog';
+import { MatProgressBarModule } from '@angular/material/progress-bar';
 
 @NgModule({
     imports: [
@@ -120,6 +121,7 @@ import { MatDialogModule } from '@angular/material/dialog';
         CoreUiModule,
         ReactiveFormsModule,
         PlatformServicesModule,
+        MatProgressBarModule,
         RouterModule.forChild([
             {
                 path: '',
diff --git a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html
index 6d83e5a5c6..ecec2cccdd 100644
--- a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html
+++ b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.html
@@ -17,7 +17,10 @@
 -->
 
 <div class="sp-dialog-container">
-    <div class="sp-dialog-content p-15">
+    <div class="sp-dialog-content p-15" fxLayout="column">
+        <sp-warning-box *ngIf="uploadError" style="font-size: smaller">
+            {{ uploadErrorMessage }}
+        </sp-warning-box>
         <div fxFlex="100" *ngIf="this.duplicateFileNames.length === 0">
             <div fxFlex="100" style="margin: 5px; width: 100%">
                 <mat-form-field
@@ -57,7 +60,7 @@
                         color="accent"
                         matSuffix
                         mat-button
-                        style="min-width: 0px"
+                        style="min-width: 0"
                     >
                         <mat-icon *ngIf="uploadStatus < 99"
                             >insert_drive_file</mat-icon
diff --git a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts
index 328cfe8494..beee8317b9 100644
--- a/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts
+++ b/ui/src/app/configuration/dialog/file-upload/file-upload-dialog.component.ts
@@ -39,6 +39,9 @@ export class FileUploadDialogComponent {
 
     uploadStatus = 0;
 
+    uploadError = false;
+    uploadErrorMessage = '';
+
     constructor(
         private dialogRef: DialogRef<FileUploadDialogComponent>,
         private filesService: FilesService,
@@ -73,6 +76,7 @@ export class FileUploadDialogComponent {
     }
 
     uploadFile(index: number): void {
+        this.uploadError = false;
         this.filesService
             .uploadFile(this.selectedUploadFiles.item(index))
             .subscribe(
@@ -90,7 +94,15 @@ export class FileUploadDialogComponent {
                         }
                     }
                 },
-                error => {},
+                error => {
+                    this.uploadError = true;
+                    if (error.error?.notifications?.length > 0) {
+                        this.uploadErrorMessage =
+                            error.error.notifications[0].title;
+                    } else {
+                        this.uploadErrorMessage = error.error;
+                    }
+                },
             );
     }
 
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
index 56c283b6c0..6ff8ea6a14 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.html
@@ -62,10 +62,18 @@
                     name="pipelineName"
                     (blur)="triggerTutorial()"
                 />
+                <mat-error
+                    >Pipeline name must have between 3 and 40
+                    characters.</mat-error
+                >
             </mat-form-field>
             <mat-form-field fxFlex color="accent">
                 <mat-label>Description</mat-label>
                 <input [formControlName]="'pipelineDescription'" matInput />
+                <mat-error
+                    >Pipeline description must not have more than 80
+                    characters.</mat-error
+                >
             </mat-form-field>
         </div>
     </form>
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
index 6f32859192..d75b2c1aed 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline-settings/save-pipeline-settings.component.ts
@@ -51,6 +51,7 @@ export class SavePipelineSettingsComponent implements OnInit {
             'pipelineName',
             new UntypedFormControl(this.pipeline.name, [
                 Validators.required,
+                Validators.minLength(3),
                 Validators.maxLength(40),
             ]),
         );
diff --git a/ui/src/app/files/files.module.ts b/ui/src/app/files/files.module.ts
deleted file mode 100644
index 991569618d..0000000000
--- a/ui/src/app/files/files.module.ts
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-import { NgModule } from '@angular/core';
-import { CommonModule } from '@angular/common';
-import { FlexLayoutModule } from '@ngbracket/ngx-layout';
-import { MatButtonModule } from '@angular/material/button';
-import { MatGridListModule } from '@angular/material/grid-list';
-import { MatIconModule } from '@angular/material/icon';
-import { MatDividerModule } from '@angular/material/divider';
-import { MatListModule } from '@angular/material/list';
-import { FilesComponent } from '../configuration/files/files.component';
-import { MatTabsModule } from '@angular/material/tabs';
-import { CoreUiModule } from '../core-ui/core-ui.module';
-import { FormsModule, ReactiveFormsModule } from '@angular/forms';
-import { MatFormFieldModule } from '@angular/material/form-field';
-import { MatProgressBarModule } from '@angular/material/progress-bar';
-import { MatInputModule } from '@angular/material/input';
-import { ServicesModule } from '../services/services.module';
-import { MatTableModule } from '@angular/material/table';
-import { MatPaginatorModule } from '@angular/material/paginator';
-import { MatChipsModule } from '@angular/material/chips';
-import { MatTooltipModule } from '@angular/material/tooltip';
-import { PlatformServicesModule } from '@streampipes/platform-services';
-import { RouterModule } from '@angular/router';
-import { SharedUiModule } from '@streampipes/shared-ui';
-import { MatDialogModule } from '@angular/material/dialog';
-
-@NgModule({
-    imports: [
-        CommonModule,
-        CoreUiModule,
-        FlexLayoutModule,
-        FormsModule,
-        ReactiveFormsModule,
-        MatButtonModule,
-        MatChipsModule,
-        MatFormFieldModule,
-        MatGridListModule,
-        MatIconModule,
-        MatInputModule,
-        MatDividerModule,
-        MatListModule,
-        MatPaginatorModule,
-        MatProgressBarModule,
-        MatTableModule,
-        MatTabsModule,
-        MatTooltipModule,
-        PlatformServicesModule,
-        ServicesModule,
-        SharedUiModule,
-        RouterModule.forChild([
-            {
-                path: '',
-                children: [
-                    {
-                        path: '',
-                        component: FilesComponent,
-                    },
-                ],
-            },
-        ]),
-        MatDialogModule,
-    ],
-    declarations: [],
-    providers: [],
-})
-export class FilesModule {}


Test patch:
diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/test/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessorTest.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/test/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessorTest.java
new file mode 100644
index 0000000000..b93127a352
--- /dev/null
+++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/test/java/org/apache/streampipes/processors/transformation/jvm/processor/transformtoboolean/TransformToBooleanProcessorTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.processors.transformation.jvm.processor.transformtoboolean;
+
+import org.apache.streampipes.test.executors.PrefixStrategy;
+import org.apache.streampipes.test.executors.ProcessingElementTestExecutor;
+import org.apache.streampipes.test.executors.TestConfiguration;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+public class TransformToBooleanProcessorTest {
+
+  @Test
+  void onEvent() {
+
+    var processor = new TransformToBooleanProcessor();
+
+    List<Map<String, Object>> inputEvents = List.of(Map.of(
+        "numberBoolean", 0,
+        "stringBoolean", "true",
+        "stringBooleanFalse", "false",
+        "floatBoolean", 1.0
+    ));
+
+    List<Map<String, Object>> outputEvents = List.of(Map.of(
+        "numberBoolean", false,
+        "stringBoolean", true,
+        "stringBooleanFalse", false,
+        "floatBoolean", true
+    ));
+
+    var configuration = TestConfiguration
+        .builder()
+        .config(TransformToBooleanProcessor.TRANSFORM_FIELDS_ID, List.of(
+            "s0::numberBoolean", "s0::stringBoolean", "s0::stringBooleanFalse", "s0::floatBoolean"))
+        .prefixStrategy(PrefixStrategy.SAME_PREFIX)
+        .build();
+
+    var testExecutor = new ProcessingElementTestExecutor(processor, configuration);
+
+    testExecutor.run(inputEvents, outputEvents);
+  }
+}
diff --git a/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java b/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java
index 8c287f2cbc..2e0204fb53 100644
--- a/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java
+++ b/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/helpers/FiletypesTest.java
@@ -29,9 +29,9 @@ class FiletypesTest {
   public void getAllFileExtensions_returnsAllExtensions() {
     var actualExtensions = Filetypes.getAllFileExtensions();
 
-    assertEquals(8, actualExtensions.size());
+    assertEquals(10, actualExtensions.size());
     assertTrue(actualExtensions.contains("jpg"));
     assertTrue(actualExtensions.contains("jpeg"));
     assertTrue(actualExtensions.contains("csv"));
   }
-}
\ No newline at end of file
+}