Repository: eyecan-ai/pipelime (language: python)

Issue:
Bump version: 0.1.2 → 0.1.3



Discussion / hints:

Fix patch:
diff --git a/.flake8 b/.flake8
index 98b7c02..cff2e5a 100644
--- a/.flake8
+++ b/.flake8
@@ -1,3 +1,6 @@
  
 [flake8]
-max-line-length = 140
\ No newline at end of file
+max-line-length = 140
+ignore = E203, W503
+per-file-ignores =
+    */__init__.py: F401
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8692a5c..e0c089c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1,10 +1,10 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python package develop
+name: Develop Merge
 
 on:
-  push:
+  pull_request:
     branches: [ develop ]
 
 jobs:
@@ -27,6 +27,7 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install -e .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fe40a63..6f0f19d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: PyPI Uploade
+name: PyPI Upload
 
 on:
   pull_request:
@@ -27,6 +27,7 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install -e .
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
diff --git a/examples/garbage/_schema.schema b/examples/garbage/_schema.schema
deleted file mode 100644
index a114c07..0000000
--- a/examples/garbage/_schema.schema
+++ /dev/null
@@ -1,20 +0,0 @@
-from typing import Any
-from schema import And, Optional
-import numpy as np
-
-# schema = {
-#     'image': And(
-#         np.ndarray,
-#         lambda x: len(x.shape) >= 3
-#     ),
-#     'metadata': {
-#         any: object,
-#         'counter': int
-#     },
-#     any: object
-# }
-schema = {
-    'image': object,
-    'metadata': object,
-    Optional(str): Optional(object)
-}
diff --git a/examples/garbage/albumentations_tst.py b/examples/garbage/albumentations_tst.py
deleted file mode 100644
index a63baa6..0000000
--- a/examples/garbage/albumentations_tst.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from pipelime.augmentations.factory import AugmentationsFactory
-from choixe.configurations import XConfig
-from albumentations.core.serialization import SerializableMeta
-from albumentations.core.six import add_metaclass
-import rich
-import albumentations as A
-import cv2
-import pipelime as PL
-# A.core.serialization.SerializableMeta.__new__(PadIfNeededV2, 'PadIfNeededV2', )
-
-
-transform = A.Compose([
-    A.RandomCrop(width=450, height=450),
-    A.HorizontalFlip(p=0.5),
-    A.RandomBrightnessContrast(p=0.2),
-],
-    bbox_params=A.BboxParams(format='coco'),
-    keypoint_params=A.KeypointParams(format='xy', label_fields=['category_id'])
-)
-
-
-#
-#transform = A.RGBShift()
-
-
-# d = A.to_dict(transform)
-
-
-# cfg = XConfig.from_dict(d)
-# rich.print(cfg.to_dict(discard_private_qualifiers=False))
-# cfg.save_to('/tmp/out.yml')
diff --git a/examples/garbage/click_experiment.py b/examples/garbage/click_experiment.py
deleted file mode 100644
index 2aa74c8..0000000
--- a/examples/garbage/click_experiment.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from os import name
-from typing import List
-import click
-
-import click
-
-
-class CLICommandInvoker(click.Command):
-
-    # def parse_args(self, ctx: click.Context, args: List[str]) -> List[str]:
-    #     print("PARSING", args)
-    #     return super().parse_args(ctx, args)
-
-    def invoke(self, ctx: click.Context):
-        command = ctx.command.name
-        print("Invoking ", ctx)
-
-        ctx.command = click.option('-p', '--pino', required=True, help='HELLOE')(ctx.command)
-        ret = super(CLICommandInvoker, self).invoke(ctx)
-        return ret
-
-
-@click.command('ciao', help='This is command', cls=CLICommandInvoker)
-@click.option('-o', '--mouse')
-def ciao(**kwargs):
-    print("Done", kwargs)
-
-
-if __name__ == '__main__':
-    ciao()
diff --git a/examples/garbage/example_underfolder.py b/examples/garbage/example_underfolder.py
deleted file mode 100644
index 56a7a2f..0000000
--- a/examples/garbage/example_underfolder.py
+++ /dev/null
@@ -1,102 +0,0 @@
-from pipelime.augmentations.transforms import PadIfNeededV2
-import rich
-import albumentations as A
-from albumentations.augmentations.functional import scale
-from pipelime.sequences.samples import Sample
-from pipelime.sequences.stages import StageAugmentations, StageIdentity, StageRemap
-from pipelime.sequences.readers.filesystem import UnderfolderReader
-import cv2
-from pipelime.sequences.operations import OperationFilterByQuery, OperationGroupBy, OperationIdentity
-import numpy as np
-
-
-def draw_keypoints(keypoints: np.ndarray, image: np.ndarray):
-
-    h, w = image.shape[:2]
-    for kp in keypoints:
-        print("KP", kp)
-        pos = np.array(kp[0:2])  # * np.array([w, h])
-        a = kp[2]
-        s = kp[3] * 0.7
-        direction = np.array([np.cos(-a), np.sin(-a)])
-
-        cv2.circle(image, tuple(pos.astype(int)), radius=5, color=(255, 255, 255))
-
-        pos2 = pos + direction * s
-        cv2.arrowedLine(image, tuple(pos.astype(int)), tuple(pos2.astype(int)), (255, 255, 255), 2)
-
-
-# folder = '/home/daniele/Desktop/experiments/2021-01-28.PlaygroundDatasets/flow_00'
-folder = '/tmp/ds'
-dataset = UnderfolderReader(folder=folder)
-print(len(dataset))
-
-op = OperationGroupBy(field='`metadata.counter`')
-op = OperationIdentity()  # OperationFilterByQuery(query='`metadata.')
-dataset = op(dataset)
-
-transform = A.Compose([
-    # A.RandomCrop(width=450, height=450),
-    # A.HorizontalFlip(p=0.5)
-    A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.6, rotate_limit=180, border_mode=cv2.BORDER_CONSTANT, p=1.0),
-    PadIfNeededV2(min_height=1800, min_width=1000),
-    # A.RandomBrightnessContrast(p=0.2),
-],
-    keypoint_params=A.KeypointParams(format='xyas', remove_invisible=True, angle_in_degrees=False),
-    # bbox_params=A.BboxParams(format='yolo')
-)
-
-stage = StageAugmentations(
-    transform_cfg=A.to_dict(transform),
-    targets={
-        'image': 'image',
-        'keypoints': 'keypoints',
-        'mask': 'mask',
-        'inst': 'mask'
-    }
-)
-
-
-# stage = StageRemap({
-#     'image': 'a'
-# }, remove_missing=False)
-# s2 = StageIdentity()
-[cv2.namedWindow(x, cv2.WINDOW_NORMAL) for x in ["image", "mask", "inst"]]
-
-for sample in dataset:
-
-    sample = stage(sample)
-
-    image = sample['image']
-    h, w = image.shape[:2]
-    keypoints = sample['keypoints']
-
-    mask = sample['mask']
-    inst = sample['inst']
-
-    # idx = keypoints[:, 0]
-
-    # # keypoints = keypoints[:, 1:5]
-    # keypoints[:, 0] *= w
-    # keypoints[:, 1] *= h
-
-    # print("pre", keypoints)
-    # transform.add_targets({'inst': 'mask', 'keypoints': 'keypoints'})
-    # _t = transform(image=image, keypoints=keypoints, mask=mask, inst=inst)  # , class_labels=idx)
-    # image = _t['image']
-    # keypoints = _t['keypoints']
-    # # idx = _t['class_labels']
-    # mask = _t['mask']
-    # inst = _t['inst']
-    # print("post", keypoints)
-    # print(idx)
-
-    draw_keypoints(keypoints, image)
-
-    mask = (mask - mask.min())/(mask.max() - mask.min())
-    inst = (inst - inst.min())/(inst.max() - inst.min())
-    cv2.imshow("image", image)
-    cv2.imshow("mask", mask)
-    cv2.imshow("inst", inst)
-    if ord('q') == cv2.waitKey(0):
-        break
diff --git a/examples/garbage/gkit_example.py b/examples/garbage/gkit_example.py
deleted file mode 100644
index c4f2efb..0000000
--- a/examples/garbage/gkit_example.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from operator import mul, sub
-import graphkit
-from graphkit import compose, operation
-# Computes |a|^p.
-from pipelime.sequences.operations import OperationShuffle, OperationSum
-
-
-def abspow(a, p):
-    c = abs(a) ** p
-    return c
-
-
-# Compose the mul, sub, and abspow operations into a computation graph.
-graph = compose(name="graph")(
-    operation(name="l1", needs=["a", "b"], provides=["ab"])(OperationSum()),
-    operation(name="l2", needs=["a", "ab"], provides=["aab"])(OperationSum()),
-    operation(name="l3", needs=["aab"], provides=["out"])(OperationShuffle())
-)
-
-# Run the graph and request all of the outputs.
-out = graph({'a': 2, 'b': 5})
-
-# Prints "{'a': 2, 'a_minus_ab': -8, 'b': 5, 'ab': 10, 'abs_a_minus_ab_cubed': 512}".
-print(out)
-
-# Run the graph and request a subset of the outputs.
-out = graph({'a': 2, 'b': 5}, outputs=["a_minus_ab"])
-
-# Prints "{'a_minus_ab': -8}".
-print(out)
diff --git a/examples/garbage/groupby_example.py b/examples/garbage/groupby_example.py
deleted file mode 100644
index b3609ef..0000000
--- a/examples/garbage/groupby_example.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import rich
-from pipelime.sequences.samples import PlainSample, SamplesSequence
-import numpy as np
-
-from pipelime.sequences.operations import OperationGroupBy
-
-
-def generate_plain_samples(
-    namespace: str,
-    N: int,
-    group_each: int = 5,
-    heavy_data: bool = False
-):
-
-    samples = []
-    for i in range(N):
-        data = {
-            'idx': f'{namespace}{i}',
-            'number': i,
-            'fraction': i / 1000.,
-            'odd': i % 2 == 1,
-            'data0': np.random.uniform(0.0, 1.0, (64, 64, 3)) if heavy_data else 1.64643,
-            'data1': np.random.uniform(0, 255, (8, 8)).astype(np.uint8) if heavy_data else 25588,
-            'metadata': {
-                'even': i % 2 == 0,
-                'name': f'{namespace}{i}',
-                'N': i,
-                'deep': {
-                    'super_deep': 0,
-                    'groupby_field': int(i / group_each),
-                    'groupby_field2': int(i % 3),
-                    'groupby_field3': int(i % 10),
-                }
-            }
-        }
-        samples.append(PlainSample(data=data))
-
-    return samples
-
-
-d = SamplesSequence(samples=generate_plain_samples('d0', 128))
-
-op = OperationGroupBy(field='metadata.deep.groupby_field')
-
-out = op(d)
-for sample in out:
-    for k, v in sample.items():
-        rich.print(k, v)
-
-op2 = OperationGroupBy(field='metadata.deep.groupby_field')
-
-out2 = op2(out)
-# for sample in out2:
-#     for k, v in sample.items():
-#         rich.print(k, v)
-
-op3 = OperationGroupBy(field='metadata.even.0.0')
-
-out3 = op3(out2)
-# for sample in out3:
-#     for k, v in sample.items():
-#         rich.print(k, v)
-print(len(out), len(out2), len(out3))
diff --git a/examples/garbage/json_schema_example.py b/examples/garbage/json_schema_example.py
deleted file mode 100644
index 697c56c..0000000
--- a/examples/garbage/json_schema_example.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from pipelime.sequences.validation import SchemaLoader
-from pipelime.sequences.samples import Sample
-from pipelime.sequences.readers.filesystem import UnderfolderReader
-
-
-s = SchemaLoader().load(filename='_schema.schema')
-
-folder = '/home/daniele/Desktop/experiments/2021-01-28.PlaygroundDatasets/lego_00'
-reader = UnderfolderReader(folder=folder)
-
-for sample in reader:
-    sample: Sample
-    print(sample.id)
-
-    sample.validate(s, deep=False)
diff --git a/examples/garbage/main.py b/examples/garbage/main.py
deleted file mode 100644
index c800538..0000000
--- a/examples/garbage/main.py
+++ /dev/null
@@ -1,13 +0,0 @@
-
-from pipelime.sequences.readers.filesystem import UnderfolderReader
-
-
-folder = '/tmp/gino'
-r = UnderfolderReader(folder=folder)
-
-print(len(r))
-
-for sample in r:
-    for k in sample:
-        if k == 'metadata':
-            print(sample[k])
diff --git a/examples/garbage/pipe_in_python.py b/examples/garbage/pipe_in_python.py
deleted file mode 100644
index b48358e..0000000
--- a/examples/garbage/pipe_in_python.py
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-import rich
-from pipelime.lib import AddOp, PlainSample, SamplesSequence, SequenceOpFactory
-from choixe.configurations import XConfig
-
-
-def decorate_with_ops():
-    def add(self, o: SamplesSequence):
-        return AddOp()([self, o])
-
-    SamplesSequence.__add__ = add
-    SamplesSequence.aaa = add
-
-
-decorate_with_ops()
-print(SamplesSequence.__add__)
-
-
-N = 10
-samples_a = [PlainSample(data={'idx': idx, 'metadata': {'idx': idx, 'name': str(idx), 'label': 0}}) for idx in range(N)]
-samples_b = [PlainSample(data={'idx': idx, 'metadata': {'idx': idx, 'name': str(idx), 'label': 1}}) for idx in range(N)]
-
-dataset_a = SamplesSequence(samples=samples_a)
-dataset_b = SamplesSequence(samples=samples_b)
-
-
-d = dataset_a + dataset_b
-print(len(d))
diff --git a/examples/garbage/pipe_tst_with_graph.py b/examples/garbage/pipe_tst_with_graph.py
deleted file mode 100644
index 3c101cb..0000000
--- a/examples/garbage/pipe_tst_with_graph.py
+++ /dev/null
@@ -1,308 +0,0 @@
-
-
-import hashlib
-import random
-import os
-import tempfile
-from networkx.drawing.nx_agraph import graphviz_layout, to_agraph
-from typing import Union
-import rich
-from pipelime.lib import AddOp, PlainSample, SamplesSequence, SequenceOpFactory
-from choixe.configurations import XConfig
-import networkx as nx
-import matplotlib.pyplot as plt
-
-
-def sample_dataset(namespace: str, N: int):
-
-    samples = []
-    for i in range(N):
-        data = {
-            'idx': f'{namespace}{i}',
-            'number': i,
-            'fraction': i / 1000.,
-            'metadata': {
-                'name': f'{namespace}{i}',
-                'N': i,
-                'deep': {
-                    'super_deep': 0
-                }
-            }
-        }
-        samples.append(PlainSample(data=data))
-
-    return SamplesSequence(samples=samples)
-
-
-class PipeNode(object):
-    ID_COUNTER = 0
-
-    def __init__(self, input_ports: Union[str, list, dict], output_ports: Union[str, list, dict], node: dict) -> None:
-        self.id = PipeNode.ID_COUNTER
-        PipeNode.ID_COUNTER += 1
-        self.input_ports = input_ports
-        self.output_ports = output_ports
-        self.node = node
-
-    def in_ports(self):
-        return self.ports_as_list(self.input_ports)
-
-    def out_ports(self):
-        return self.ports_as_list(self.output_ports)
-
-    def ports_as_list(self, v: any):
-        if v is None:
-            return []
-        elif isinstance(v, str):
-            return [v]
-        elif isinstance(v, list):
-            return v
-        elif isinstance(v, dict):
-            return list(v.values())
-        else:
-            raise NotImplementedError(f'{type(v)}')
-
-    def __repr__(self) -> str:
-        return str(self.id)
-
-
-class DataNode(object):
-
-    def __init__(self, name: str) -> None:
-        self.name = name
-
-    def __repr__(self) -> str:
-        return self.name
-
-    def __hash__(self) -> int:
-        return hash(self.name)
-
-    def __eq__(self, o: object) -> bool:
-        return self.name == o.name
-
-
-pipe = {
-    'input': {
-        'GOOD': None,
-        'BAD': None
-    },
-    'modules': [
-        {
-            'input': None,
-            'output': 'GOOD',
-            'node': {
-                'type': 'Source',
-                'data': sample_dataset('good_', 100)
-            }
-        },
-        {
-            'input': None,
-            'output': 'BAD',
-            'node': {
-                'type': 'Source',
-                'data': sample_dataset('bad_', 100)
-            }
-        },
-        # {
-        #     'input': None,
-        #     'output': 'SUPER',
-        #     'node': {
-        #         'type': 'Source',
-        #         'data': sample_dataset('super_', 100)
-        #     }
-        # },
-        {
-            'input': 'BAD',
-            'output': {
-                'a': 'BAD_0',
-                'b': 'BAD_1'
-            },
-            'node': {
-                'type': 'SplitsOp',
-                'options': {
-                    'split_map': {
-                        'a': 0.5,
-                        'b': 0.5
-                    }
-                }
-            }
-        },
-        {
-            'input': 'GOOD',
-            'output': {
-                'a': 'GOOD_0',
-                'b': 'GOOD_1'
-            },
-            'node': {
-                'type': 'SplitsOp',
-                'options': {
-                    'split_map': {
-                        'a': 0.5,
-                        'b': 0.5
-                    }
-                }
-            }
-        },
-        {
-            'input': ['BAD_0', 'GOOD_0'],
-            'output': 'train',
-            'node': {
-                'type': 'AddOp',
-                'options': {}
-            }
-        },
-        {
-            'input': ['BAD_1', 'GOOD_1'],  # 'SUPER'],
-            'output': 'testval',
-            'node': {
-                'type': 'AddOp',
-                'options': {}
-            }
-        },
-        {
-            'input': 'testval',
-            'output': '_t',
-            'node': {
-                'type': 'ShuffleOp',
-                'options': {
-                    'seed': -1
-                }
-            }
-        },
-        {
-            'input': '_t',
-            'output': '_t2',
-            'node': {
-                'type': 'SubsampleOp',
-                'options': {
-                    'factor': 2
-                }
-            }
-        },
-        {
-            'input': '_t2',
-            'output': {
-                'a': 'test',
-                'b': 'val'
-            },
-            'node': {
-                'type': 'SplitsOp',
-                'options': {
-                    'split_map': {
-                        'a': 0.5,
-                        'b': 0.5
-                    }
-                }
-            }
-        },
-        # {
-        #     'input': 's',
-        #     'output': 's',
-        #     'node': {
-        #         'type': 'SubsampleOp',
-        #         'options': {
-        #             'factor': 0.5
-        #         }
-        #     }
-        # },
-        # {
-        #     'input': 's',
-        #     'output': {
-        #         'a': 'train',
-        #         'b': 'val',
-        #         'c': 'test',
-        #     },
-        #     'node': {
-        #         'type': 'SplitsOp',
-        #         'options': {
-        #             'split_map': {
-        #                 'a': 0.8,
-        #                 'b': 0.1,
-        #                 'c': 0.1
-        #             }
-        #         }
-        #     }
-        # }
-    ]
-}
-
-
-g = nx.DiGraph()
-
-modules = pipe['modules']
-random.shuffle(modules)
-nodes = []
-for m in pipe['modules']:
-
-    node = PipeNode(
-        input_ports=m['input'],
-        output_ports=m['output'],
-        node=m['node']
-    )
-    nodes.append(node)
-
-    print("NODE", node.id, node.in_ports(), node.out_ports())
-
-    [g.add_edge(DataNode(x), node) for x in node.in_ports()]
-    [g.add_edge(node, DataNode(x)) for x in node.out_ports()]
-
-
-# nx.draw_spring(g, with_labels=True)
-# plt.show()
-
-def prepare_in_data(node: PipeNode, cache: dict):
-    v = node.input_ports
-    if v is None:
-        return None
-    elif isinstance(v, str):
-        return cache[v]
-    elif isinstance(v, list):
-        return [cache[x] for x in v]
-    elif isinstance(v, dict):
-        return {k: cache[v] for k, v in v.items()}
-    else:
-        raise NotImplementedError(f'{type(v)}')
-
-
-def prepare_out_data(o: Union[SamplesSequence, list, dict], node: PipeNode, cache: dict):
-    v = node.output_ports
-    if v is None:
-        pass
-    elif isinstance(v, str):
-        cache[v] = o
-    elif isinstance(v, list):
-        for idx, name in enumerate(v):
-            cache[name] = o[idx]
-    elif isinstance(v, dict):
-        for k, name in v.items():
-            cache[name] = o[k]
-    else:
-        raise NotImplementedError(f'{type(v)}')
-
-
-_cache = {}
-A = nx.topological_sort(g)
-# for a in A:
-#     print("N: ", a)
-for a in A:
-    print("N: ", a)
-    if isinstance(a, PipeNode):
-        if a.node['type'] != 'Source':
-            op = SequenceOpFactory.create(a.node)
-            data = prepare_in_data(a, _cache)
-            out = op(data)
-            prepare_out_data(out, a, _cache)
-        else:
-            _cache[a.output_ports] = a.node['data']
-
-
-for k, v in _cache.items():
-    print(k, len(v))
-
-A = to_agraph(g)
-print(A)
-A.layout('dot')
-fname = f'{tempfile.NamedTemporaryFile().name}.png'
-print(fname)
-A.draw(fname)
-os.system(f'open {fname}')
diff --git a/examples/garbage/pipeline_language/example.py b/examples/garbage/pipeline_language/example.py
deleted file mode 100644
index de5e51f..0000000
--- a/examples/garbage/pipeline_language/example.py
+++ /dev/null
@@ -1,22 +0,0 @@
-class Port:
-    pass
-
-    def __le__(self, x):
-        pass
-
-
-class Node:
-
-    def __init__(self, **kwargs) -> None:
-        self.input = [Port()] * 100
-        self.output = [Port()] * 100
-
-    def __ge__(self, x):
-        print(self, '->', x)
-
-
-alfa = Node(folder='/tmp')
-beta = Node(folder='/tmp')
-sum = Node()
-
-alfa > sum.input[0]
diff --git a/examples/garbage/pipeline_language/example.txt b/examples/garbage/pipeline_language/example.txt
deleted file mode 100644
index 6acb8c9..0000000
--- a/examples/garbage/pipeline_language/example.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-flow {
-  
-  a = Reader("/tmp/");
-  b = Reader($2);
-
-  out_train = Writer("/tmp/");
-  out_test = Writer("/tmp/");
-  out_val = Writer("/tmp/");
-
-  sum = OperationSum();
-  split = OperationSplit(a=0.2,b=0.8,);
-  split_by = OperationSplitByQuery(`a`);
-
-  a -> sum;
-  b -> sum;
-
-  sum -> split_by;
-
-  split_by:out[0] -> out_train;
-  split_by:out[1] -> split;
-
-  split:out[a] -> out_val;
-  split:out[b] -> out_test;
-
-}
\ No newline at end of file
diff --git a/examples/garbage/pipeline_language/languange_example.py b/examples/garbage/pipeline_language/languange_example.py
deleted file mode 100644
index 623f666..0000000
--- a/examples/garbage/pipeline_language/languange_example.py
+++ /dev/null
@@ -1,71 +0,0 @@
-from textx import metamodel_from_file
-
-
-class Reader:
-
-    def __init__(self, parent, folder):
-        self.parent = parent
-        self.folder = folder
-        print("NEW READER", self.folder)
-
-
-class Link:
-
-    def __init__(
-        self,
-        parent,
-        start,
-        end
-    ) -> None:
-        pass
-
-
-class Node:
-
-    def __init__(self, parent, name, port) -> None:
-        self.name = name
-        self.port = port
-
-
-class Port:
-
-    def __init__(self, parent, name, arity) -> None:
-        self.name = name
-        self.arity = arity
-
-
-class Arity:
-
-    def __init__(self, parent, idx, name):
-        self.parent = parent
-        self.idx = idx
-        self.name = name
-        #print("ARITY CREATED", parent, idx, name)
-
-
-class Declaration:
-
-    def __init__(self, parent, name, operation) -> None:
-        self.name = name
-        self.operation = operation
-
-
-class OperationSplitByQuery:
-
-    def __init__(self, parent, name, query) -> None:
-        self.name = name
-        self.query = query
-
-
-hello_meta = metamodel_from_file(
-    'model.tx',
-    classes=[Link, Node, Arity, Declaration, OperationSplitByQuery, Reader]
-)
-
-example_hello_model = hello_meta.model_from_file('example.txt')
-
-for c in example_hello_model.rows:
-    print(c, type(c))
-    if isinstance(c, Link):
-        link = c
-        print("LINK", link.start, link.end)
diff --git a/examples/garbage/pipeline_language/model.tx b/examples/garbage/pipeline_language/model.tx
deleted file mode 100644
index e8f23c6..0000000
--- a/examples/garbage/pipeline_language/model.tx
+++ /dev/null
@@ -1,92 +0,0 @@
-
-Global:
-  'flow' '{'
-    rows *= GenericRow
-  '}'
-;
-
-GenericRow:
-  Link | Declaration | Comment
-;
-
-Comment:
- '#' Text '\n'
-;
-
-Declaration:
-  name = NSTRING '=' operation = Operation ';'
-;
-
-OperationName:
-  'OperationSum' | 
-  'OperationSubsample' | 
-  'OperationSplit'  | 
-  'Reader' | 
-  'Writer'
-;
-
-OperationSum: name ='OperationSum' '(' arguments *= Argument ')';
-OperationSplit: name ='OperationSplit' '(' arguments *= Argument')';
-OperationSplitByQuery: name ='OperationSplitByQuery' '(' query = ASTRING ')';
-Reader: 'Reader' '(' folder = FOLDER ')';
-Writer: 'Writer' '(' folder = FOLDER ')';
-
-
-FOLDER: QSTRING | Placeholder;
-
-Operation:
-  OperationSum | OperationSplit | OperationSplitByQuery | Reader | Writer
-;
-
-Link:
-  start=Node
-  '->'
-  end=Node
-  ';'
-;
-
-Node:
-  name = NSTRING
-  port=Port?
-  
-;
-
-Port:
-  ':' name= NSTRING
-  arity=Arity?
-;
-
-Arity:
-  '['  idx=/[0-9]+/? name=NSTRING? ']'
-;
-
-
-
-ASTRING:
-  /([`])(?:(?=(\\?))\2.)*?\1/ 
-;
-
-QSTRING:
-  /(["'])(?:(?=(\\?))\2.)*?\1/ 
-;
-
-NSTRING:
-  /[A-Za-z0-9_.]+/
-;
-
-VariableValue:
-  NSTRING | Placeholder 
-;
-
-Placeholder:
-  name = /\$[A-Za-z0-9_]+/
-;
-
-
-Argument:
-  name = NSTRING '=' value = VariableValue ','
-;
-
-Text:
-  content = /[^\n]*/
-;
diff --git a/examples/garbage/pipes.py b/examples/garbage/pipes.py
deleted file mode 100644
index 7b0d2a4..0000000
--- a/examples/garbage/pipes.py
+++ /dev/null
@@ -1,121 +0,0 @@
-from choixe.spooks import Spook
-from pipelime.sequences.pipes import NodeGraph
-
-graph = {
-    "__spook__": "NodeGraph",
-    "options": {
-        "nodes": [
-            {
-                "__spook__": "ReaderNode",
-                "options": {
-                    "output_data": "A",
-                    "reader": {
-                        "__spook__": "UnderfolderReader",
-                        "options": {
-                            "folder": "/Users/daniele/Downloads/lego_dataset/lego_00",
-                        },
-                    },
-                },
-            },
-            {
-                "__spook__": "ReaderNode",
-                "options": {
-                    "output_data": "B",
-                    "reader": {
-                        "__spook__": "UnderfolderReader",
-                        "options": {
-                            "folder": "/Users/daniele/Downloads/lego_dataset/lego_01"
-                        },
-                    },
-                },
-            },
-            {
-                "__spook__": "OperationNode",
-                "options": {
-                    "input_data": "A",
-                    "output_data": "A1",
-                    "operation": {
-                        "__spook__": "OperationFilterByQuery",
-                        "options": {"query": '`metadata.tag` == "black"'},
-                    },
-                },
-            },
-            {
-                "__spook__": "OperationNode",
-                "options": {
-                    "input_data": ["A1", "B"],
-                    "output_data": "C",
-                    "operation": {"__spook__": "OperationSum", "options": {}},
-                },
-            },
-            {
-                "__spook__": "OperationNode",
-                "options": {
-                    "input_data": "C",
-                    "output_data": "D",
-                    "operation": {
-                        "__spook__": "OperationFilterByQuery",
-                        "options": {"query": '`metadata.tag` == "black"'},
-                    },
-                },
-            },
-            {
-                "__spook__": "OperationNode",
-                "options": {
-                    "input_data": "D",
-                    "output_data": "D1",
-                    "operation": {
-                        "__spook__": "OperationSubsample",
-                        "options": {"factor": 0.1},
-                    },
-                },
-            },
-            {
-                "__spook__": "OperationNode",
-                "options": {
-                    "input_data": "D1",
-                    "output_data": {"a": "train", "b": "test", "c": "val"},
-                    "operation": {
-                        "__spook__": "OperationSplits",
-                        "options": {"split_map": {"a": 0.45, "b": 0.45, "c": 0.1}},
-                    },
-                },
-            },
-            {
-                "__spook__": "WriterNode",
-                "options": {
-                    "input_data": "train",
-                    "writer": {
-                        "__spook__": "UnderfolderWriter",
-                        "options": {"folder": "/tmp/zizzino/train"},
-                    },
-                },
-            },
-            {
-                "__spook__": "WriterNode",
-                "options": {
-                    "input_data": "test",
-                    "writer": {
-                        "__spook__": "UnderfolderWriter",
-                        "options": {"folder": "/tmp/zizzino/test"},
-                    },
-                },
-            },
-            {
-                "__spook__": "WriterNode",
-                "options": {
-                    "input_data": "val",
-                    "writer": {
-                        "__spook__": "UnderfolderWriter",
-                        "options": {"folder": "/tmp/zizzino/val"},
-                    },
-                },
-            },
-        ]
-    },
-}
-
-
-graph: NodeGraph = Spook.create(graph)
-
-graph.execute()
diff --git a/examples/garbage/writer.py b/examples/garbage/writer.py
deleted file mode 100644
index db37599..0000000
--- a/examples/garbage/writer.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import re
-from pipelime.sequences.readers.filesystem import UnderfolderReader
-from pipelime.sequences.writers.filesystem import UnderfolderWriter
-from pipelime.sequences.operations import OperationFilterByQuery, OperationSubsample
-
-folder = '/Users/daniele/Downloads/lego_dataset/lego_00'
-d = UnderfolderReader(folder=folder, copy_root_files=True)
-print(len(d))
-
-
-op = OperationFilterByQuery(query='`metadata.tag` == "black"')
-op2 = OperationSubsample(factor=0.1)
-d = op2(op(d))
-print(len(d))
-
-extensions_map = {
-    '.*image.*': 'jpg',
-    '.*mask.*': 'png',
-    '.*pose.*|.*keypoint.*': 'txt',
-    '.*metadata.*|.*camera.*|.*charuco.*': 'yml',
-}
-
-w = UnderfolderWriter(
-    folder='/tmp/gino',
-    root_files_keys=['.*camera.*|.*charuco.*|.*keypoints.*'],
-    extensions_map=None
-)
-w(d)
-
-# print(re.match('.*image.*', '_image0'))
-# writer = UnderfolderWriter(
-#     folder='/tmp/gino',
-#     root_files_keys=['camera', 'charuco', 'keypoints'],
-#     extensions_map={
-#     }
-# )
-# print(len(d))
-
-
-# writer(d)
-
-# for sample in d:
-#     print(list(sample.keys()))
-#     break
diff --git a/examples/h5py/compare_speed.py b/examples/h5py/compare_speed.py
index 4bf46c7..4e2f603 100644
--- a/examples/h5py/compare_speed.py
+++ b/examples/h5py/compare_speed.py
@@ -1,16 +1,14 @@
-from pipelime.sequences.writers.h5 import H5Writer
 from pipelime.sequences.readers.filesystem import UnderfolderReader
 from pipelime.sequences.readers.h5 import H5Reader
-import tempfile
-import rich
-from pathlib import Path
 import time
 
 
 # ➡️➡️➡️ Reader
 readers = {
-    'underfolder': UnderfolderReader(folder='/tmp/whilly', copy_root_files=True, lazy_samples=True),
-    'h5': H5Reader(filename='/tmp/dataset.h5', copy_root_files=True, lazy_samples=True)
+    "underfolder": UnderfolderReader(
+        folder="/tmp/whilly", copy_root_files=True, lazy_samples=True
+    ),
+    "h5": H5Reader(filename="/tmp/dataset.h5", copy_root_files=True, lazy_samples=True),
 }
 
 for name, reader in readers.items():
diff --git a/examples/h5py/underfolder_to_h5.py b/examples/h5py/underfolder_to_h5.py
index 8386274..53c9a81 100644
--- a/examples/h5py/underfolder_to_h5.py
+++ b/examples/h5py/underfolder_to_h5.py
@@ -5,11 +5,13 @@
 from pathlib import Path
 
 # ➡️➡️➡️ Reader
-reader = UnderfolderReader(folder='../data/underfolder/example_dataset', copy_root_files=True)
+reader = UnderfolderReader(
+    folder="../data/underfolder/example_dataset", copy_root_files=True
+)
 reader_template = reader.get_reader_template()
 
 # ➡️➡️➡️ Writer with manual template input
-writer_filename = Path(tempfile.mkdtemp()) / 'dataset.h5'
+writer_filename = Path(tempfile.mkdtemp()) / "dataset.h5"
 writer = H5Writer(
     filename=writer_filename,
     root_files_keys=reader_template.root_files_keys,
diff --git a/examples/pipelines/pipeline_example.py b/examples/pipelines/pipeline_example.py
deleted file mode 100644
index 2230ff7..0000000
--- a/examples/pipelines/pipeline_example.py
+++ /dev/null
@@ -1,101 +0,0 @@
-
-
-import os
-from pipelime.sequences.writers.filesystem import UnderfolderWriter
-from pipelime.sequences.operations import OperationFilterByQuery, OperationOrderBy, OperationResetIndices, OperationSubsample, OperationSum
-from pipelime.sequences.readers.filesystem import UnderfolderReader
-from pipelime.tools.idgenerators import IdGeneratorInteger, IdGeneratorUUID
-from pipelime.sequences.pipes import NodeGraph, OperationNode, ReaderNode, WriterNode
-
-
-folder_a = '/home/daniele/Desktop/experiments/2021-02-10.TMLightsExperiments/datasets/tap_00'
-folder_b = '/home/daniele/Desktop/experiments/2021-02-10.TMLightsExperiments/datasets/tap_01'
-
-idg = IdGeneratorUUID()
-subsampling = 1.0
-nodes = [
-
-    ReaderNode(
-        id=idg.generate(),
-        output_data='A',
-        reader=UnderfolderReader(
-            folder=folder_a
-        )
-    ),
-
-    ReaderNode(
-        id=idg.generate(),
-        output_data='B',
-        reader=UnderfolderReader(
-            folder=folder_b
-        )
-    ),
-    OperationNode(
-        id=idg.generate(),
-        input_data='A',
-        output_data='A_',
-        operation=OperationSubsample(
-            factor=subsampling
-        )
-    ),
-    OperationNode(
-        id=idg.generate(),
-        input_data='B',
-        output_data='B_',
-        operation=OperationSubsample(
-            factor=subsampling
-        )
-    ),
-    OperationNode(
-        id=idg.generate(),
-        input_data=['A_', 'B_'],
-        output_data='X',
-        operation=OperationSum()
-    ),
-
-    OperationNode(
-        id=idg.generate(),
-        input_data='X',
-        output_data='X_filtered',
-        operation=OperationFilterByQuery(
-            query='`metadata.counter` >= 0'
-        )
-    ),
-    OperationNode(
-        id=idg.generate(),
-        input_data='X_filtered',
-        output_data='X_ordered',
-        operation=OperationOrderBy(
-            order_keys=['counter']
-        )
-    ),
-    OperationNode(
-        id=idg.generate(),
-        input_data='X_ordered',
-        output_data='X_reset',
-        operation=OperationResetIndices(
-            generator=IdGeneratorInteger()
-        )
-    ),
-    WriterNode(
-        id=idg.generate(),
-        input_data='X_reset',
-        writer=UnderfolderWriter(
-            folder='/tmp/zzizzino',
-            root_files_keys=['camera', 'charuco'],
-            extensions_map={
-                '.*image.*': 'jpg',
-                '.*pose.*': 'txt',
-                '.*camera.*|.*charuco.*|.*metadata.*': 'yml',
-            }
-        )
-    )
-]
-
-graph = NodeGraph(nodes=nodes)
-
-
-graph.draw_to_file('/tmp/graph.png')
-os.system('eog /tmp/graph.png')
-
-# graph.execute()
diff --git a/examples/piper/README.md b/examples/piper/README.md
new file mode 100644
index 0000000..da88dcd
--- /dev/null
+++ b/examples/piper/README.md
@@ -0,0 +1,7 @@
+# Piper 
+
+## [Compile](compile/)
+## [Draw](draw/)
+## [Execute](execute/)
+
+
diff --git a/examples/piper/compile/README.md b/examples/piper/compile/README.md
new file mode 100644
index 0000000..4d70e1a
--- /dev/null
+++ b/examples/piper/compile/README.md
@@ -0,0 +1,21 @@
+# Piper: Compile
+
+A DAG file can contains placeholders and repetions blocks. The compile procedure will replace the placeholders with the values picked from the configuration file and replace the repetions blocks with valid values.
+
+From this folder execute the following command:
+
+```
+pipelime piper compile -i dag.yml -p params.yml
+```
+
+This command will print the compiled dag file to stdout. The compiled dag file can be saved to a file (**$COMPILED_FILE**):
+
+```
+pipelime piper compile -i dag.yml -p params.yml -o $COMPILED_FILE
+```
+
+If you try to launch the compile tool without providing a configuration file, the tool will fail due to missing replacement for the placeholders:
+
+```
+pipelime piper compile -i dag.yml
+```
diff --git a/examples/piper/compile/dag.yml b/examples/piper/compile/dag.yml
new file mode 100644
index 0000000..4e042c1
--- /dev/null
+++ b/examples/piper/compile/dag.yml
@@ -0,0 +1,169 @@
+# --------------------------------------------------------------------------------------
+# ROOT key is "nodes", a mapping of {node_name:str -> node_params:dict}
+nodes:
+  # ------------------------------------------------------------------------------------
+  # GENERATOR NODE, creates several ToyDatasets as input
+  # ------------------------------------------------------------------------------------
+  
+  # ↓ this is the node name, has to be unique
+  generator:
+    # ↓ this keyword indicates that the node will be repeated several times based on 
+    # ↓ values in the 'items' field
+    foreach:
+      # ↓ 'items' field is a list of values to be used in the 'foreach' loop
+      items: $var(params.flags)
+      # ↓ 'do' field contains the real node definition with placeholders for the 
+      # ↓ elements that will be replaced by the 'foreach' loop (e.e. $iter(item) will
+      # ↓ be replaced by the value of 'item' in the 'foreach' loop)
+      do:
+        command: pipelime datasets generate_toy_dataset
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_generated
+        args:
+          suffix: _$iter(item)
+          size: $var(params.generated_dataset_size)
+        # ↓ these are the schemas of the outputs of the node
+        outputs_schema:
+          output_folder: $var(params.schemas.generated)
+  # ------------------------------------------------------------------------------------
+  # FILTER_KEYS NODE, filter some keys for each sample, for each dataset
+  # ------------------------------------------------------------------------------------
+  filter_keys:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime underfolder filter_keys
+        inputs:
+          input_folder: $var(params.output_folder)/$iter(item)_generated
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_filtered
+        args:
+          keys:
+            foreach:
+              items: $var(params.filter_keys)
+              do: $argiter(item)_$iter(item)
+  # ------------------------------------------------------------------------------------
+  # SUBSAMPLE NODE, reduce size of each dataset
+  # ------------------------------------------------------------------------------------
+  subsample:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime underfolder subsample
+        inputs:
+          input_folder: $var(params.output_folder)/$iter(item)_filtered
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_subsampled
+        args:
+          factor: 0.5
+        
+  # ------------------------------------------------------------------------------------
+  # REMAP_KEYS NODE, rename keys of each dataset into a common naming scheme
+  # ------------------------------------------------------------------------------------
+  remap_keys:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime underfolder remap_keys
+        inputs:
+          input_folder: $var(params.output_folder)/$iter(item)_subsampled
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_remapped
+        args:
+          keys@0:
+            foreach:
+              items: $var(params.filter_keys)
+              do: $argiter(item)_$iter(item)
+          keys@1:
+            foreach:
+              items: $var(params.filter_keys)
+              do: $argiter(item)
+          remove: {}
+        # ↓ these are the schemas of the inputs of the node
+        inputs_schema:
+          input_folder: $var(params.schemas.generated)
+  # ------------------------------------------------------------------------------------
+  # DETECTION NODE, launch custom detection node on each input dataset
+  # ------------------------------------------------------------------------------------
+  detection:
+    # ↓ this is the base command to be executed
+    command: $var(params.custom_command)
+    # ↓ this is mapping of inputs arguments {arg_name:str -> values:str}
+    inputs:
+      input_folders:
+        # ↓ this is a for each for arguments itself, it is a list of values to be used
+        # ↓ in the 'foreach' loop with the same behavior as the 'foreach' keyword used
+        # ↓ above for nodes
+        foreach:
+          items: $var(params.flags)
+          do: $var(params.output_folder)/$argiter(item)_remapped
+    # ↓ this is mapping of outputs arguments {arg_name:str -> values:str}
+    outputs:
+      output_folders:
+        foreach:
+          items: $var(params.flags)
+          do: $var(params.output_folder)/$argiter(item)_detected
+    # ↓ this is mapping of generic arguments {arg_name:str -> values:str}     
+    args:
+      fake_time: 0.01
+    inputs_schema:
+      input_folders: $var(params.schemas.remapped)
+    outputs_schema:
+      output_folders: $var(params.schemas.detected)
+  # ------------------------------------------------------------------------------------
+  # SUMMATION NODE, sum each dataset into a single dataset
+  # ------------------------------------------------------------------------------------
+  summation:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders: 
+        foreach:
+          items: $var(params.flags)
+          do: $var(params.output_folder)/$argiter(item)_detected
+    outputs:
+      output_folder: $var(params.output_folder)/summed
+  # ------------------------------------------------------------------------------------
+  # SPLIT NODE, split datasets into train, validation and test
+  # ------------------------------------------------------------------------------------
+  split:
+    command: pipelime underfolder split
+    args:
+      splits:
+        foreach:
+          items: $var(params.splits)
+          do: $argiter(item.p)
+    inputs:
+      input_folder: $var(params.output_folder)/summed
+    outputs:
+      output_folders:
+        foreach:
+          items: $var(params.splits)
+          do: $var(params.output_folder)/split_$argiter(item.name)
+  # ------------------------------------------------------------------------------------
+  # SHUFFLE NODE, useless shuffle after split
+  # ------------------------------------------------------------------------------------
+  shuffle:
+    foreach:
+      items: $var(params.splits)
+      do:
+        command: pipelime underfolder shuffle
+        inputs:
+          input_folder: $var(params.output_folder)/split_$iter(item.name)
+        outputs:
+          output_folder: $var(params.output_folder)/split_$iter(item.name)_shuffled
+        args:
+          seed: 10
+  # ------------------------------------------------------------------------------------
+  # SUMUP NODE, final merge
+  # ------------------------------------------------------------------------------------
+  sumup:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders:
+        foreach:
+          items: $var(params.splits)
+          do: $var(params.output_folder)/split_$argiter(item.name)_shuffled
+    outputs:
+      output_folder: $var(params.final_folder)
+
+
diff --git a/examples/piper/compile/params.yml b/examples/piper/compile/params.yml
new file mode 100644
index 0000000..879ea71
--- /dev/null
+++ b/examples/piper/compile/params.yml
@@ -0,0 +1,24 @@
+params:
+  filter_keys:
+  - image
+  - inst
+  - metadata
+  flags:
+  - A
+  - B
+  - C
+  custom_command: python fake_detector.py
+  output_folder: /tmp/pipe/output
+  final_folder: /tmp/pipe/final
+  generated_dataset_size: 10
+  splits:
+  - name: train
+    p: 0.8
+  - name: validation
+    p: 0.1
+  - name: test
+    p: 0.1
+  schemas:
+    generated: /path/to/validation.schema
+    remapped: /path/to/validation_remapped.schema
+    detected: /path/to/validation_detected.schema
diff --git a/examples/piper/draw/README.md b/examples/piper/draw/README.md
new file mode 100644
index 0000000..b729e16
--- /dev/null
+++ b/examples/piper/draw/README.md
@@ -0,0 +1,30 @@
+# Piper: Draw
+
+A DAG file can be drawn using a specific drawing backend. The backend can be specified using the **-b** option (**-help** to list avaiable backends). 
+
+## Draw Graphviz version to PDF 
+
+In order to generate a PDF representation of the DAG using [Graphviz](https://graphviz.org/) (or better [PyGraphviz](https://pygraphviz.github.io/)), use the following command:
+
+```
+pipelime piper draw -i drawable_dag.yml -b graphviz -o /tmp/drawable_dag.pdf
+```
+
+To auto-open file after execution use the following command:
+
+```
+pipelime piper draw -i drawable_dag.yml -b graphviz -o /tmp/drawable_dag.pdf --open
+```
+
+Try also **PNG** ...
+
+
+## Draw Mermaid to Markdown
+
+In order to generate a Markdown representation of the DAG with [Mermaid](https://github.com/mermaid-js/mermaid) syntax, use the following command:
+
+```
+pipelime piper draw -i drawable_dag.yml -b mermaid -o /tmp/drawable_dag.md --open
+```
+
+Try also **PNG** ...
diff --git a/examples/piper/draw/drawable_dag.yml b/examples/piper/draw/drawable_dag.yml
new file mode 100644
index 0000000..768ca0d
--- /dev/null
+++ b/examples/piper/draw/drawable_dag.yml
@@ -0,0 +1,213 @@
+nodes:
+  detection:
+    args:
+      fake_time: 0.01
+    command: python fake_detector.py
+    inputs:
+      input_folders:
+      - /tmp/pipe/output/A_remapped
+      - /tmp/pipe/output/B_remapped
+      - /tmp/pipe/output/C_remapped
+      checkpoint: checkpoint.ckpt
+      decoder_config: demo_config.yml
+      random_dataset: /tmp/random_dataset
+    inputs_schema:
+      input_folders: /path/to/validation_remapped.schema
+    outputs:
+      output_folders:
+      - /tmp/pipe/output/A_detected
+      - /tmp/pipe/output/B_detected
+      - /tmp/pipe/output/C_detected
+    outputs_schema:
+      output_folders: /path/to/validation_detected.schema
+  filter_keys@A:
+    args:
+      keys:
+      - image_A
+      - inst_A
+      - metadata_A
+    command: pipelime underfolder filter_keys
+    inputs:
+      input_folder: /tmp/pipe/output/A_generated
+    outputs:
+      output_folder: /tmp/pipe/output/A_filtered
+  filter_keys@B:
+    args:
+      keys:
+      - image_B
+      - inst_B
+      - metadata_B
+    command: pipelime underfolder filter_keys
+    inputs:
+      input_folder: /tmp/pipe/output/B_generated
+    outputs:
+      output_folder: /tmp/pipe/output/B_filtered
+  filter_keys@C:
+    args:
+      keys:
+      - image_C
+      - inst_C
+      - metadata_C
+    command: pipelime underfolder filter_keys
+    inputs:
+      input_folder: /tmp/pipe/output/C_generated
+    outputs:
+      output_folder: /tmp/pipe/output/C_filtered
+  generator@A:
+    args:
+      size: 10
+      suffix: _A
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/A_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@B:
+    args:
+      size: 10
+      suffix: _B
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/B_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@C:
+    args:
+      size: 10
+      suffix: _C
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/C_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  remap_keys@A:
+    args:
+      keys:
+      - - image_A
+        - image
+      - - inst_A
+        - inst
+      - - metadata_A
+        - metadata
+      remove: {}
+    command: pipelime underfolder remap_keys
+    inputs:
+      input_folder: /tmp/pipe/output/A_subsampled
+    inputs_schema:
+      input_folder: /path/to/validation.schema
+    outputs:
+      output_folder: /tmp/pipe/output/A_remapped
+  remap_keys@B:
+    args:
+      keys:
+      - - image_B
+        - image
+      - - inst_B
+        - inst
+      - - metadata_B
+        - metadata
+      remove: {}
+    command: pipelime underfolder remap_keys
+    inputs:
+      input_folder: /tmp/pipe/output/B_subsampled
+    inputs_schema:
+      input_folder: /path/to/validation.schema
+    outputs:
+      output_folder: /tmp/pipe/output/B_remapped
+  remap_keys@C:
+    args:
+      keys:
+      - - image_C
+        - image
+      - - inst_C
+        - inst
+      - - metadata_C
+        - metadata
+      remove: {}
+    command: pipelime underfolder remap_keys
+    inputs:
+      input_folder: /tmp/pipe/output/C_subsampled
+    inputs_schema:
+      input_folder: /path/to/validation.schema
+    outputs:
+      output_folder: /tmp/pipe/output/C_remapped
+  shuffle@0:
+    args:
+      seed: 10
+    command: pipelime underfolder shuffle
+    inputs:
+      input_folder: /tmp/pipe/output/split_train
+    outputs:
+      output_folder: /tmp/pipe/output/split_train_shuffled
+  shuffle@1:
+    args:
+      seed: 10
+    command: pipelime underfolder shuffle
+    inputs:
+      input_folder: /tmp/pipe/output/split_validation
+    outputs:
+      output_folder: /tmp/pipe/output/split_validation_shuffled
+  shuffle@2:
+    args:
+      seed: 10
+    command: pipelime underfolder shuffle
+    inputs:
+      input_folder: /tmp/pipe/output/split_test
+    outputs:
+      output_folder: /tmp/pipe/output/split_test_shuffled
+  split:
+    args:
+      splits:
+      - 0.8
+      - 0.1
+      - 0.1
+    command: pipelime underfolder split
+    inputs:
+      input_folder: /tmp/pipe/output/summed
+    outputs:
+      output_folders:
+      - /tmp/pipe/output/split_train
+      - /tmp/pipe/output/split_validation
+      - /tmp/pipe/output/split_test
+  subsample@A:
+    args:
+      factor: 0.5
+    command: pipelime underfolder subsample
+    inputs:
+      input_folder: /tmp/pipe/output/A_filtered
+    outputs:
+      output_folder: /tmp/pipe/output/A_subsampled
+  subsample@B:
+    args:
+      factor: 0.5
+    command: pipelime underfolder subsample
+    inputs:
+      input_folder: /tmp/pipe/output/B_filtered
+    outputs:
+      output_folder: /tmp/pipe/output/B_subsampled
+  subsample@C:
+    args:
+      factor: 0.5
+    command: pipelime underfolder subsample
+    inputs:
+      input_folder: /tmp/pipe/output/C_filtered
+    outputs:
+      output_folder: /tmp/pipe/output/C_subsampled
+  summation:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders:
+      - /tmp/pipe/output/A_detected
+      - /tmp/pipe/output/B_detected
+      - /tmp/pipe/output/C_detected
+    outputs:
+      output_folder: /tmp/pipe/output/summed
+  sumup:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders:
+      - /tmp/pipe/output/split_train_shuffled
+      - /tmp/pipe/output/split_validation_shuffled
+      - /tmp/pipe/output/split_test_shuffled
+    outputs:
+      output_folder: /tmp/pipe/final
diff --git a/pipelime/augmentations/__init__.py b/examples/piper/draw/params.yml
similarity index 100%
rename from pipelime/augmentations/__init__.py
rename to examples/piper/draw/params.yml
diff --git a/examples/piper/execute/.gitignore b/examples/piper/execute/.gitignore
new file mode 100644
index 0000000..9b1960e
--- /dev/null
+++ b/examples/piper/execute/.gitignore
@@ -0,0 +1,1 @@
+output/
\ No newline at end of file
diff --git a/examples/piper/execute/README.md b/examples/piper/execute/README.md
new file mode 100644
index 0000000..679f7fb
--- /dev/null
+++ b/examples/piper/execute/README.md
@@ -0,0 +1,59 @@
+# Piper: Execute
+
+A DAG file can be executed with a specific backend. The backend can be specified using the **-b** option (**-help** to list avaiable backends). 
+
+To execute the demo graph execute (note that the output will be written to the **output** subfolder , excluded from git, pay attention to not produce in any other folder):
+
+```
+pipelime piper execute -i dag.yml -p params.yml
+```
+
+The demo graph should execute this kind of dag:
+
+
+```mermaid
+  graph TD;
+      id0[(ToyGenerator_0)]-->Subsample_0;
+      id1[(ToyGenerator_1)]-->Subsample_1;
+      id2[(ToyGenerator_2)]-->Subsample_2;
+      Subsample_0-->Detection_0;
+      Subsample_1-->Detection_1;
+      Subsample_2-->Detection_2;
+      Detection_0-->Summation;
+      Detection_1-->Summation;
+      Detection_2-->Summation;
+```
+
+As described in **dag.yml** file. The Dag shows the execution of mixed underfolder baked
+nodes and user nodes (as **fake_detector.py**). Each node has to be a **PiperCommand** in 
+order to be executed in a Piper Dag.
+
+The dag also has the capability to validate intermediate results. This is done by using *inputs_schema*/*outputs_schema* in nodes. See the *outputs_schema* in the *detection* node 
+in **dag.yml** file. 
+
+
+
+## Monitoring Execution [EXPERIMENTAL]
+
+Befour launching the execution, you can monitor the execution using the **watcher_demo.py** script. The watcher demo accept a "TOKEN" as argument. The token is used to identify the execution, each nodes should share the same token. In a terminal launch:
+
+```
+python watcher_demo.py --token token_eyecan
+```
+
+Then relaunch the DAG with the same token:
+
+```
+pipelime piper execute -i dag.yml -p params.yml -t token_eyecan
+```
+
+The Token **token_eyecan** can be any string you want.
+
+### Choosing the channel implementation 
+You can specify the type of backend to use (Choixe BulletinBoard by default) by setting the environment variable:
+
+```bash
+export PIPELIME_PIPER_CHANNEL_TYPE=$CH_TYPE
+```
+
+Where `$CH_TYPE` can be one of `"BULLETIN"` or `"REDIS"`.
diff --git a/examples/piper/execute/dag.yml b/examples/piper/execute/dag.yml
new file mode 100644
index 0000000..1d6bd37
--- /dev/null
+++ b/examples/piper/execute/dag.yml
@@ -0,0 +1,52 @@
+nodes:
+
+  ## GENERATE SAMPLES
+  generator:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime datasets generate_toy_dataset
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_generated
+        args:
+          size: $var(params.size)
+
+  ## SUBSAMPLE EACH DATASET
+  subsample:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime underfolder subsample
+        inputs:
+          input_folder: $var(params.output_folder)/$iter(item)_generated
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_subsampled
+        args:
+          factor: 0.5
+  
+  ## EXECUTE CUSTOM NODE ON EACH DATASET
+  detection:
+    foreach:
+        items: $var(params.flags)
+        do:
+          command: python fake_detector.py
+          inputs:
+            input_folder: $var(params.output_folder)/$iter(item)_subsampled 
+          outputs:
+            output_folder: $var(params.output_folder)/$iter(item)_detected
+          args:
+            fake_time: 0.01
+          outputs_schema:
+            output_folder: $var(params.schemas.detected)
+
+  ## SUM UP RESULTS
+  summation:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders: 
+        foreach:
+          items: $var(params.flags)
+          do: $var(params.output_folder)/$argiter(item)_detected
+    outputs:
+      output_folder: $var(params.output_folder)/summed
+  
\ No newline at end of file
diff --git a/examples/piper/execute/fake_detector.py b/examples/piper/execute/fake_detector.py
new file mode 100644
index 0000000..57956c6
--- /dev/null
+++ b/examples/piper/execute/fake_detector.py
@@ -0,0 +1,84 @@
+import click
+from pipelime.pipes.piper import Piper, PiperCommand
+from pipelime.tools.progress import pipelime_track
+
+
+@click.command("fake_detector")
+@click.option(
+    "-i",
+    "--input_folder",
+    type=click.Path(exists=True),
+    required=True,
+    help="The input folder",
+)
+@click.option(
+    "-o",
+    "--output_folder",
+    type=click.Path(),
+    required=True,
+    help="The output folder",
+)
+@click.option(
+    "-t",
+    "--fake_time",
+    type=float,
+    default=0.01,
+    help="Fake delay time",
+)
+@Piper.command(
+    inputs=["input_folder"],
+    outputs=["outuput_folder"],
+)
+def fake_detector(
+    input_folder: str,
+    output_folder: str,
+    fake_time: float,
+):
+
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+    from pipelime.sequences.samples import SamplesSequence
+    import time
+    import numpy as np
+
+    reader = UnderfolderReader(input_folder)
+
+    # ADD fake detections to each sample
+    out_samples = []
+    for sample in pipelime_track(
+        reader,
+        # This progess callback has as CHUNK_ID=0 on a total of 2. Beacause this
+        # should map the first part of a global progress bar associated to this node.
+        track_callback=PiperCommand.instance.generate_progress_callback(0, 2),
+    ):
+        sample["fake_detection"] = {
+            "keypoints": [np.random.randint(0, 100, (10, 4)).tolist()]
+        }
+        out_samples.append(sample)
+        time.sleep(fake_time)
+
+    # build out sequence
+    out_sequence = SamplesSequence(out_samples)
+
+    # add fake detections to the extensions map
+    template = reader.get_reader_template()
+    template.extensions_map.update({"fake_detection": "yml"})
+
+    # write out sequence
+    try:
+        writer = UnderfolderWriterV2(
+            output_folder,
+            reader_template=template,
+            file_handling=UnderfolderWriterV2.FileHandling.COPY_IF_NOT_CACHED,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+            # This progess callback has as CHUNK_ID =1 on a total of 2. Beacause this
+            # should map the second part of a global progress bar associated to this node.
+            progress_callback=PiperCommand.instance.generate_progress_callback(1, 2),
+        )
+        writer(out_sequence)
+    except Exception as e:
+        raise click.UsageError(str(e))
+
+
+if __name__ == "__main__":
+    fake_detector()
diff --git a/examples/piper/execute/params.yml b/examples/piper/execute/params.yml
new file mode 100644
index 0000000..a531b0a
--- /dev/null
+++ b/examples/piper/execute/params.yml
@@ -0,0 +1,7 @@
+params:
+  flags: ['A','B','C']
+  custom_command: python fake_detector.py
+  output_folder: output
+  size: 50
+  schemas:
+    detected: schema_detected.schema
diff --git a/examples/piper/execute/schema_detected.schema b/examples/piper/execute/schema_detected.schema
new file mode 100644
index 0000000..6ed3202
--- /dev/null
+++ b/examples/piper/execute/schema_detected.schema
@@ -0,0 +1,8 @@
+from schema import Optional
+
+deep = True
+schema = {
+    "image": object,
+    "fake_detection": {"keypoints": object},
+    Optional(str): object,
+}
diff --git a/examples/piper/execute/watcher_demo.py b/examples/piper/execute/watcher_demo.py
new file mode 100644
index 0000000..c095d12
--- /dev/null
+++ b/examples/piper/execute/watcher_demo.py
@@ -0,0 +1,112 @@
+import threading
+import time
+
+import click
+from pydantic import BaseModel
+import rich
+from rich.live import Live
+from rich.table import Table
+
+from pipelime.pipes.communication import PiperCommunicationChannelFactory
+
+
+def percentage_string(v: float):
+    color = "red"
+    if v > 0.33:
+        color = "yellow"
+    if v > 0.66:
+        color = "green"
+    return f"[{color}]{v*100:.1f}%[/{color}]"
+
+
+def generate_table(tasks_map: dict) -> Table:
+    """Make a new table."""
+    table = Table()
+    table.add_column("File")
+    table.add_column("Method")
+    table.add_column("ID")
+    table.add_column("Progress")
+
+    for task_id in tasks_map.keys():
+        progress = [percentage_string(v) for _, v in tasks_map[task_id].items()]
+        progress = " ‖ ".join(progress)
+
+        filename, method, unique = task_id.split(":")
+        table.add_row(
+            f"{filename}",
+            f"{method}",
+            f"{unique}",
+            f"{progress}",
+        )
+
+    return table
+
+
+class ChunkProgress(BaseModel):
+    id: str
+    chunk_index: int
+    progress: float
+
+    @property
+    def file(self):
+        return self.id.split(":")[0]
+
+    @property
+    def method(self):
+        return self.id.split(":")[1]
+
+    @property
+    def unique(self):
+        return self.id.split(":")[2]
+
+
+@click.command("piper_watcher")
+@click.option("-t", "--token", default="", help="Token to use for the piper.")
+def piper_watcher(token: str):
+
+    tasks_map = {}
+
+    def callback(data: dict):
+        if "_progress" in data["payload"]:
+            chunk_progress = ChunkProgress(
+                id=data["id"],
+                chunk_index=data["payload"]["_progress"]["chunk_index"],
+                progress=data["payload"]["_progress"]["progress_data"]["advance"]
+                / data["payload"]["_progress"]["progress_data"]["total"],
+            )
+
+            if chunk_progress.id not in tasks_map:
+                tasks_map[chunk_progress.id] = {}
+            if chunk_progress.chunk_index not in tasks_map[chunk_progress.id]:
+                tasks_map[chunk_progress.id][chunk_progress.chunk_index] = 0.0
+
+            tasks_map[chunk_progress.id][
+                chunk_progress.chunk_index
+            ] += chunk_progress.progress
+        elif "event" in data["payload"]:
+            event = data["payload"]["event"]
+
+            if event not in tasks_map:
+                tasks_map[event] = {0: 0.0}
+
+            tasks_map[event][0] += 1
+
+    def listener_thread():
+        channel = PiperCommunicationChannelFactory.create_channel(token)
+        rich.print(f"Channel class: {channel.__class__}")
+        channel.register_callback(callback)
+        channel.listen()
+
+    thread = threading.Thread(target=listener_thread, daemon=True)
+    thread.start()
+
+    with Live(generate_table(tasks_map), refresh_per_second=4) as live:
+        while True:
+            time.sleep(0.1)
+            live.update(generate_table(tasks_map))
+
+    thread.join()
+
+
+if __name__ == "__main__":
+    piper_watcher()
diff --git a/examples/proposal/verify/verify_dataset.py b/examples/proposal/verify/verify_dataset.py
new file mode 100644
index 0000000..ca3bfab
--- /dev/null
+++ b/examples/proposal/verify/verify_dataset.py
@@ -0,0 +1,116 @@
+from typing import Any
+import numpy as np
+from pydantic import BaseModel, validator
+from pipelime.sequences.stages import SampleStage
+from pipelime.sequences.samples import PlainSample, Sample, SamplesSequence
+from pipelime.sequences.operations import OperationStage
+import rich
+
+
+#  ___ _  _ ____    _  _  __   __  ____ _
+#   |  |__| |___    |\/| |  | |  \ |___ |
+#   |  |  | |___    |  | |__| |__/ |___ |___
+
+# The Pydantic ecosystem: https://pydantic-docs.helpmanual.io/usage/validators/
+
+
+class MetadataModel(BaseModel):
+    a: bool
+    b: float
+    c: str
+
+    @validator("c")
+    def _verify_c(cls, value):
+        if not value.startswith("hello"):
+            raise ValueError("C must start with hello")
+
+
+class SampleModel(BaseModel):
+    image: Any
+    metadata: MetadataModel
+    sample_id: int
+
+    @validator("image")
+    def _verify_image(cls, value):
+        if not isinstance(value, np.ndarray):
+            raise ValueError("image must be a numpy array")
+
+
+#  __ __ __ __ __ __ __ __ __ __ __ __
+
+
+#  ___ _  _ ____     ___ ___ __   ___ ____
+#   |  |__| |___    [__   | |__| | __ |___
+#   |  |  | |___    ___]  | |  | |__] |___
+
+
+class StageVerify(SampleStage):
+    class StageVerifyException(Exception):
+        pass
+
+    def __init__(self, sample_model: type):
+        self._sample_model = sample_model
+
+    def __call__(self, x: Sample) -> Sample:
+        try:
+            self._sample_model.parse_obj(x)
+        except Exception as e:
+            raise StageVerify.StageVerifyException(f"Sample ID: {x.id} -> {str(e)}")
+        return x
+
+
+#  __ __ __ __ __ __ __ __ __ __ __ __
+
+
+#  ___ _  _ ____     __   __  ____  __  __  ___ _  __  _  _
+#   |  |__| |___    |  | |__] |___ |__/|__|  |  | |  | |\ |
+#   |  |  | |___    |__| |    |___ |  \|  |  |  | |__| | \|
+
+
+class OperationVerify(OperationStage):
+    def __init__(self, sample_model: type, **kwargs):
+        super().__init__(stage=StageVerify(sample_model), **kwargs)
+
+
+#  __ __ __ __ __ __ __ __ __ __ __ __
+
+
+def main():
+
+    # 🅳🅰🆃🅰🆂🅴🆃 🅲🆁🅴🅰🆃🅸🅾🅽
+    samples = [
+        PlainSample(
+            {
+                "sample_id": idx,
+                "metadata": {
+                    "a": True,
+                    "b": 2.0,
+                    "c": "hello",  # UNCOMMENT TO PASS VERIFICATION
+                    # "c": "custom",  # UNCOMMENT TO INVALIDATE VERIFICATION
+                },
+                "image": np.random.uniform(0, 1, (256, 256, 3)),
+            },
+            id=idx,
+        )
+        for idx in range(100)
+    ]
+    samples = [PlainSample(x, id=x["sample_id"]) for x in samples]
+    sequence = SamplesSequence(samples=samples)
+
+    # 🆅🅴🆁🅸🅵🆈 🅾🅿🅴🆁🅰🆃🅸🅾🅽
+    verify_op = OperationVerify(
+        sample_model=SampleModel,
+        num_workers=4,
+        progress_bar=True,
+    )
+
+    # 🆃🆁🅰🅿 🅲🆄🆂🆃🅾🅼 🅴🆇🅲🅴🅿🆃🅸🅾🅽
+    try:
+        verify_op(sequence)
+        rich.print("[green] Verified! [/green]")
+    except StageVerify.StageVerifyException as e:
+        print("Validation failed!", e)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/samples/validation/validate_samples.py b/examples/samples/validation/validate_samples.py
index b0f2f3c..f57abba 100644
--- a/examples/samples/validation/validate_samples.py
+++ b/examples/samples/validation/validate_samples.py
@@ -8,24 +8,26 @@
 samples = []
 for i in range(N):
     samples.append(
-        PlainSample({
-            'counter': i,
-            'name': f'sample_{str(i).zfill(10)}',
-            'data': np.random.uniform(-1, 1, (100, 100, 3))
-        })
+        PlainSample(
+            {
+                "counter": i,
+                "name": f"sample_{str(i).zfill(10)}",
+                "data": np.random.uniform(-1, 1, (100, 100, 3)),
+            }
+        )
     )
 sequence = SamplesSequence(samples=samples)
 
 # validate sample (should be used to check keys only)
 rich.print("Simple validation:")
-schema = SchemaLoader.load('simple_schema.schema')
+schema = SchemaLoader.load("simple_schema.schema")
 for idx, sample in enumerate(sequence):
     schema.validate(sample)
-    rich.print("\tValidating", idx, 'OK')
+    rich.print("\tValidating", idx, "OK")
 
 # validate sample in depth
 rich.print("In-depth validation:")
-schema = SchemaLoader.load('deep_schema.schema')
+schema = SchemaLoader.load("deep_schema.schema")
 for idx, sample in enumerate(sequence):
     schema.validate(sample)
-    rich.print("\tValidating", idx, 'OK')
+    rich.print("\tValidating", idx, "OK")
diff --git a/examples/underfolder/api/README.md b/examples/underfolder/api/README.md
new file mode 100644
index 0000000..3844cc8
--- /dev/null
+++ b/examples/underfolder/api/README.md
@@ -0,0 +1,27 @@
+# The UnderfolderAPI Server Example
+
+## Requirements
+
+```
+pip install uvicorn
+pip install webbrowser
+```
+
+
+This example launches the UnderfolderAPI server over a target Underfolder. The API exposes the following endpoints:
+
+![Endpoints](images/endpoints.png)
+
+To launche the server, run the following command:
+
+```
+python underfolder_api_server.py 
+```
+
+The server will target the test Minimnist underfolder. To launch the server on custom underfolder, use the following command:
+
+```
+python underfolder_api_server.py -i $UNDERFOLDER_PATH
+```
+
+The script will also open a browser to the **index.html** page provided statically by the same server. This page contains some HTML/CSS/JS sample code to interact with the APIs.
diff --git a/examples/underfolder/api/images/endpoints.png b/examples/underfolder/api/images/endpoints.png
new file mode 100644
index 0000000..894ec6b
Binary files /dev/null and b/examples/underfolder/api/images/endpoints.png differ
diff --git a/examples/underfolder/api/index.css b/examples/underfolder/api/index.css
new file mode 100644
index 0000000..32b80e8
--- /dev/null
+++ b/examples/underfolder/api/index.css
@@ -0,0 +1,59 @@
+body{
+    font-family: Roboto;
+}
+
+.binary-tag{
+    transition: all 0.3s;
+}
+.binary-tag:hover{
+    transform: scale(2);
+}
+.container-binary-data span{
+    background: #880E4F;
+    padding: 5px 10px 5px 10px;
+    border-radius: 5px;
+    margin-right: 5px;
+    color: white;
+    font-family: roboto;
+}
+
+.container-images{
+    display:block;
+}
+.container-images h5{
+    margin-right: 10px;
+}
+.container-images .thumbnail{
+    display: inline-block;
+    position: relative;
+    width: 200;
+    height: 150;
+    background: #333;
+    overflow: hidden;
+    margin-right: 10px;
+    box-shadow: 0px 0px 15px #333;
+}
+
+.container-images .thumbnail .title{
+    position: absolute;
+    background: white;
+    bottom: 0;
+    left: 0;
+    width: 100%;
+    opacity: 0.8;
+    z-index: 100;
+    font-size: 18;
+    text-align: center;
+}
+.container-images .thumbnail img{
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    height: 100%;
+    opacity: 0.8;
+    font-size: 19px;
+    text-align: center;
+    right: 0;
+    top: 0;
+    margin: auto;
+}
diff --git a/examples/underfolder/api/index.html b/examples/underfolder/api/index.html
new file mode 100644
index 0000000..588dce7
--- /dev/null
+++ b/examples/underfolder/api/index.html
@@ -0,0 +1,165 @@
+<html>
+
+<head>
+    <link rel="stylesheet" href="index.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.3/css/bulma.min.css">
+    <link href="https://cdn.materialdesignicons.com/2.1.19/css/materialdesignicons.min.css" rel="stylesheet">
+    <script src="https://cdn.jsdelivr.net/gh/google/code-prettify@master/loader/run_prettify.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
+    <script src="https://unpkg.com/axios/dist/axios.min.js"></script>
+</head>
+
+<body>
+
+    <div id="app">
+
+        <!-- RELOAD PANEL -->
+        <div class="box">
+            <button class="button is-primary" @click='reload'>Reload</button>
+        </div>
+
+        <!-- DATASET LIST PANEL -->
+        <div class="box" v-if='datasets.length > 0'>
+            <h1 class="title">Datasets list:</h1>
+            <div class="block">
+                <button class="button is-info mr-2 mb-1" v-for='(dataset,index) in datasets'
+                    @click="loadDataset(dataset)">
+                    <i class="mdi mdi-database mdi-light"></i> {{dataset.name}}
+                </button>
+            </div>
+        </div>
+
+        <!-- SELECTED DATASET PANEL -->
+        <div class="box" v-if='dataset !== null'>
+            <h1 class="title">Dataset: {{dataset.name}}</h1>
+
+            <!--  DATASET MANIFEST -->
+            <div class="block">
+                <h2 class="subtitle">Manifest:</h2>
+                <code class="prettyprint lang-js">{{dataset.manifest}}</code>
+            </div>
+
+            <!-- DATASET SAMPLES LIST -->
+            <div class="block">
+                <h2 class="subtitle">Samples List:</h2>
+                <div class='block'>
+                    <button class="button is-primary mr-2 mb-1" v-for="(sample_id, index) in samples_ids"
+                        @click="loadSample(sample_id)">Sample[{{sample_id}}]</button>
+                </div>
+            </div>
+        </div>
+
+        <!-- SELECTED SAMPLE PANEL -->
+        <div class="box" v-if='sample !== null'>
+
+            <!-- SAMPLE METADATA -->
+            <h1 class="title">Sample [{{sample.id}}]</h1>
+            <div class="block">
+                <h2 class="subtitle">Metadata:</h2>
+                <code class="zoomcode prettyprint lang-js">{{sample.metadata}}</code>
+            </div>
+
+            <!-- SAMPLE BINARY DATA LIST -->
+            <div class="block">
+                <h2 class="subtitle">Binary data:</h2>
+                <div class='block container-binary-data'>
+                    <div class="tag is-success mr-2 binary-tag" v-for="(dataValue, dataName) in sample.data">
+                        {{dataName}}</div>
+                </div>
+            </div>
+
+            <!-- SAMPLE IMAGES -->
+            <div class='block container-images'>
+                <h2 class="subtitle">Images:</h2>
+                <div class="block">
+                    <div v-for="(dataValue, dataName) in sample.data" v-if="dataValue.type == 'image'"
+                        class='thumbnail'>
+                        <div class="title">{{dataName}}</div>
+                        <a :href="dataUrl(dataName)" target="_blank">
+                            <img :src="dataUrl(dataName)" />
+                        </a>
+                    </div>
+                </div>
+            </div>
+
+        </div>
+    </div>
+
+    <script type="text/javascript">
+        var app = new Vue({
+            el: '#app',
+            data: {
+                // ****************************************************************************
+                // 
+                // THIS IS THE BASE URL, CHANGE IT IF YOU WANT TO USE ANOTHER SERVER HOST:PORT
+                // 
+                // ****************************************************************************
+                baseUrl: 'http://localhost:8099',
+                datasets: [],
+                dataset: null,
+                samples_ids: [],
+                sample: null
+            },
+            methods: {
+                /**
+                 * Reload the datasets list
+                 */
+                reload() {
+                    this.dataset = null
+                    this.datasets = []
+                    this.samples_ids = []
+                    this.sample = null
+                    axios.get(this.baseUrl + '/datasets').then(response => {
+                        this.datasets = Object.values(response.data)
+                    });
+                },
+                /**
+                 * Load a dataset given its dataset Entity
+                 * @param {Object} dataset
+                 */
+                loadDataset(dataset) {
+                    axios.get(this.baseUrl + '/dataset/' + dataset.name).then(response => {
+
+                        // Sets response entity as current dataset 
+                        this.dataset = response.data;
+
+                        // Pretty-fy the manifest json 
+                        this.$nextTick(() => { PR.prettyPrint() })
+
+                        // Loads the samples list
+                        this.samples_ids = []
+                        this.dataset.manifest.sample_ids.forEach((sample_id) => {
+                            this.samples_ids.push(sample_id)
+                        });
+                    });
+                },
+                /**
+                 * Load a sample given its id
+                 * @param {Number} sample_id
+                 */
+                loadSample(sample_id) {
+                    axios.get(this.baseUrl + '/dataset/' + this.dataset.name + "/" + sample_id).then(response => {
+
+                        // Sets response entity as current sample
+                        this.sample = response.data;
+
+                        // Pretty-fy the metadata json
+                        this.$nextTick(() => { PR.prettyPrint() })
+                    });
+                },
+                /**
+                 * Return the data url for a given data name
+                 */
+                dataUrl(name) {
+                    var url = this.baseUrl
+                    url += '/dataset/' + this.dataset.name
+                    url += '/' + this.sample.id
+                    url += '/' + name
+                    return url
+                }
+            }
+        })
+    </script>
+</body>
+
+</html>
\ No newline at end of file
diff --git a/examples/underfolder/api/underfolder_api_server.py b/examples/underfolder/api/underfolder_api_server.py
new file mode 100644
index 0000000..23085ce
--- /dev/null
+++ b/examples/underfolder/api/underfolder_api_server.py
@@ -0,0 +1,67 @@
+import click
+from fastapi.applications import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from pipelime.sequences.api.underfolder import UnderfolderAPI
+from fastapi.staticfiles import StaticFiles
+import uvicorn
+import webbrowser
+import threading
+
+
+@click.command("underfolder_api_server")
+@click.option("--host", default="127.0.0.1", help="Host to run the server on.")
+@click.option("--port", default=8099, help="Port to run the server on.")
+@click.option(
+    "-i",
+    "--input_folder",
+    default="../../../tests/sample_data/datasets/underfolder_minimnist",
+    help="Input dataset folder",
+)
+@click.option(
+    "-n",
+    "--name",
+    default="underfolder_api_server",
+    help="Name of the dataset endpoint.",
+)
+def underfolder_api_server(host: str, port: int, input_folder: str, name: str):
+
+    # creates the api base url
+    baseUrl = "http://" + host + ":" + str(port)
+
+    # UnderfolderAPI endpoints
+    endpoint = UnderfolderAPI({name: input_folder}, auth_callback=None)
+
+    # FastAPI app
+    app = FastAPI()
+
+    # FastAPI static files provider (for serving static files like index.html)
+    app.mount("/static", StaticFiles(directory=".", html=True), name="static")
+
+    # Add UnderfolderAPI microservice to main app
+    app.include_router(endpoint)
+
+    # Add CORS middleware (for allowing cross-origin requests)
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Open browser with 2.0 seconds delay
+    threading.Timer(
+        2.0, webbrowser.open, args=(f"{baseUrl}/static/index.html",)
+    ).start()
+
+    # Run ASGI webserver
+    uvicorn.run(
+        app,
+        host="127.0.0.1",
+        port=8099,
+        debug=True,
+    )
+
+
+if __name__ == "__main__":
+    underfolder_api_server()
diff --git a/examples/underfolder/filter_by_script/external_script.py b/examples/underfolder/operations/filter_by_script/external_script.py
similarity index 66%
rename from examples/underfolder/filter_by_script/external_script.py
rename to examples/underfolder/operations/filter_by_script/external_script.py
index fecef86..b815e2a 100644
--- a/examples/underfolder/filter_by_script/external_script.py
+++ b/examples/underfolder/operations/filter_by_script/external_script.py
@@ -5,10 +5,10 @@
 
 
 def check_sample(x: Sample, s: SamplesSequence) -> bool:
-    if 'pose' in x:
-        pose = x['pose']
-        z = pose[:3, 2].reshape((3, ))
-        gravity = np.array([0, 0, -1.]).reshape((3,))
+    if "pose" in x:
+        pose = x["pose"]
+        z = pose[:3, 2].reshape((3,))
+        gravity = np.array([0, 0, -1.0]).reshape((3,))
         angle = 180 * np.arccos(np.dot(z, gravity)) / np.pi
         return angle >= MIN_ANGLE
     return True
diff --git a/examples/underfolder/filter_by_script/filter_by_script.py b/examples/underfolder/operations/filter_by_script/filter_by_script.py
similarity index 66%
rename from examples/underfolder/filter_by_script/filter_by_script.py
rename to examples/underfolder/operations/filter_by_script/filter_by_script.py
index f12da74..b363e5f 100644
--- a/examples/underfolder/filter_by_script/filter_by_script.py
+++ b/examples/underfolder/operations/filter_by_script/filter_by_script.py
@@ -5,10 +5,12 @@
 import rich
 
 
-@click.command('filter_by_script', help='Filter UnderfolderReader by external script')
-@click.option('-d', '--dataset', required=True, type=str, help='Input Dataset')
-@click.option('-o', '--output', required=True, type=str, help='Output Dataset folder')
-@click.option('-s', '--script', default='external_script.py', type=str, help='Filtering Script')
+@click.command("filter_by_script", help="Filter UnderfolderReader by external script")
+@click.option("-d", "--dataset", required=True, type=str, help="Input Dataset")
+@click.option("-o", "--output", required=True, type=str, help="Output Dataset folder")
+@click.option(
+    "-s", "--script", default="external_script.py", type=str, help="Filtering Script"
+)
 def filter_by_script(dataset, output, script):
 
     reader = UnderfolderReader(folder=dataset)
@@ -22,12 +24,12 @@ def filter_by_script(dataset, output, script):
         extensions_map=reader_template.extensions_map,
         zfill=reader_template.idx_length,
         copy_files=True,
-        use_symlinks=False
+        use_symlinks=False,
     )
     out = op(reader)
     writer(out)
     rich.print(f"Removed [red]{len(out) - len(reader)}[/red] samples!")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     filter_by_script()
diff --git a/examples/underfolder/pipelines/pipeline_example.py b/examples/underfolder/pipelines/pipeline_example.py
new file mode 100644
index 0000000..7caa824
--- /dev/null
+++ b/examples/underfolder/pipelines/pipeline_example.py
@@ -0,0 +1,88 @@
+import os
+from pipelime.sequences.writers.filesystem import UnderfolderWriter
+from pipelime.sequences.operations import (
+    OperationFilterByQuery,
+    OperationOrderBy,
+    OperationResetIndices,
+    OperationSubsample,
+    OperationSum,
+)
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.tools.idgenerators import IdGeneratorInteger, IdGeneratorUUID
+from pipelime.sequences.pipes import NodeGraph, OperationNode, ReaderNode, WriterNode
+
+
+folder_a = "../../tests/sample_data/datasets/underfolder_minimnist"
+folder_b = "../../tests/sample_data/datasets/underfolder_minimnist"
+
+idg = IdGeneratorUUID()
+subsampling = 1.0
+nodes = [
+    ReaderNode(
+        id=idg.generate(), output_data="A", reader=UnderfolderReader(folder=folder_a)
+    ),
+    ReaderNode(
+        id=idg.generate(), output_data="B", reader=UnderfolderReader(folder=folder_b)
+    ),
+    OperationNode(
+        id=idg.generate(),
+        input_data="A",
+        output_data="A_",
+        operation=OperationSubsample(factor=subsampling),
+    ),
+    OperationNode(
+        id=idg.generate(),
+        input_data="B",
+        output_data="B_",
+        operation=OperationSubsample(factor=subsampling),
+    ),
+    OperationNode(
+        id=idg.generate(),
+        input_data=["A_", "B_"],
+        output_data="X",
+        operation=OperationSum(),
+    ),
+    OperationNode(
+        id=idg.generate(),
+        input_data="X",
+        output_data="X_filtered",
+        operation=OperationFilterByQuery(query="`metadata.sample_id` >= 5"),
+    ),
+    OperationNode(
+        id=idg.generate(),
+        input_data="X_filtered",
+        output_data="X_ordered",
+        operation=OperationOrderBy(order_keys=["counter"]),
+    ),
+    OperationNode(
+        id=idg.generate(),
+        input_data="X_ordered",
+        output_data="X_reset",
+        operation=OperationResetIndices(generator=IdGeneratorInteger()),
+    ),
+    WriterNode(
+        id=idg.generate(),
+        input_data="X_reset",
+        writer=UnderfolderWriter(
+            folder="/tmp/zzizzino",
+            root_files_keys=["camera", "charuco"],
+            extensions_map={
+                ".*image.*": "jpg",
+                ".*pose.*": "txt",
+                ".*camera.*|.*charuco.*|.*metadata.*": "yml",
+            },
+        ),
+    ),
+]
+
+graph = NodeGraph(nodes=nodes)
+
+graph.execute()
+
+graph.draw_to_file("/tmp/graph.png")
+try:
+    os.system("open /tmp/graph.png")
+except Exception:
+    os.system("open /tmp/graph.png")
+
+# graph.execute()
diff --git a/examples/underfolder/plugins/linking/README.md b/examples/underfolder/plugins/linking/README.md
new file mode 100644
index 0000000..a7ba614
--- /dev/null
+++ b/examples/underfolder/plugins/linking/README.md
@@ -0,0 +1,62 @@
+# Examples of UnderfolderLinksPlugin
+
+An **UnderfolderPlugin** is a generic logic executed at the end of the Underfolder constructor.
+
+The **UnderfolderLinksPlugin**, for example, is responsible for creating links to other Underfolder. 
+Recursively, each Underfolder loads its linked Underfolder and merge its samples with the samples of the linked one (*i.e* fuse sample keys).
+
+A plugin is created with a *private* root file in the Undefolder. Root files starting with `_` are considered as *private* and reserved for guys like plugins. The **UnderfolderLinksPlugin**, for example, is spawned with:
+
+```
+_underfolder_links.yml
+```
+
+The plugin name is simply the key name without the `_` prefix and the plugin data is the content of the file itself. 
+
+In this specific case the plugin data is a plain list of folders of other Underfolder to link.
+
+
+**No loop are allowed**! Loops are detected and an exception is raised.
+
+
+## example_link_incremental_update.py
+
+In this example is shown how to use links to create *incremenal update* of the samples. If, for example, you have a stage which transform a sample into another one (*e.g.* a labeling GUI adding labels to the samples, an AI model adding prediction etc..) and you want to use this content in the next stage, you can use the links to create incremental update of the dataset, instead of overwriting previous underfolder or copying it to the next stage.
+
+The example read an Underfolder, add some keys to each sample, and stores
+only the added content as a new Underfolder with a link to the previous one. Creating an 
+**UnderfolderReader** on the last folder will load a single dataset with all data 
+together.
+
+```
+python example_link_incremental_update.py
+```
+
+
+## example_link_tree.py
+
+In this example is shown a mid-complex example of nested underfolder like these:
+
+```
+            ┌────┐
+        ┌───►    ├──┐
+        │   │B   │  │   ┌────┐
+        │   └────┘  ├───►    │
+        │           │   │E   │
+┌────┐  │   ┌────┐  │   └────┘
+│    ├──┼───►    │  │
+│A   │  │   │C   │  │
+└────┘  │   └────┘  │   ┌────┐
+        │           └───►    │
+        │   ┌────┐      │F   │
+        └───►    │      └────┘
+            │D   │
+            └────┘
+```
+
+The **UnderfolderReader** starting from the **A** folder will load all the tree of Underfolder and merge the samples together. The example will plot also the **NetworkX**
+graph of the tree. The Graph is automatically generated and saved in the plugin data.
+
+```
+python example_link_tree.py
+```
diff --git a/examples/underfolder/plugins/linking/datasets/A/_underfolder_links.yml b/examples/underfolder/plugins/linking/datasets/A/_underfolder_links.yml
new file mode 100644
index 0000000..fc796cf
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/A/_underfolder_links.yml
@@ -0,0 +1,9 @@
+- datasets/B
+- datasets/C
+- datasets/D
+- datasets/B
+- datasets/C
+- datasets/D
+- datasets/B
+- datasets/C
+- datasets/D
diff --git a/examples/underfolder/plugins/linking/datasets/A/data/00000_A.yml b/examples/underfolder/plugins/linking/datasets/A/data/00000_A.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/A/data/00000_A.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/A/data/00001_A.yml b/examples/underfolder/plugins/linking/datasets/A/data/00001_A.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/A/data/00001_A.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/B/_underfolder_links.yml b/examples/underfolder/plugins/linking/datasets/B/_underfolder_links.yml
new file mode 100644
index 0000000..4a63eb4
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/B/_underfolder_links.yml
@@ -0,0 +1,9 @@
+- datasets/E
+- datasets/F
+- datasets/G
+- datasets/E
+- datasets/F
+- datasets/G
+- datasets/E
+- datasets/F
+- datasets/G
diff --git a/examples/underfolder/plugins/linking/datasets/B/data/00000_B.yml b/examples/underfolder/plugins/linking/datasets/B/data/00000_B.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/B/data/00000_B.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/B/data/00001_B.yml b/examples/underfolder/plugins/linking/datasets/B/data/00001_B.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/B/data/00001_B.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/C/_underfolder_links.yml b/examples/underfolder/plugins/linking/datasets/C/_underfolder_links.yml
new file mode 100644
index 0000000..18a2ea2
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/C/_underfolder_links.yml
@@ -0,0 +1,9 @@
+- datasets/H
+- datasets/I
+- datasets/J
+- datasets/H
+- datasets/I
+- datasets/J
+- datasets/H
+- datasets/I
+- datasets/J
diff --git a/examples/underfolder/plugins/linking/datasets/C/data/00000_C.yml b/examples/underfolder/plugins/linking/datasets/C/data/00000_C.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/C/data/00000_C.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/C/data/00001_C.yml b/examples/underfolder/plugins/linking/datasets/C/data/00001_C.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/C/data/00001_C.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/D/_underfolder_links.yml b/examples/underfolder/plugins/linking/datasets/D/_underfolder_links.yml
new file mode 100644
index 0000000..64bddb2
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/D/_underfolder_links.yml
@@ -0,0 +1,3 @@
+- datasets/K
+- datasets/K
+- datasets/K
diff --git a/examples/underfolder/plugins/linking/datasets/D/data/00000_D.yml b/examples/underfolder/plugins/linking/datasets/D/data/00000_D.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/D/data/00000_D.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/D/data/00001_D.yml b/examples/underfolder/plugins/linking/datasets/D/data/00001_D.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/D/data/00001_D.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/E/data/00000_E.yml b/examples/underfolder/plugins/linking/datasets/E/data/00000_E.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/E/data/00000_E.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/E/data/00001_E.yml b/examples/underfolder/plugins/linking/datasets/E/data/00001_E.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/E/data/00001_E.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/F/data/00000_F.yml b/examples/underfolder/plugins/linking/datasets/F/data/00000_F.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/F/data/00000_F.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/F/data/00001_F.yml b/examples/underfolder/plugins/linking/datasets/F/data/00001_F.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/F/data/00001_F.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/G/data/00000_G.yml b/examples/underfolder/plugins/linking/datasets/G/data/00000_G.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/G/data/00000_G.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/G/data/00001_G.yml b/examples/underfolder/plugins/linking/datasets/G/data/00001_G.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/G/data/00001_G.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/H/data/00000_H.yml b/examples/underfolder/plugins/linking/datasets/H/data/00000_H.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/H/data/00000_H.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/H/data/00001_H.yml b/examples/underfolder/plugins/linking/datasets/H/data/00001_H.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/H/data/00001_H.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/I/data/00000_I.yml b/examples/underfolder/plugins/linking/datasets/I/data/00000_I.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/I/data/00000_I.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/I/data/00001_I.yml b/examples/underfolder/plugins/linking/datasets/I/data/00001_I.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/I/data/00001_I.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/J/data/00000_J.yml b/examples/underfolder/plugins/linking/datasets/J/data/00000_J.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/J/data/00000_J.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/J/data/00001_J.yml b/examples/underfolder/plugins/linking/datasets/J/data/00001_J.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/J/data/00001_J.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/datasets/K/data/00000_K.yml b/examples/underfolder/plugins/linking/datasets/K/data/00000_K.yml
new file mode 100644
index 0000000..9591813
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/K/data/00000_K.yml
@@ -0,0 +1,1 @@
+data: 0
diff --git a/examples/underfolder/plugins/linking/datasets/K/data/00001_K.yml b/examples/underfolder/plugins/linking/datasets/K/data/00001_K.yml
new file mode 100644
index 0000000..937ac32
--- /dev/null
+++ b/examples/underfolder/plugins/linking/datasets/K/data/00001_K.yml
@@ -0,0 +1,1 @@
+data: 1
diff --git a/examples/underfolder/plugins/linking/example_link_incremental_update.py b/examples/underfolder/plugins/linking/example_link_incremental_update.py
new file mode 100644
index 0000000..4fcccb9
--- /dev/null
+++ b/examples/underfolder/plugins/linking/example_link_incremental_update.py
@@ -0,0 +1,54 @@
+from pipelime.sequences.readers.filesystem import (
+    UnderfolderLinksPlugin,
+    UnderfolderReader,
+)
+import rich
+import tempfile
+from pathlib import Path
+from pipelime.sequences.samples import PlainSample, SamplesSequence
+from pipelime.sequences.writers.filesystem import UnderfolderWriter
+
+# Base temp folder
+base_folder = Path(tempfile.mkdtemp())
+rich.print("Working directory:", base_folder)
+
+# Versions as Incremental upgrade of base dataset
+versions = ["B", "C", "D", "E", "F"]
+
+# Generate first dataset
+last_key = "A"
+last_folder = base_folder / last_key
+UnderfolderWriter(folder=last_folder)(
+    SamplesSequence([PlainSample({last_key: i}, id=i) for i in range(10)])
+)
+
+# Generate incremental upgrade datasets
+for version in versions:
+    # read previous sequence
+    reader = UnderfolderReader(folder=last_folder)
+
+    # generate a new sequence whose samples are a modified version of the previous
+    new_sequence = SamplesSequence([])
+    for sample_index, sample in enumerate(reader):
+        new_sequence.samples.append(
+            PlainSample({version: sample[last_key] * 2}, id=sample_index)
+        )
+
+    # Write dataset
+    new_folder = base_folder / version
+    UnderfolderWriter(folder=new_folder)(new_sequence)
+    rich.print("Writing", version, new_folder)
+
+    # Here the link is created
+    UnderfolderLinksPlugin.link_underfolders(str(new_folder), str(last_folder))
+
+    # update pointers
+    last_folder = new_folder
+    last_key = version
+
+# Print samples of last reader, which is a cumulative dataset of all versions
+last_reader = UnderfolderReader(folder=last_folder)
+for sample in last_reader:
+    rich.print("Sample:")
+    for key in sorted(sample.keys()):
+        rich.print("\t", key, ":", sample[key])
diff --git a/examples/underfolder/plugins/linking/example_link_tree.py b/examples/underfolder/plugins/linking/example_link_tree.py
new file mode 100644
index 0000000..d8f6a78
--- /dev/null
+++ b/examples/underfolder/plugins/linking/example_link_tree.py
@@ -0,0 +1,56 @@
+from pipelime.sequences.readers.filesystem import (
+    UnderfolderLinksPlugin,
+    UnderfolderReader,
+)
+import networkx as nx
+import matplotlib.pyplot as plt
+from rich.table import Table
+from rich.console import Console
+
+
+###############################################################################
+###############################################################################
+# CODE TO re-GENERATE THE DATASETS SUBFOLDER
+###############################################################################
+###############################################################################
+# from pipelime.sequences.readers.filesystem import UnderfolderLinksPlugin
+# from pipelime.sequences.samples import PlainSample, SamplesSequence
+# from pipelime.sequences.writers.filesystem import UnderfolderWriter
+# import networkx as nx
+# from pathlib import Path
+
+# dataset_folder = Path("datasets")
+# keys = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K"]
+# subfolders = {x: dataset_folder / x for x in keys}
+
+# for key in keys:
+#     UnderfolderWriter(folder=subfolders[key], extensions_map={key: "yml"})(
+#         SamplesSequence([PlainSample({key: {"data": i}}, id=i) for i in range(2)])
+#     )
+
+# g = nx.full_rary_tree(3, len(keys))
+# for u, v, a in g.edges(data=True):
+#     key_source = keys[u]
+#     key_target = keys[v]
+#     folder_source = str(subfolders[key_source])
+#     folder_target = str(subfolders[key_target])
+#     UnderfolderLinksPlugin.link_underfolders(folder_source, folder_target)
+#     print(u, v, keys[u], keys[v])
+###############################################################################
+###############################################################################
+
+
+table = Table(show_header=True, header_style="bold magenta")
+base_reader = UnderfolderReader(folder="datasets/A")
+for sample in base_reader:
+    table.add_row("Sample", sample.id, "")
+    for key in sorted(sample.keys()):
+        table.add_row("", key, str(sample[key]))
+
+Console().print(table)
+
+# Load plugins and show Links Graph if any
+for plugin_name, plugin in base_reader.plugins_map.items():
+    if isinstance(plugin, UnderfolderLinksPlugin):
+        nx.draw(plugin.links_graph, with_labels=True)
+        plt.show()
diff --git a/examples/underfolder/plugins/stages/README.md b/examples/underfolder/plugins/stages/README.md
new file mode 100644
index 0000000..eec9c6f
--- /dev/null
+++ b/examples/underfolder/plugins/stages/README.md
@@ -0,0 +1,30 @@
+# Examples of UnderfolderStagePlugin
+
+An **UnderfolderPlugin** is a generic logic executed at the end of the Underfolder constructor.
+
+The **UnderfolderStagePlugin**, for example, is responsible for adding custom stages to each Sample before the __getitem__ method is called.
+
+
+A plugin is created with a *private* root file in the Undefolder. Root files starting with `_` are considered as *private* and reserved for guys like plugins. The **UnderfolderStagePlugin**, for example, is spawned with:
+
+```
+_underfolder_stage.yml
+```
+
+The plugin name is simply the key name without the `_` prefix and the plugin data is the content of the file itself. 
+
+In this specific case the plugin data is a serialization of a generic **StageCompose** mapping together multiple **SampleStage**.
+
+## example_stages.py
+
+In this example is shown how to inject a custom stage into Underfolder as a plugin. The custom stage is a stage which add a 'result' key to each sample where the value is a function of sample["value"]:
+
+$sample["result"] = sample["value"] \times a + b$ 
+
+launch with:
+
+```
+python example_stages.py
+```
+
+⚠️⚠️ Pay attention ⚠️⚠️! If you use a custom stage class, like in this example, ensure that the class is imported before the plugin is loaded. **SampleStage** are **Spooks** so their registration to the **SampleStage** factory happens in metaclass declaration.
\ No newline at end of file
diff --git a/examples/underfolder/plugins/stages/dataset/_underfolder_stage.yml b/examples/underfolder/plugins/stages/dataset/_underfolder_stage.yml
new file mode 100644
index 0000000..fbb571d
--- /dev/null
+++ b/examples/underfolder/plugins/stages/dataset/_underfolder_stage.yml
@@ -0,0 +1,4 @@
+__spook__: __main__.MyStageEquation
+args:
+  a: 2.0
+  b: 3.0
diff --git a/examples/underfolder/plugins/stages/dataset/data/000_value.txt b/examples/underfolder/plugins/stages/dataset/data/000_value.txt
new file mode 100644
index 0000000..171538e
--- /dev/null
+++ b/examples/underfolder/plugins/stages/dataset/data/000_value.txt
@@ -0,0 +1,1 @@
+0.0
\ No newline at end of file
diff --git a/examples/underfolder/plugins/stages/dataset/data/001_value.txt b/examples/underfolder/plugins/stages/dataset/data/001_value.txt
new file mode 100644
index 0000000..9f8e9b6
--- /dev/null
+++ b/examples/underfolder/plugins/stages/dataset/data/001_value.txt
@@ -0,0 +1,1 @@
+1.0
\ No newline at end of file
diff --git a/examples/underfolder/plugins/stages/example_stages.py b/examples/underfolder/plugins/stages/example_stages.py
new file mode 100644
index 0000000..cc5722c
--- /dev/null
+++ b/examples/underfolder/plugins/stages/example_stages.py
@@ -0,0 +1,46 @@
+from pipelime.sequences.readers.filesystem import (
+    UnderfolderReader,
+    UnderfolderStagePlugin,
+)
+from pipelime.sequences.samples import Sample
+from pipelime.sequences.stages import SampleStage
+import rich
+
+
+class MyStageEquation(SampleStage):
+    def __init__(self, a: float, b: float):
+        self._a = a
+        self._b = b
+
+    def __call__(self, x: Sample) -> Sample:
+        x["result"] = x["value"] * self._a + self._b
+        return x
+
+    @classmethod
+    def spook_schema(cls) -> dict:
+        return {"a": float, "b": float}
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        return MyStageEquation(a=d["a"], b=d["b"])
+
+    def to_dict(self):
+        return {"a": self._a, "b": self._b}
+
+
+# create the stage
+stage = MyStageEquation(a=2.0, b=3.0)
+
+# inject the stage to the dataset in 'dataset' folder
+UnderfolderStagePlugin.set_stages("dataset", stage)
+
+###############################################################################
+# Stages are automatically applied to the dataset samples when retreived
+
+# read the dataset (with stage)
+reader = UnderfolderReader("dataset")
+
+for sample in reader:
+    rich.print("Sample:", sample.id)
+    rich.print("\t", "Original Value:", sample["value"])
+    rich.print("\t", "Transformed Value:", sample["result"])
diff --git a/examples/underfolder/proxies/proxy_example.py b/examples/underfolder/proxies/proxy_example.py
new file mode 100644
index 0000000..567ed96
--- /dev/null
+++ b/examples/underfolder/proxies/proxy_example.py
@@ -0,0 +1,184 @@
+from pipelime.sequences.samples import SamplesSequence
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.proxies import (
+    FilteredSamplesSequence,
+    ConcatSamplesSequence,
+    SortedSamplesSequence,
+    CachedSamplesSequence,
+)
+from pipelime.sequences.stages import StageKeysFilter, StageAugmentations
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from time import perf_counter_ns
+
+#
+# We are going to create this pipeline:
+#
+#         dataset
+#     ______|______
+#    |             |
+# reader_a      reader_b
+#    |             |
+# filter_odd    filter_even
+#  + augment     + augment
+#    |             |
+#    |          sort_descending
+#    |_____________|
+#           |
+#         concat
+#           |
+#         cache
+#
+# Then we will go through all the samples multiple times, measuring time performances.
+#
+
+dataset = str(
+    Path(__file__).resolve().parent
+    / "../../../tests/sample_data/datasets/underfolder_minimnist"
+)
+
+# composing with a helper sequence to set a Stage on top of any serialized Stage
+reader_a = SamplesSequence(
+    UnderfolderReader(dataset),
+    StageKeysFilter(("image", "image_mask", "image_maskinv", "label")),
+)
+reader_b = SamplesSequence(
+    UnderfolderReader(dataset),
+    StageKeysFilter(("image", "image_mask", "image_maskinv", "label")),
+)
+
+augmentation = {
+    "__version__": "1.0.3",
+    "transform": {
+        "__class_fullname__": "Compose",
+        "p": 1.0,
+        "transforms": [
+            {
+                "__class_fullname__": "Resize",
+                "always_apply": False,
+                "p": 1,
+                "height": 1024,
+                "width": 1024,
+                "interpolation": 1,
+            },
+            {"__class_fullname__": "HorizontalFlip", "always_apply": False, "p": 1},
+            {
+                "__class_fullname__": "ShiftScaleRotate",
+                "always_apply": False,
+                "p": 1.0,
+                "shift_limit_x": [-0.2, 0.2],
+                "shift_limit_y": [-0.2, 0.2],
+                "scale_limit": [-0.6, 0.6000000000000001],
+                "rotate_limit": [-180, 180],
+                "interpolation": 1,
+                "border_mode": 4,
+                "value": None,
+                "mask_value": None,
+            },
+            {
+                "__class_fullname__": "RandomBrightnessContrast",
+                "always_apply": False,
+                "p": 1,
+                "brightness_limit": [-0.2, 0.2],
+                "contrast_limit": [-0.2, 0.2],
+                "brightness_by_max": True,
+            },
+        ],
+        "bbox_params": None,
+        "keypoint_params": None,
+        "additional_targets": {},
+    },
+}
+
+filter_odd = FilteredSamplesSequence(
+    reader_a,
+    lambda x: int(x["label"]) % 2 == 1,
+    StageAugmentations(
+        augmentation,
+        targets={"image": "image", "image_mask": "mask", "image_maskinv": "mask"},
+    ),
+)
+filter_even = FilteredSamplesSequence(
+    reader_b,
+    lambda x: int(x["label"]) % 2 == 0,
+    StageAugmentations(
+        augmentation,
+        targets={"image": "image", "image_mask": "mask", "image_maskinv": "mask"},
+    ),
+)
+
+sort_descending = SortedSamplesSequence(filter_even, lambda x: -int(x["label"]))
+
+concat = ConcatSamplesSequence([filter_odd, sort_descending])
+
+# create a temp dir that will be removed at exit
+with TemporaryDirectory() as tmp_cache_folder:
+    cached_seq = CachedSamplesSequence(
+        concat,
+        CachedSamplesSequence.PersistentCachePolicy(tmp_cache_folder),
+        ("image", "image_mask", "image_maskinv", "label"),
+    )
+
+    print("*** Traversing the whole dataset:")
+    for i, s in enumerate(cached_seq):
+        print(f"[{i:03d}] " + ", ".join(s.keys()) + f" - {s['label']}")
+
+    # PERFORMANCE UTILITIES
+    def time_perf(sseq):
+        tic = perf_counter_ns()
+        for s in sseq:
+            for k in s.keys():
+                _ = s[k]
+        for s in reversed(sseq):
+            for k in s.keys():
+                _ = s[k]
+        toc = perf_counter_ns()
+        return (toc - tic) * 1e-6
+
+    def double_run_perf(sseq, lbl, numbers=5):
+        for i in range(numbers):
+            elapsed = time_perf(sseq)
+            print(f"** {lbl} (run #{i}): {elapsed} ms")
+
+    print("*** Traversing dataset in both directions for performance evaluation")
+
+    #######################
+    # SIMPLE CACHING TO DISK
+    #######################
+    cached_seq = CachedSamplesSequence(
+        concat,
+        CachedSamplesSequence.PersistentCachePolicy(tmp_cache_folder, clear_cache=True),
+        ("image", "image_mask", "image_maskinv", "label"),
+    )
+    double_run_perf(cached_seq, "SIMPLE DISK CACHE")
+
+    #######################
+    # SHARING CACHE DATA
+    #######################
+    cached_seq = CachedSamplesSequence(
+        concat,
+        CachedSamplesSequence.PersistentCachePolicy(tmp_cache_folder),
+        ("image", "image_mask", "image_maskinv", "label"),
+    )
+    double_run_perf(cached_seq, "SHARED CACHE")
+
+    #######################
+    # CACHING WITH MEMORY BUFFER
+    #######################
+    cached_seq = CachedSamplesSequence(
+        concat,
+        CachedSamplesSequence.PersistentCachePolicy(
+            tmp_cache_folder, clear_cache=True, max_buffer_size=len(concat) // 2
+        ),
+        ("image", "image_mask", "image_maskinv", "label"),
+    )
+    double_run_perf(cached_seq, "HALF MEMORY BUFFER")
+
+    cached_seq = CachedSamplesSequence(
+        concat,
+        CachedSamplesSequence.PersistentCachePolicy(
+            tmp_cache_folder, clear_cache=True, max_buffer_size=len(concat)
+        ),
+        ("image", "image_mask", "image_maskinv", "label"),
+    )
+    double_run_perf(cached_seq, "FULL MEMORY BUFFER")
diff --git a/examples/underfolder/remotes/README.md b/examples/underfolder/remotes/README.md
new file mode 100644
index 0000000..297c8e5
--- /dev/null
+++ b/examples/underfolder/remotes/README.md
@@ -0,0 +1,59 @@
+# How to setup a versioned dataset using remotes
+
+Datasets are usually full of binary data, eg, images, which are not
+suitable for versioning. Luckily, Pipelime can replace such heavy data with small text
+files storing references to external data lakes, so that:
+* the full dataset can be easily versioned and shared
+* the data is automatically downloaded when reading the item
+
+To setup a versioned dataset, first you need a data lake, eg, a shared folder or a
+S3-compatible data server. Then, prepare your data in Underfolder format and run the
+`remote_add` operation:
+
+```
+pipelime underfolder remote_add -i path/to/input -o path/to/output -r file://localhost/path/to/root/bucket-name -r s3://172.218.0.0:9000/bucket-name?access_key=user:secret_key=pwd:secure_connection=False -k image -k mask --hardlink
+```
+
+The remotes are listed as url with optional init arguments as query data. Note that the
+S3 interface can load the access and secret keys from environment variables or aws
+config files as well. The output dataset will have the listed keys, ie, `image` and `mask`,
+replaced with `.remote` text files referencing to both remotes.
+Similarly, to remove a remote from a remote list, just call `remote_remove`:
+
+```
+pipelime underfolder remote_remove -i path/to/input -o path/to/output -r s3://172.218.0.0:9000/bucket-name -k image -k mask --hardlink
+```
+
+Note that the file is **not** removed on the remote.
+
+Reading a dataset with data on remotes is straightforward, provided that at list one
+remote is accessible. Then, actual data can be easily cached:
+
+```python
+reader = UnderfolderReader("path/to/dataset")
+filtered_data = SamplesSequence(
+    reader,
+    StageKeysFilter(("image", "mask", "label")),
+)
+cached_seq = CachedSamplesSequence(
+    filtered_data,
+    CachedSamplesSequence.PersistentCachePolicy("path/to/cache_folder"),
+    ("image", "mask", "label"),
+)
+```
+
+The first iteration over `cached_seq` will _transparently_ download the data from the remote,
+then subsequent calls will just pick the cached binary files. This way, a simple git-clone
+of the data repository can be enough to share any binary dataset.
+
+### WARNING
+
+The remote must be accessible in order to retrieve any data. If you need to, eg, set the
+access and secret keys of an S3 data server from code, make sure to create the remote
+interface _before_ iterating over the data:
+
+```python
+from pipelime.filesystem.remotes import create_remote
+
+create_remote("s3", "172.218.0.0:9000", access_key='user', secret_key='pwd', secure_connection=False)
+```
diff --git a/examples/underfolder/stages/README.md b/examples/underfolder/stages/README.md
new file mode 100644
index 0000000..ffeacb5
--- /dev/null
+++ b/examples/underfolder/stages/README.md
@@ -0,0 +1,5 @@
+# How to load Stages from file and apply them to an Undefolder reader
+
+```
+python example_stages.py
+```
\ No newline at end of file
diff --git a/examples/underfolder/stages/example_stages.py b/examples/underfolder/stages/example_stages.py
new file mode 100644
index 0000000..b4014f5
--- /dev/null
+++ b/examples/underfolder/stages/example_stages.py
@@ -0,0 +1,20 @@
+from choixe.spooks import Spook
+import rich
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+
+# load stages from file
+stage = Spook.create_from_file("stages.yml")
+
+# Input dataset
+dataset = UnderfolderReader("../data/underfolder/example_dataset")
+
+# Before stage
+rich.print("Before stage:")
+for sample in dataset:
+    rich.print(list(sample.keys()))
+
+# After stage
+rich.print("After stage:")
+dataset.stage = stage
+for sample in dataset:
+    rich.print(list(sample.keys()))
diff --git a/examples/underfolder/stages/stages.yml b/examples/underfolder/stages/stages.yml
new file mode 100644
index 0000000..c39a3de
--- /dev/null
+++ b/examples/underfolder/stages/stages.yml
@@ -0,0 +1,17 @@
+__spook__: pipelime.sequences.stages.StageCompose
+args:
+  stages:
+  - __spook__: pipelime.sequences.stages.StageIdentity
+    args: {}
+  - __spook__: pipelime.sequences.stages.StageKeysFilter
+    args:
+      key_list:
+      - value
+      negate: false
+  - __spook__: pipelime.sequences.stages.StageRemap
+    args:
+      remap:
+        value: result
+      remove_missing: true
+  - __spook__: pipelime.sequences.stages.StageIdentity
+    args: {}
diff --git a/examples/underfolder/streams/README.md b/examples/underfolder/streams/README.md
new file mode 100644
index 0000000..fddd4e6
--- /dev/null
+++ b/examples/underfolder/streams/README.md
@@ -0,0 +1,25 @@
+# Underfolder Stream is used to read/write an underfolder interactively
+
+The Underfolder Stream has the following architecture:
+
+```
+             ┌───────────────────┐  ┌─────────────┐
+     ┌───────►                   │  │             │
+     │       │ UnderfolderReader ├──►             ├──────► get_data
+┌────┴────┐  │                   │  │             │
+│  Disk   │  └───────────────────┘  │ Underfolder │
+│ ------  │                         │ Stream      │
+│ ------  │  ┌───────────────────┐  │             │
+└────▲────┘  │                   │  │             ◄─────── set_data
+     │       │ UnderfolderWriter ◄──┤             │
+     └───────┤                   │  │             │
+             └───────────────────┘  └─────────────┘
+```
+
+
+The following examples creates a Stream on a Minimnist underfolder. Resize images and
+pushes new infos in metadata:
+
+```sh
+python streams_example.py
+```
diff --git a/examples/underfolder/streams/streams_example.py b/examples/underfolder/streams/streams_example.py
new file mode 100644
index 0000000..a886176
--- /dev/null
+++ b/examples/underfolder/streams/streams_example.py
@@ -0,0 +1,44 @@
+import io
+from pipelime.sequences.streams.underfolder import UnderfolderStream
+import rich
+import tempfile
+import cv2
+import shutil
+from pathlib import Path
+import imageio
+import numpy as np
+
+input_folder = "../../../tests/sample_data/datasets/underfolder_minimnist/"
+output_folder = Path(tempfile.mkdtemp()) / "dataset"
+shutil.copytree(input_folder, output_folder)
+
+stream = UnderfolderStream(folder=output_folder)
+
+for sample_id in range(len(stream)):
+
+    # RESIZE IMAGE AND PUSH
+    # fetch image
+    data, _ = stream.get_data(sample_id, "image", "jpg")
+    image = imageio.imread(data)
+
+    # resize image and conver to bytes
+    image: np.ndarray = cv2.resize(image, (256, 256))
+    image_bytes = io.BytesIO()
+    imageio.imwrite(image_bytes, image, format="jpg")
+
+    # push image
+    stream.set_data(sample_id, "image", image_bytes, "jpg")
+
+    # UPDATE METADATA
+    # fetch metadata
+    metadata, _ = stream.get_data(sample_id, "metadata", "dict")
+
+    # update metadata
+    metadata["updated"] = True
+    metadata["resize"] = 256
+
+    # push metadata
+    stream.set_data(sample_id, "metadata", metadata, "dict")
+
+
+rich.print("[red]Output written to:[/red]", output_folder)
diff --git a/examples/underfolder/writer_multiprocessing/writer_multiprocessing.py b/examples/underfolder/writer_multiprocessing/writer_multiprocessing.py
deleted file mode 100644
index 44123e4..0000000
--- a/examples/underfolder/writer_multiprocessing/writer_multiprocessing.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from pipelime.sequences.samples import PlainSample,  SamplesSequence
-from pipelime.sequences.writers.filesystem import UnderfolderWriter
-import tempfile
-import rich
-from rich.progress import track
-import numpy as np
-import time
-import shutil
-import multiprocessing
-
-
-def new_sample(i) -> PlainSample:
-    return PlainSample(data={'image': np.random.uniform(0, 255, (256, 256, 3)).astype(np.uint8)})
-
-
-# ➡️➡️➡️ Reader
-N = 1000
-pool = multiprocessing.Pool()
-samples = list(track(pool.imap_unordered(new_sample, range(N)), total=N))
-sequence = SamplesSequence(samples=samples)
-
-print(len(sequence))
-
-workers_options = [0, 1, 2, 3, 4, -1]
-
-for num_workers in workers_options:
-    # ➡️➡️➡️ Writer with manual template input
-    writer_folder = tempfile.mkdtemp()
-    writer = UnderfolderWriter(
-        folder=writer_folder,
-        extensions_map={'image': 'png'},
-        num_workers=num_workers
-    )
-    t1 = time.perf_counter()
-    writer(sequence)
-    t2 = time.perf_counter()
-    rich.print(f"num_workers={num_workers} -> Time: {t2-t1} s")
-    shutil.rmtree(writer_folder)
diff --git a/examples/underfolder/reader_writer_template/README.md b/examples/underfolder/writers/reader_writer_template/README.md
similarity index 100%
rename from examples/underfolder/reader_writer_template/README.md
rename to examples/underfolder/writers/reader_writer_template/README.md
diff --git a/examples/underfolder/reader_writer_template/example.py b/examples/underfolder/writers/reader_writer_template/example.py
similarity index 84%
rename from examples/underfolder/reader_writer_template/example.py
rename to examples/underfolder/writers/reader_writer_template/example.py
index 01a1c97..d2a44aa 100644
--- a/examples/underfolder/reader_writer_template/example.py
+++ b/examples/underfolder/writers/reader_writer_template/example.py
@@ -4,7 +4,9 @@
 import rich
 
 # ➡️➡️➡️ Reader
-reader = UnderfolderReader(folder='../../data/underfolder/example_dataset', copy_root_files=True)
+reader = UnderfolderReader(
+    folder="../../data/underfolder/example_dataset", copy_root_files=True
+)
 reader_template = reader.get_reader_template()
 
 # ➡️➡️➡️ Writer with manual template input
@@ -13,7 +15,7 @@
     folder=writer_folder,
     root_files_keys=reader_template.root_files_keys,
     extensions_map=reader_template.extensions_map,
-    zfill=reader_template.idx_length
+    zfill=reader_template.idx_length,
 )
 writer(reader)
 rich.print("Writer output to:", writer_folder)
diff --git a/examples/underfolder/writer_copy_and_symlinks/README.md b/examples/underfolder/writers/writer_copy_and_symlinks/README.md
similarity index 100%
rename from examples/underfolder/writer_copy_and_symlinks/README.md
rename to examples/underfolder/writers/writer_copy_and_symlinks/README.md
diff --git a/examples/underfolder/writers/writer_multiprocessing/writer_multiprocessing.py b/examples/underfolder/writers/writer_multiprocessing/writer_multiprocessing.py
new file mode 100644
index 0000000..3c57249
--- /dev/null
+++ b/examples/underfolder/writers/writer_multiprocessing/writer_multiprocessing.py
@@ -0,0 +1,46 @@
+from pipelime.sequences.samples import PlainSample, SamplesSequence
+from pipelime.sequences.writers.filesystem import UnderfolderWriter
+import tempfile
+import rich
+from rich.progress import track
+import numpy as np
+import time
+import shutil
+import multiprocessing
+
+
+def new_sample(i) -> PlainSample:
+    return PlainSample(
+        data={"image": np.random.uniform(0, 255, (256, 256, 3)).astype(np.uint8)}
+    )
+
+
+def main():
+
+    # ➡️➡️➡️ Reader
+    N = 1000
+    pool = multiprocessing.Pool()
+    samples = list(track(pool.imap_unordered(new_sample, range(N)), total=N))
+    sequence = SamplesSequence(samples=samples)
+
+    print(len(sequence))
+
+    workers_options = [0, 1, 2, 3, 4, -1]
+
+    for num_workers in workers_options:
+        # ➡️➡️➡️ Writer with manual template input
+        writer_folder = tempfile.mkdtemp()
+        writer = UnderfolderWriter(
+            folder=writer_folder,
+            extensions_map={"image": "png"},
+            num_workers=num_workers,
+        )
+        t1 = time.perf_counter()
+        writer(sequence)
+        t2 = time.perf_counter()
+        rich.print(f"num_workers={num_workers} -> Time: {t2-t1} s")
+        shutil.rmtree(writer_folder)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/pipelime/__init__.py b/pipelime/__init__.py
index b92b20d..80a181c 100644
--- a/pipelime/__init__.py
+++ b/pipelime/__init__.py
@@ -2,18 +2,13 @@
 
 __author__ = "eyecan"
 __email__ = "daniele.degregorio@eyecan.ai"
-__version__ = "0.1.2"
+__version__ = "0.1.3"
 
 
-# Import transforms in order to register classes for serialization
 from pathlib import Path
-from pipelime.augmentations.transforms import PadIfNeededV2
-from pipelime.sequences.readers.filesystem import UnderfolderReader
-from pipelime.sequences.writers.filesystem import UnderfolderWriter
 from appdirs import AppDirs
 import pipelime
 
 
 def user_data_dir() -> Path:
-    # TODO: esurface and eyecan: move strings as package dunder
     return Path(AppDirs(appname=pipelime.__name__, appauthor=__author__).user_data_dir)
diff --git a/pipelime/augmentations/factory.py b/pipelime/augmentations/factory.py
deleted file mode 100644
index 60f85d8..0000000
--- a/pipelime/augmentations/factory.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import albumentations as A
-from choixe.configurations import XConfig
-from schema import Optional, Schema
-
-
-class AugmentationsFactory(object):
-
-    AUGMENTATIONS_CFG_SCHEMA = Schema({Optional("__version__"): str, "transform": dict})
-
-    @classmethod
-    def build_from_file(cls, filename: str):
-        cfg = XConfig(filename=filename)
-        return cls.build_from_dict(cfg.to_dict())
-
-    @classmethod
-    def build_from_dict(cls, cfg: dict):
-        cfg = XConfig.from_dict(cfg)
-        cfg.set_schema(cls.AUGMENTATIONS_CFG_SCHEMA)
-        cfg.validate()
-        return A.from_dict(cfg.to_dict(discard_private_qualifiers=False))
diff --git a/pipelime/augmentations/transforms.py b/pipelime/augmentations/transforms.py
deleted file mode 100644
index 9b6a91e..0000000
--- a/pipelime/augmentations/transforms.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import albumentations as A
-import cv2
-
-
-class PadIfNeededV2(A.PadIfNeeded):
-    def __init__(
-        self,
-        min_height=1024,
-        min_width=1024,
-        border_mode=cv2.BORDER_REFLECT_101,
-        value=None,
-        mask_value=None,
-        row_align="center",
-        col_align="center",
-        always_apply=False,
-        p=1.0,
-    ):
-        super(PadIfNeededV2, self).__init__(
-            min_height,
-            min_width,
-            border_mode=border_mode,
-            value=value,
-            mask_value=mask_value,
-            always_apply=always_apply,
-            p=p,
-        )
-        self.row_align = row_align
-        self.col_align = col_align
-
-    def update_params(self, params, **kwargs):
-        params = super(PadIfNeededV2, self).update_params(params, **kwargs)
-        rows = params["rows"]
-        cols = params["cols"]
-
-        if rows < self.min_height:
-            if self.row_align == "center":
-                h_pad_top = int((self.min_height - rows) / 2.0)
-                h_pad_bottom = self.min_height - rows - h_pad_top
-            elif self.row_align == "top":
-                h_pad_top = 0
-                h_pad_bottom = self.min_height - rows
-            elif self.row_align == "bottom":
-                h_pad_top = self.min_height - rows
-                h_pad_bottom = 0
-            else:
-                raise ValueError(f'Invalid value "{self.row_align}" for row_align')
-        else:
-            h_pad_top = 0
-            h_pad_bottom = 0
-
-        if cols < self.min_width:
-            if self.col_align == "center":
-                w_pad_left = int((self.min_width - cols) / 2.0)
-                w_pad_right = self.min_width - cols - w_pad_left
-            elif self.col_align == "left":
-                w_pad_left = 0
-                w_pad_right = self.min_width - cols
-            elif self.col_align == "right":
-                w_pad_left = self.min_width - cols
-                w_pad_right = 0
-            else:
-                raise ValueError(f'Invalid value "{self.col_align}" for col_align')
-        else:
-            w_pad_left = 0
-            w_pad_right = 0
-
-        params.update(
-            {
-                "pad_top": h_pad_top,
-                "pad_bottom": h_pad_bottom,
-                "pad_left": w_pad_left,
-                "pad_right": w_pad_right,
-            }
-        )
-        return params
-
-    def get_transform_init_args_names(self):
-        return (
-            "min_height",
-            "min_width",
-            "border_mode",
-            "value",
-            "mask_value",
-            "row_align",
-            "col_align",
-        )
diff --git a/pipelime/cli/conversions/conversions.py b/pipelime/cli/conversions/conversions.py
index 20bbe5b..271d668 100644
--- a/pipelime/cli/conversions/conversions.py
+++ b/pipelime/cli/conversions/conversions.py
@@ -2,6 +2,7 @@
 
 from pipelime.cli.conversions.h5_to_underfolder import h5_to_underfolder
 from pipelime.cli.conversions.underfolder_to_h5 import underfolder_to_h5
+from pipelime.cli.conversions.subfolders_to_underfolder import subfolders_to_underfolder
 
 
 @click.group()
@@ -11,3 +12,4 @@ def conversions():
 
 conversions.add_command(underfolder_to_h5)
 conversions.add_command(h5_to_underfolder)
+conversions.add_command(subfolders_to_underfolder)
diff --git a/pipelime/cli/conversions/h5_to_underfolder.py b/pipelime/cli/conversions/h5_to_underfolder.py
index 01b5131..04fc911 100644
--- a/pipelime/cli/conversions/h5_to_underfolder.py
+++ b/pipelime/cli/conversions/h5_to_underfolder.py
@@ -1,7 +1,7 @@
 import click
 
 
-@click.command("h5_to_underfolder", help="Sum input underfolders")
+@click.command("h5_to_underfolder", help="Converts H5 dataset to Underfolder")
 @click.option(
     "-i", "--input_filename", required=True, type=str, help="Input H5 Filename"
 )
diff --git a/pipelime/cli/conversions/subfolders_to_underfolder.py b/pipelime/cli/conversions/subfolders_to_underfolder.py
new file mode 100644
index 0000000..8789de9
--- /dev/null
+++ b/pipelime/cli/conversions/subfolders_to_underfolder.py
@@ -0,0 +1,58 @@
+import click
+
+from pipelime.pipes.piper import Piper, PiperCommand
+
+
+@click.command("subfolders_to_underfolder", help="Converts Subfolders to Underfolder")
+@click.option("-i", "--input_folder", required=True, type=str, help="Input H5 Filename")
+@click.option(
+    "-o", "--output_folder", required=True, type=str, help="Output Underfolder"
+)
+@click.option(
+    "-e",
+    "--extension",
+    required=True,
+    type=str,
+    help="Image extension to include in conversion",
+)
+@click.option(
+    "-s",
+    "--use_symlinks",
+    required=False,
+    type=bool,
+    default=False,
+    help="Use symlinks instead of copying files",
+)
+@click.option(
+    "-w",
+    "--num_workers",
+    required=False,
+    type=int,
+    default=0,
+    help="Number of workers to use",
+)
+@Piper.command(
+    inputs=["input_folder"],
+    outputs=["output_folder"],
+)
+def subfolders_to_underfolder(
+    input_folder: str,
+    output_folder: str,
+    extension: str,
+    use_symlinks: bool,
+    num_workers: int,
+):
+
+    import rich
+
+    from pipelime.converters.subfolders2underfolder import Subfolders2Underfolder
+
+    converter = Subfolders2Underfolder(
+        folder=input_folder,
+        images_extension=extension,
+        use_symlinks=use_symlinks,
+        num_workers=num_workers,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
+    )
+    converter.convert(output_folder)
+    rich.print("Underfolder Writer output to:", output_folder)
diff --git a/pipelime/cli/conversions/underfolder_to_h5.py b/pipelime/cli/conversions/underfolder_to_h5.py
index 6e0eebe..076503a 100644
--- a/pipelime/cli/conversions/underfolder_to_h5.py
+++ b/pipelime/cli/conversions/underfolder_to_h5.py
@@ -1,7 +1,7 @@
 import click
 
 
-@click.command("underfolder_to_h5", help="Sum input underfolders")
+@click.command("underfolder_to_h5", help="Converts Underfolder to H5")
 @click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
 @click.option("-o", "--output_filename", required=True, type=str, help="Output H5 File")
 @click.option(
diff --git a/pipelime/cli/datasets/generate_toy_dataset.py b/pipelime/cli/datasets/generate_toy_dataset.py
index 689c5cd..1130329 100644
--- a/pipelime/cli/datasets/generate_toy_dataset.py
+++ b/pipelime/cli/datasets/generate_toy_dataset.py
@@ -1,17 +1,51 @@
 import click
 
-from pipelime.tools.toydataset import ToyDatasetGenerator
+from pipelime.pipes.piper import Piper, PiperCommand
 
 
 @click.command("generate_toy_dataset")
-@click.option("--output_folder", type=str, required=True, help="Output folder")
-@click.option("--size", type=int, required=True, help="Number of output samples")
+@click.option("-o", "--output_folder", type=str, required=True, help="Output folder")
+@click.option("-s", "--size", type=int, required=True, help="Number of output samples")
 @click.option("--image_size", type=int, default=256, help="Output image size")
 @click.option("--zfill", type=int, default=5, help="Zero padding digits")
-def generate_toy_dataset(output_folder, size, image_size, zfill):
+@click.option(
+    "--max_label", type=int, default=5, help="Generate class labels in range [0, <arg>]"
+)
+@click.option(
+    "--nr_objs",
+    type=int,
+    nargs=2,
+    default=(1, 4),
+    help="The number of objects in every image will be in this range (endpoints included)",
+)
+@click.option(
+    "--suffix", type=str, default="", help="Suffix to add to the output files"
+)
+@Piper.command(outputs=["output_folder"])
+def generate_toy_dataset(
+    output_folder,
+    size,
+    image_size,
+    zfill,
+    max_label,
+    nr_objs,
+    suffix,
+):
 
+    from pathlib import Path
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+    from pipelime.tools.toydataset import ToyDatasetGenerator
+
+    output_folder = Path(output_folder) / UnderfolderReader.DATA_SUBFOLDER
     ToyDatasetGenerator.generate_toy_dataset(
-        output_folder=output_folder, size=size, image_size=image_size, zfill=zfill
+        output_folder=output_folder,
+        size=size,
+        image_size=image_size,
+        zfill=zfill,
+        suffix=suffix,
+        max_label=max_label,
+        objects_number_range=(nr_objs[0], nr_objs[1] + 1),
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
 
 
diff --git a/pipelime/cli/dump.py b/pipelime/cli/dump.py
new file mode 100644
index 0000000..accf90a
--- /dev/null
+++ b/pipelime/cli/dump.py
@@ -0,0 +1,428 @@
+from pipelime.sequences.samples import Sample, FileSystemItem
+from pipelime.filesystem.toolkit import FSToolkit
+
+import numpy as np
+import imageio
+from rich.progress import Progress
+from rich.console import Console
+from sys import stderr
+
+import shutil
+from enum import Enum
+from pathlib import Path
+from itertools import islice
+
+from abc import ABC, abstractmethod
+from typing import Union, Optional, Any, Sequence, Mapping, Collection, Tuple
+
+
+class DataWriter(ABC):
+    @abstractmethod
+    def index_node(self, idx: int) -> "DataWriter":
+        ...
+
+    @abstractmethod
+    def key_node(self, key: str) -> "DataWriter":
+        ...
+
+    @abstractmethod
+    def coordinate_node(self, coord: Tuple) -> "DataWriter":
+        ...
+
+    @abstractmethod
+    def dump_leaf(self, elem) -> str:
+        ...
+
+
+class HeaderDataWriter(DataWriter):
+    def __init__(self, prefix: str, sep: str):
+        self._prefix = prefix
+        self._sep = sep
+
+    def index_node(self, idx: int) -> DataWriter:
+        return HeaderDataWriter(self._prefix + f"[{idx}]", self._sep)
+
+    def key_node(self, key: str) -> DataWriter:
+        return HeaderDataWriter(
+            self._prefix + f">{key}" if self._prefix else f"{key}", self._sep
+        )
+
+    def coordinate_node(self, coords: Tuple) -> DataWriter:
+        coord_str = ":".join([str(idx) for idx in coords])
+        return HeaderDataWriter(self._prefix + f"({coord_str})", self._sep)
+
+    def dump_leaf(self, elem) -> str:
+        return self._sep + self._prefix
+
+
+class HeaderDataTypeWriter(DataWriter):
+    def __init__(self, node_name: str, sep: str, type_map: Mapping[str, str]):
+        self._node_name = node_name
+        self._sep = sep
+        self._type_map = type_map
+
+    def index_node(self, idx: int) -> DataWriter:
+        return HeaderDataTypeWriter(
+            self._node_name + f"#{idx}", self._sep, self._type_map
+        )
+
+    def key_node(self, key: str) -> DataWriter:
+        return HeaderDataTypeWriter(
+            self._node_name + f"${key}", self._sep, self._type_map
+        )
+
+    def coordinate_node(self, coords: Tuple) -> DataWriter:
+        coord_str = ":".join([str(idx) for idx in coords])
+        return HeaderDataTypeWriter(
+            self._node_name + f"@{coord_str}", self._sep, self._type_map
+        )
+
+    def dump_leaf(self, elem) -> str:
+        type_str = self._type_map.get(self._node_name)
+        if type_str is None:
+            type_str = (
+                "s"
+                if isinstance(elem, (str, bytes))
+                else "c"
+            )
+        return self._sep + type_str
+
+
+class HeaderDataRoleWriter(DataWriter):
+    def __init__(
+        self,
+        node_name: str,
+        sep: str,
+        output_path: Optional[Union[Path, str]],
+        role_map: Mapping[str, str],
+    ):
+        self._node_name = node_name
+        self._sep = sep
+        self._output_path = "" if output_path is None else str(output_path)
+        self._role_map = role_map
+
+    def index_node(self, idx: int) -> DataWriter:
+        return HeaderDataRoleWriter(
+            self._node_name + f"#{idx}", self._sep, self._output_path, self._role_map
+        )
+
+    def key_node(self, key: str) -> DataWriter:
+        return HeaderDataRoleWriter(
+            self._node_name + f"${key}", self._sep, self._output_path, self._role_map
+        )
+
+    def coordinate_node(self, coords: Tuple) -> DataWriter:
+        coord_str = ":".join([str(idx) for idx in coords])
+        return HeaderDataRoleWriter(
+            self._node_name + f"@{coord_str}",
+            self._sep,
+            self._output_path,
+            self._role_map,
+        )
+
+    def dump_leaf(self, elem) -> str:
+        role_str = self._role_map.get(self._node_name)
+        if role_str is None:
+            if isinstance(elem, (str, bytes)):
+                elem = str(elem)
+                if Path(elem).is_file() and FSToolkit.is_image_file(elem):
+                    role_str = f"type=image origin={self._output_path}"
+                else:
+                    role_str = "meta"
+            else:
+                role_str = ""
+        return self._sep + role_str
+
+
+class ValueDataWriter(DataWriter):
+    def __init__(self, sep: str, output_path: Optional[Union[Path, str]]):
+        self._sep = sep
+        self._output_path = None if output_path is None else Path(output_path)
+
+    def index_node(self, idx: int) -> DataWriter:
+        return ValueDataWriter(self._sep, self._output_path)
+
+    def key_node(self, key: str) -> DataWriter:
+        return ValueDataWriter(self._sep, self._output_path)
+
+    def coordinate_node(self, coord: Tuple) -> DataWriter:
+        return ValueDataWriter(self._sep, self._output_path)
+
+    def dump_leaf(self, elem) -> str:
+        if isinstance(elem, (str, bytes)):
+            elem = str(elem)
+            if Path(elem).is_file() and FSToolkit.is_image_file(elem):
+                elem = Path(elem)
+                if self._output_path is not None:
+                    elem = str(elem.relative_to(self._output_path))
+        return self._sep + str(elem)
+
+
+def _dump_collection(x: Any, writer: DataWriter) -> str:
+    if isinstance(x, Collection) and not isinstance(x, (str, bytes)):
+        outline = ""
+        if isinstance(x, Mapping):
+            for key, elem in x.items():
+                outline += _dump_collection(elem, writer.key_node(key))
+        elif isinstance(x, np.ndarray):
+            it = np.nditer(x, flags=["multi_index"])
+            for elem in it:
+                outline += _dump_collection(
+                    elem.item(), writer.coordinate_node(it.multi_index)  # type: ignore
+                )
+        else:
+            for idx, elem in enumerate(x):
+                outline += _dump_collection(elem, writer.index_node(idx))
+        return outline
+    else:
+        return writer.dump_leaf(x)
+
+
+class LinkType(Enum):
+    DEEP_COPY = 0
+    SYM_LINK = 1
+    HARD_LINK = 2
+    # alias to ease cli creation
+    copy = DEEP_COPY
+    soft = SYM_LINK
+    hard = HARD_LINK
+
+
+def _make_sample_iterator(
+    samples: Sequence[Sample],
+    start: Optional[int],
+    stop: Optional[int],
+    step: Optional[int],
+):
+    index_range = range(len(samples))
+    if step is None:
+        step = 1
+    if start is None:
+        start = 0 if step > 0 else -1
+    if start < 0:
+        start = len(samples) + start
+    if stop is not None and stop < 0:
+        stop = len(samples) + stop
+    if step < 0:
+        start = len(samples) - 1 - start
+        if stop is not None:
+            stop = len(samples) - 1 - stop
+        step = -step
+        samples = reversed(samples)  # type: ignore
+        index_range = reversed(index_range)
+
+    from math import trunc
+
+    return zip(
+        islice(index_range, start, stop, step), islice(samples, start, stop, step)
+    ), trunc(((len(samples) if stop is None else stop) - 1 - start) / step + 1)
+
+
+def _link_file(
+    src_path: Path, target_folder: Optional[Path], link_type: LinkType
+) -> str:
+    if target_folder is None or not target_folder.exists():
+        return str((Path("") / src_path.name).resolve())  # fake path
+
+    trg_file = target_folder / src_path.name
+
+    if link_type is LinkType.DEEP_COPY:
+        shutil.copy(src_path, str(trg_file))
+    elif link_type is LinkType.SYM_LINK:
+        trg_file.symlink_to(src_path)
+    elif link_type is LinkType.HARD_LINK:
+        try:
+            # (new in version 3.10)
+            trg_file.hardlink_to(src_path)  # type: ignore
+        except AttributeError:
+            import os
+
+            os.link(str(src_path), str(trg_file))
+
+    return str(trg_file)
+
+
+def _write_image(
+    image: np.ndarray, filename: str, target_folder: Optional[Path]
+) -> str:
+    if target_folder is None or not target_folder.exists():
+        return str((Path("") / filename).resolve())  # fake path
+
+    trg_file = target_folder / Path(filename)
+    imageio.imwrite(trg_file, image)
+    return str(trg_file)
+
+
+def _link_or_load(
+    sample: Sample, item_key: str, target_folder: Optional[Path], link_type: LinkType
+) -> Any:
+    # link images, load everything else
+    item = sample.metaitem(item_key)
+    return (
+        _link_file(Path(item.source()), target_folder, link_type)
+        if isinstance(item, FileSystemItem)
+        and FSToolkit.is_image_file(str(item.source()))
+        else sample[item_key]  # this may load a file from disk
+    )
+
+
+def _maybe_write_data(
+    data: Any, sample_index: int, item_key: str, target_folder: Optional[Path]
+) -> Any:
+    # dump image-like data to disk
+    if (
+        isinstance(data, np.ndarray)
+        and data.dtype == "uint8"
+        and (
+            len(data.shape) == 2  # (H, W) mono
+            or (
+                len(data.shape) == 3  # mono, rgb, rgba
+                and (
+                    data.shape[0] in (1, 3, 4)  # (C, H, W)
+                    or data.shape[-1] in (1, 3, 4)  # (H, W, C)
+                )
+            )
+        )
+    ):
+        if len(data.shape) == 3:
+            if data.shape[0] in (1, 3, 4):
+                data = np.moveaxis(data, -3, -1)
+            if data.shape[-1] == 1:
+                data = np.squeeze(data, axis=-1)
+        return _write_image(data, f"{sample_index}-{item_key}.png", target_folder)
+
+    return data
+
+
+def _extract_samples(
+    assets_path: Optional[Path],
+    link_type: LinkType,
+    samples: Sequence[Sample],
+    start: Optional[int],
+    stop: Optional[int],
+    step: Optional[int],
+    progress: Progress,
+) -> Sequence[Tuple[int, Mapping[str, Any]]]:
+    filtered_samples: Sequence[Tuple[int, Mapping[str, Any]]] = []
+    sample_it, size = _make_sample_iterator(samples, start, stop, step)
+
+    task = progress.add_task("Reading samples...", total=size)
+    for index, src in sample_it:
+        dst: dict[str, Any] = {}
+        for k in src:
+            dst_data = _link_or_load(src, k, assets_path, link_type)
+            dst[k] = _maybe_write_data(dst_data, len(filtered_samples), k, assets_path)
+
+        filtered_samples.append((index, dst))
+        progress.update(task, advance=1)
+
+    return filtered_samples
+
+
+def _print_line(first_col: Any, others: str, file: Optional[Any]):
+    print(first_col, end="", file=file)
+    print(others, end="\n", file=file)
+
+
+def dump_data(
+    samples: Sequence[Sample],
+    output_assets_path: Optional[Union[Path, str]] = None,
+    start: Optional[int] = None,
+    stop: Optional[int] = None,
+    step: Optional[int] = None,
+    format: str = "orange",
+    link_type: LinkType = LinkType.HARD_LINK,
+    file: Optional[Any] = None,
+) -> None:
+    """Sample data dumping to console or file.
+
+    Args:
+        samples (Sequence[Sample]): the sequence of samples.
+        output_assets_path (Optional[Union[Path, str]]): where images and other file
+            assets will be saved.
+        start (Optional[int], optional): first sample index. Defaults to None.
+        stop (Optional[int], optional): last sample index (excluded). Defaults to None.
+        step (Optional[int], optional): range step. Defaults to None.
+        format (str, optional): output format ('csv', 'orange'). Defaults to 'orange'.
+        link_type (LinkType, optional): if soft/hard links should be used whenever
+            possible when writing assets. Defaults to LinkType.HARD_LINK.
+        file (Optional[Any], optional): an optional opened file stream, if None it
+            prints to stdout. Defaults to None.
+    """
+    if output_assets_path is not None:
+        output_assets_path = Path(output_assets_path).resolve()
+        output_assets_path.mkdir(parents=True, exist_ok=True)
+
+    with Progress(console=Console(file=stderr)) as progress:
+        filtered_samples = _extract_samples(
+            output_assets_path,
+            link_type,
+            samples,
+            start,
+            stop,
+            step,
+            progress,
+        )
+
+        format = format.lower()
+        separator = "\t" if format == "orange" else ","
+        samples_it = iter(filtered_samples)
+
+        # write header from a prototype
+        first_idx, first_sample = next(samples_it)
+
+        task = progress.add_task("Writing data...", total=len(filtered_samples) + 1)
+
+        _print_line(
+            "$index",
+            _dump_collection(first_sample, HeaderDataWriter(prefix="", sep=separator)),
+            file,
+        )
+
+        if format == "orange":
+            _print_line(
+                "c",
+                _dump_collection(
+                    first_sample,
+                    HeaderDataTypeWriter(node_name="", sep=separator, type_map={}),
+                ),
+                file,
+            )
+            _print_line(
+                "meta",
+                _dump_collection(
+                    first_sample,
+                    HeaderDataRoleWriter(
+                        node_name="",
+                        sep=separator,
+                        output_path=output_assets_path,
+                        role_map={},
+                    ),
+                ),
+                file,
+            )
+
+        progress.update(task, advance=1)
+
+        # write values
+        _print_line(
+            first_idx,
+            _dump_collection(
+                first_sample,
+                ValueDataWriter(sep=separator, output_path=output_assets_path),
+            ),
+            file,
+        )
+
+        progress.update(task, advance=1)
+
+        for sample_idx, sample_data in samples_it:
+            _print_line(
+                sample_idx,
+                _dump_collection(
+                    sample_data,
+                    ValueDataWriter(sep=separator, output_path=output_assets_path),
+                ),
+                file,
+            )
+            progress.update(task, advance=1)
diff --git a/pipelime/cli/main.py b/pipelime/cli/main.py
index 5151031..7c5690b 100644
--- a/pipelime/cli/main.py
+++ b/pipelime/cli/main.py
@@ -5,6 +5,7 @@
 from pipelime.cli.h5.h5 import h5
 from pipelime.cli.underfolder.underfolder import underfolder
 from pipelime.cli.workflow.workflow import workflow
+from pipelime.cli.piper.piper import piper
 
 
 @click.group()
@@ -12,6 +13,7 @@ def pipelime():
     pass
 
 
+pipelime.add_command(piper)
 pipelime.add_command(datasets)
 pipelime.add_command(workflow)
 pipelime.add_command(conversions)
diff --git a/tests/pipelime/augmentations/__init__.py b/pipelime/cli/piper/__init__.py
similarity index 100%
rename from tests/pipelime/augmentations/__init__.py
rename to pipelime/cli/piper/__init__.py
diff --git a/pipelime/cli/piper/piper.py b/pipelime/cli/piper/piper.py
new file mode 100644
index 0000000..8514450
--- /dev/null
+++ b/pipelime/cli/piper/piper.py
@@ -0,0 +1,220 @@
+from calendar import c
+from typing import Optional
+import click
+
+
+@click.group()
+def piper():
+    pass
+
+
+@piper.command(
+    "compile",
+    context_settings=dict(
+        ignore_unknown_options=True,
+        allow_extra_args=True,
+    ),
+)
+@click.option(
+    "-i",
+    "--piper_file",
+    type=click.Path(exists=True),
+    default="dag.yml",
+)
+@click.option(
+    "-p",
+    "--piper_params_file",
+    type=click.Path(),
+    default="params.yml",
+)
+@click.option(
+    "-o",
+    "--output_file",
+    type=click.Path(),
+    default="",
+)
+@click.pass_context
+def compile(
+    ctx: click.Context,
+    piper_file: str,
+    piper_params_file: str,
+    output_file: str,
+):
+
+    import rich
+    from pipelime.tools.click import ClickTools
+    from choixe.configurations import XConfig
+    import rich
+    from pipelime.pipes.parsers.factory import DAGConfigParserFactory
+
+    # DAG Model (abstraction layer to allow several parsing methods)
+    try:
+        dag = DAGConfigParserFactory.parse_file(
+            cfg_file=piper_file,
+            params_file=piper_params_file,
+            additional_args=ClickTools.parse_additional_args(ctx),  # additional args
+        )
+    except KeyError as e:
+        raise click.UsageError(f"Something is missing! -> {e}")
+
+    cfg = XConfig.from_dict(dag.purged_dict())
+
+    if len(output_file) > 0:
+        cfg.save_to(output_file)
+        rich.print("Saved to:", output_file)
+    else:
+        rich.print(cfg.to_dict())
+
+
+@piper.command(
+    "draw",
+    context_settings=dict(
+        ignore_unknown_options=True,
+        allow_extra_args=True,
+    ),
+)
+@click.option(
+    "-i",
+    "--piper_file",
+    type=click.Path(exists=True),
+    default="dag.yml",
+)
+@click.option(
+    "-p",
+    "--piper_params_file",
+    type=click.Path(exists=True),
+    default="params.yml",
+)
+@click.option(
+    "-b",
+    "--draw_backend",
+    # TODO: don't use factory list to avoid missing dependencies check on Workflows
+    type=click.Choice(["graphviz", "mermaid"]),
+    default="graphviz",
+)
+@click.option(
+    "-o",
+    "--output_file",
+    type=click.Path(),
+    default="",
+)
+@click.option(
+    "--open/--no-open",
+    type=bool,
+    default=False,
+    help="If TRUE, Open the output with corresponding default app",
+)
+@click.pass_context
+def draw(
+    ctx: click.Context,
+    piper_file: str,
+    piper_params_file: str,
+    draw_backend: str,
+    output_file: str,
+    open: bool,
+):
+
+    from pipelime.pipes.drawing.factory import NodesGraphDrawerFactory
+    from pipelime.pipes.parsers.factory import DAGConfigParserFactory
+    from pipelime.filesystem.toolkit import FSToolkit
+    from pipelime.pipes.graph import DAGNodesGraph
+    from pipelime.tools.click import ClickTools
+    import numpy as np
+    import rich
+    import cv2
+
+    # DAG Model (abstraction layer to allow several parsing methods)
+    dag = DAGConfigParserFactory.parse_file(
+        cfg_file=piper_file,
+        params_file=piper_params_file,
+        additional_args=ClickTools.parse_additional_args(ctx),  # additional args
+    )
+
+    # Graph
+    graph = DAGNodesGraph.build_nodes_graph(dag)
+
+    # Drawn image
+    graph_image: Optional[np.ndarray] = None
+
+    # Draw with selected backend
+    drawer = NodesGraphDrawerFactory.create(draw_backend)
+
+    # Show or Write
+    if len(output_file) > 0:
+        drawer.export(graph, output_file)
+        if open:
+            FSToolkit.start_file(output_file)
+        else:
+            rich.print("graph image saved to:", output_file)
+    else:
+        graph_image = drawer.draw(graph=graph)
+        cv2.imshow("graph", cv2.cvtColor(graph_image, cv2.COLOR_RGB2BGR))
+        cv2.waitKey(0)
+
+
+@piper.command(
+    "execute",
+    context_settings=dict(
+        ignore_unknown_options=True,
+        allow_extra_args=True,
+    ),
+)
+@click.option(
+    "-i",
+    "--piper_file",
+    type=click.Path(exists=True),
+    default="dag.yml",
+)
+@click.option(
+    "-p",
+    "--piper_params_file",
+    type=click.Path(),
+    default="params.yml",
+)
+@click.option(
+    "-b",
+    "--execution_backend",
+    type=click.Choice(["naive"]),
+    default="naive",
+)
+@click.option(
+    "-t",
+    "--token",
+    type=str,
+    default="",
+)
+@click.pass_context
+def execute(
+    ctx: click.Context,
+    piper_file: str,
+    piper_params_file: str,
+    execution_backend: str,
+    token: str,
+):
+
+    from pipelime.pipes.parsers.factory import DAGConfigParserFactory
+    from pipelime.pipes.executors.naive import NaiveGraphExecutor
+    from pipelime.pipes.graph import DAGNodesGraph
+    from pipelime.tools.click import ClickTools
+
+    # DAG Model (abstraction layer to allow several parsing methods)
+    dag = DAGConfigParserFactory.parse_file(
+        cfg_file=piper_file,
+        params_file=piper_params_file,
+        additional_args=ClickTools.parse_additional_args(ctx),  # additional args
+    )
+
+    # Graph
+    graph = DAGNodesGraph.build_nodes_graph(dag)
+
+    if execution_backend == "naive":
+        executor = NaiveGraphExecutor()
+        executor.exec(graph, token=token)
+    else:
+        raise NotImplementedError(
+            f"Execution backend {execution_backend} not implemented"
+        )
+
+
+if __name__ == "__main__":
+    piper()
diff --git a/pipelime/cli/summary.py b/pipelime/cli/summary.py
index 4f4e6ab..7ac9f19 100644
--- a/pipelime/cli/summary.py
+++ b/pipelime/cli/summary.py
@@ -83,7 +83,7 @@ def print(self, summary: ReaderSummary) -> None:
             str_ = "based on first sample"
         else:
             str_ = "based on all samples"
-        console.print(f"[{style}]Contents[/{style}] \[{str_}]:")
+        console.print(f"[{style}]Contents[/{style}] \\[{str_}]:")
 
         table = Table(header_style=self.TITLE_STYLE)
         table.add_column("Item Name")
diff --git a/pipelime/cli/underfolder/operations.py b/pipelime/cli/underfolder/operations.py
index b50fefd..f74fb3e 100644
--- a/pipelime/cli/underfolder/operations.py
+++ b/pipelime/cli/underfolder/operations.py
@@ -1,7 +1,14 @@
 from pathlib import Path
+from typing import Sequence
 
 import click
 
+from pipelime.pipes.piper import Piper, PiperCommand
+
+hardlink_option = click.option(
+    "--hardlink", is_flag=True, help="Use hardlinks for cached items"
+)
+
 
 @click.command("sum", help="Sum input underfolders")
 @click.option(
@@ -9,11 +16,15 @@
     "--input_folders",
     required=True,
     multiple=True,
-    type=str,
+    type=click.Path(exists=True),
     help="Input Underfolder [multiple]",
 )
 @click.option(
-    "-o", "--output_folder", required=True, type=str, help="Output Underfolder"
+    "-o",
+    "--output_folder",
+    required=True,
+    type=click.Path(),
+    help="Output Underfolder",
 )
 @click.option(
     "-c",
@@ -22,11 +33,20 @@
     type=str,
     help="Convert a root file into an item to avoid conflicts [multiple]",
 )
-def operation_sum(input_folders, output_folder, convert_root_file):
+@hardlink_option
+@Piper.command(inputs=["input_folders"], outputs=["output_folder"])
+def operation_sum(
+    input_folders: Sequence[str],
+    output_folder: str,
+    convert_root_file: Sequence[str],
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import OperationResetIndices, OperationSum
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    pipercmd = PiperCommand.instance
 
     if len(input_folders) > 0:
         datasets = [
@@ -40,15 +60,20 @@ def operation_sum(input_folders, output_folder, convert_root_file):
 
         prototype_reader = datasets[0]
         template = prototype_reader.get_reader_template()
+        template.root_files_keys = list(
+            set(template.root_files_keys) - set(convert_root_file)
+        )
 
-        writer = UnderfolderWriter(
+        copy_mode = (
+            UnderfolderWriterV2.CopyMode.HARD_LINK
+            if hardlink
+            else UnderfolderWriterV2.CopyMode.DEEP_COPY
+        )
+        writer = UnderfolderWriterV2(
             folder=output_folder,
-            copy_files=True,
-            root_files_keys=list(
-                set(template.root_files_keys) - set(convert_root_file)
-            ),
-            extensions_map=template.extensions_map,
-            zfill=template.idx_length,
+            copy_mode=copy_mode,
+            reader_template=template,
+            progress_callback=pipercmd.generate_progress_callback(),
         )
         writer(output_dataset)
 
@@ -59,19 +84,30 @@ def operation_sum(input_folders, output_folder, convert_root_file):
     "--input_folders",
     required=True,
     multiple=True,
-    type=str,
+    type=click.Path(exists=True),
     help="Input Underfolder [multiple]",
 )
 @click.option(
-    "-o", "--output_folder", required=True, type=str, help="Output Underfolder"
+    "-o",
+    "--output_folder",
+    required=True,
+    type=click.Path(),
+    help="Output Underfolder",
 )
-def operation_mix(input_folders, output_folder):
+@hardlink_option
+@Piper.command(inputs=["input_folders"], outputs=["output_folder"])
+def operation_mix(
+    input_folders: Sequence[str],
+    output_folder: str,
+    hardlink: bool,
+):
 
     from click import ClickException
 
     from pipelime.sequences.operations import OperationMix
+    from pipelime.sequences.readers.base import ReaderTemplate
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
 
     if len(input_folders) > 0:
         datasets = [
@@ -84,27 +120,42 @@ def operation_mix(input_folders, output_folder):
             output_dataset = op_mix(datasets)
         except AssertionError as e:
             raise ClickException(
-                "Input underfolders must have the same length and their item sets must be disjoint."
+                "Input underfolders must have the same length and their item sets must "
+                f"be disjoint: {e}"
             )
 
         root_files = []
         ext_map = {}
+        idx_length = 0
         for d in datasets:
             template = d.get_reader_template()
             root_files.extend(template.root_files_keys)
             ext_map.update(template.extensions_map)
+            idx_length = max(idx_length, template.idx_length)
+        template = ReaderTemplate(ext_map, root_files, idx_length)
 
-        writer = UnderfolderWriter(
+        copy_mode = (
+            UnderfolderWriterV2.CopyMode.HARD_LINK
+            if hardlink
+            else UnderfolderWriterV2.CopyMode.DEEP_COPY
+        )
+        writer = UnderfolderWriterV2(
             folder=output_folder,
-            copy_files=True,
-            root_files_keys=root_files,
-            extensions_map=ext_map,
+            copy_mode=copy_mode,
+            reader_template=template,
+            progress_callback=PiperCommand.instance.generate_progress_callback(),
         )
         writer(output_dataset)
 
 
 @click.command("subsample", help="Subsample input underfolder")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
 @click.option(
     "-f",
     "--factor",
@@ -113,42 +164,96 @@ def operation_mix(input_folders, output_folder):
     help="Subsamplig factor. INT (1,inf) or FLOAT [0.0,1.0]",
 )
 @click.option(
-    "-o", "--output_folder", required=True, type=str, help="Output Underfolder"
+    "-s",
+    "--start",
+    type=float,
+    default=0,
+    help="Subsamplig start. INT (0,inf) or FLOAT [0.0,1.0]",
 )
-def operation_subsample(input_folder, factor, output_folder):
+@click.option(
+    "-o",
+    "--output_folder",
+    required=True,
+    type=click.Path(),
+    help="Output Underfolder",
+)
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_subsample(
+    input_folder: str,
+    factor: float,
+    start: float,
+    output_folder: str,
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import OperationResetIndices, OperationSubsample
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
 
     # operations
-    op_subsample = OperationSubsample(factor=int(factor) if factor > 1 else factor)
+    op_subsample = OperationSubsample(
+        factor=int(factor) if factor > 1 else factor,
+        start=int(start) if factor > 1 else start,
+    )
     op_reindex = OperationResetIndices()
     output_dataset = op_reindex(op_subsample(dataset))
 
-    writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
         folder=output_folder,
-        copy_files=True,
-        root_files_keys=template.root_files_keys,
-        extensions_map=template.extensions_map,
+        copy_mode=copy_mode,
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
     writer(output_dataset)
 
 
 @click.command("shuffle", help="Shuffle input underfolder")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
 @click.option(
-    "-o", "--output_folder", required=True, type=str, help="Output Underfolder"
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
 )
-@click.option("-s", "--seed", default=-1, type=int, help="Random seed")
-def operation_shuffle(input_folder, output_folder, seed):
+@click.option(
+    "-o",
+    "--output_folder",
+    required=True,
+    type=click.Path(),
+    help="Output Underfolder",
+)
+@click.option(
+    "-s",
+    "--seed",
+    default=-1,
+    type=int,
+    help="Random seed",
+)
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_shuffle(
+    input_folder: str,
+    output_folder: str,
+    seed: int,
+    hardlink: bool,
+):
+
+    PiperCommand.instance
 
     from pipelime.sequences.operations import OperationResetIndices, OperationShuffle
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
@@ -158,75 +263,137 @@ def operation_shuffle(input_folder, output_folder, seed):
     op_reindex = OperationResetIndices()
     output_dataset = op_reindex(op_shuffle(dataset))
 
-    writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
         folder=output_folder,
-        copy_files=True,
-        root_files_keys=template.root_files_keys,
-        extensions_map=template.extensions_map,
+        copy_mode=copy_mode,
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
     writer(output_dataset)
 
 
 @click.command("split", help="Split input underfolder")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
+@click.option(
+    "-o",
+    "--output_folders",
+    required=True,
+    multiple=True,
+    type=click.Path(),
+    help="Splits path [multple]",
+)
 @click.option(
     "-s",
     "--splits",
     required=True,
     multiple=True,
-    nargs=2,
-    help="Splits map, pairs (path, percentage) [multple]",
+    type=float,
+    help="Splits percentages [multple]",
 )
-def operation_split(input_folder, splits):
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folders"])
+def operation_split(
+    input_folder: str,
+    output_folders: Sequence[str],
+    splits: Sequence[float],
+    hardlink: bool,
+):
 
     from functools import reduce
 
     from pipelime.sequences.operations import OperationResetIndices, OperationSplits
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
 
     # splitmap for nargs
-    split_map = {str(k): float(v) for k, v in splits}
+    if len(output_folders) != len(splits):
+        raise click.UsageError(
+            "Number of splits must be equal to number of output folders"
+        )
 
+    split_map = {str(k): float(v) for k, v in zip(output_folders, splits)}
     cumulative = reduce(lambda a, b: a + b, split_map.values())
-    assert cumulative <= 1.0, "Sums of splits percentages must be <= 1.0"
+    if cumulative != 1.0:
+        raise click.UsageError("Sums of splits percentages must be = 1.0")
 
     # operations
     op_splits = OperationSplits(split_map=split_map)
     op_reindex = OperationResetIndices()
     output_datasets_map = op_splits(dataset)
 
-    for path, dataset in output_datasets_map.items():
-        writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    for index, (path, dataset) in enumerate(output_datasets_map.items()):
+        writer = UnderfolderWriterV2(
             folder=path,
-            copy_files=True,
-            root_files_keys=template.root_files_keys,
-            extensions_map=template.extensions_map,
+            copy_mode=copy_mode,
+            reader_template=template,
+            progress_callback=PiperCommand.instance.generate_progress_callback(
+                chunk_index=index,
+                total_chunks=len(output_datasets_map),
+            ),
         )
         writer(op_reindex(dataset))
 
 
 @click.command("filter_by_query", help="Filter input underfolder by query")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
-@click.option("-q", "--query", required=True, type=str, help="Filtering query")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
+@click.option(
+    "-q",
+    "--query",
+    required=True,
+    type=str,
+    help="Filtering query",
+)
 @click.option(
     "-o",
     "--output_folder",
     required=True,
-    type=str,
+    type=click.Path(),
     help="Output Underfolder for positive matches",
 )
-def operation_filterbyquery(input_folder, query, output_folder):
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_filterbyquery(
+    input_folder: str,
+    query: str,
+    output_folder: str,
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import (
         OperationFilterByQuery,
         OperationResetIndices,
     )
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
@@ -236,40 +403,70 @@ def operation_filterbyquery(input_folder, query, output_folder):
     op_reindex = OperationResetIndices()
     output_dataset = op_reindex(op_filterbyquery(dataset))
 
-    writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
         folder=output_folder,
-        copy_files=True,
-        root_files_keys=template.root_files_keys,
-        extensions_map=template.extensions_map,
+        copy_mode=copy_mode,
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
     writer(output_dataset)
 
 
 @click.command("split_by_query", help="Filter input underfolder by query")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
-@click.option("-q", "--query", required=True, type=str, help="Filtering query")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
+@click.option(
+    "-q",
+    "--query",
+    required=True,
+    type=str,
+    help="Filtering query",
+)
 @click.option(
     "-o1",
     "--output_folder_1",
     required=True,
-    type=str,
+    type=click.Path(),
     help="Output Underfolder for positive matches",
 )
 @click.option(
     "-o2",
     "--output_folder_2",
     required=True,
-    type=str,
+    type=click.Path(),
     help="Output Underfolder for negative matches",
 )
-def operation_splitbyquery(input_folder, query, output_folder_1, output_folder_2):
+@hardlink_option
+@Piper.command(
+    inputs=["input_folder"],
+    outputs=["output_folder_1", "output_folder_2"],
+)
+def operation_splitbyquery(
+    input_folder: str,
+    query: str,
+    output_folder_1: str,
+    output_folder_2: str,
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import (
         OperationResetIndices,
         OperationSplitByQuery,
     )
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
@@ -281,34 +478,64 @@ def operation_splitbyquery(input_folder, query, output_folder_1, output_folder_2
 
     output_folders = [output_folder_1, output_folder_2]
 
-    for output_folder, output_dataset in zip(output_folders, output_datasets):
-        writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    for index, (output_folder, output_dataset) in enumerate(
+        zip(output_folders, output_datasets)
+    ):
+        writer = UnderfolderWriterV2(
             folder=output_folder,
-            copy_files=True,
-            root_files_keys=template.root_files_keys,
-            extensions_map=template.extensions_map,
+            copy_mode=copy_mode,
+            reader_template=template,
+            progress_callback=PiperCommand.instance.generate_progress_callback(
+                chunk_index=index, total_chunks=len(output_folders)
+            ),
         )
         writer(op_reindex(output_dataset))
 
 
 @click.command("filter_by_script", help="Filter input underfolder by query")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
-@click.option("-s", "--script", required=True, type=str, help="Filtering python script")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
+@click.option(
+    "-s",
+    "--script",
+    required=True,
+    type=str,
+    help="Filtering python script",
+)
 @click.option(
     "-o",
     "--output_folder",
     required=True,
-    type=str,
+    type=click.Path(),
     help="Output Underfolder for positive matches",
 )
-def operation_filterbyscript(input_folder, script, output_folder):
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_filterbyscript(
+    input_folder: str,
+    script: str,
+    output_folder: str,
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import (
         OperationFilterByScript,
         OperationResetIndices,
     )
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
@@ -318,17 +545,28 @@ def operation_filterbyscript(input_folder, script, output_folder):
     op_reindex = OperationResetIndices()
     output_dataset = op_reindex(op_filterbyquery(dataset))
 
-    writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
         folder=output_folder,
-        copy_files=True,
-        root_files_keys=template.root_files_keys,
-        extensions_map=template.extensions_map,
+        copy_mode=copy_mode,
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
     writer(output_dataset)
 
 
 @click.command("filter_keys", help="Filter input underfolder keys")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
 @click.option(
     "-k",
     "--keys",
@@ -347,14 +585,24 @@ def operation_filterbyscript(input_folder, script, output_folder):
     "-o",
     "--output_folder",
     required=True,
-    type=str,
+    type=click.Path(),
     help="Output Underfolder for positive matches",
 )
-def operation_filterkeys(input_folder, keys, negate, output_folder):
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_filterkeys(
+    input_folder: str,
+    keys: Sequence[str],
+    negate: bool,
+    output_folder: str,
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import OperationFilterKeys, OperationResetIndices
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
@@ -364,17 +612,28 @@ def operation_filterkeys(input_folder, keys, negate, output_folder):
     op_reindex = OperationResetIndices()
     output_dataset = op_reindex(op_filterbyquery(dataset))
 
-    writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
         folder=output_folder,
-        copy_files=True,
-        root_files_keys=template.root_files_keys,
-        extensions_map=template.extensions_map,
+        copy_mode=copy_mode,
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
     writer(output_dataset)
 
 
 @click.command("order_by", help="Order input underfolder samples")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
 @click.option(
     "-k",
     "--keys",
@@ -387,14 +646,23 @@ def operation_filterkeys(input_folder, keys, negate, output_folder):
     "-o",
     "--output_folder",
     required=True,
-    type=str,
+    type=click.Path(),
     help="Output Underfolder for positive matches",
 )
-def operation_orderby(input_folder, keys, output_folder):
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_orderby(
+    input_folder: str,
+    keys: Sequence[str],
+    output_folder: str,
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import OperationOrderBy, OperationResetIndices
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
     template = dataset.get_reader_template()
@@ -404,22 +672,41 @@ def operation_orderby(input_folder, keys, output_folder):
     op_reindex = OperationResetIndices()
     output_dataset = op_reindex(op_filterbyquery(dataset))
 
-    writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
         folder=output_folder,
-        copy_files=True,
-        root_files_keys=template.root_files_keys,
-        extensions_map=template.extensions_map,
+        copy_mode=copy_mode,
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
     writer(output_dataset)
 
 
 @click.command("split_by_value", help="Split input underfolder by value")
 @click.option(
-    "-i", "--input_folder", required=True, type=Path, help="Input Underfolder"
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
 )
-@click.option("-k", "--key", required=True, type=str, help="split key")
 @click.option(
-    "-o", "--output_folder", required=True, type=Path, help="Output parent folder"
+    "-k",
+    "--key",
+    required=True,
+    type=str,
+    help="split key",
+)
+@click.option(
+    "-o",
+    "--output_folder",
+    required=True,
+    type=click.Path(),
+    help="Output parent folder",
 )
 @click.option(
     "-p",
@@ -428,7 +715,16 @@ def operation_orderby(input_folder, keys, output_folder):
     type=str,
     help="Prefix for each output underfolder",
 )
-def operation_split_by_value(input_folder, key, output_folder, output_prefix):
+@hardlink_option
+def operation_split_by_value(
+    input_folder: str,
+    key: str,
+    output_folder: str,
+    output_prefix: str,
+    hardlink: bool,
+):
+
+    # #### CANNOT BE PIPER-ED!! DYNAMIC OUTPUTS PATHS SHOULD BE AVOIDED #####
 
     from math import ceil, log10
 
@@ -437,7 +733,7 @@ def operation_split_by_value(input_folder, key, output_folder, output_prefix):
         OperationSplitByValue,
     )
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
 
     dataset = UnderfolderReader(input_folder)
     template = dataset.get_reader_template()
@@ -446,49 +742,88 @@ def operation_split_by_value(input_folder, key, output_folder, output_prefix):
     splits = op_split(dataset)
 
     zfill = max(ceil(log10(len(splits) + 1)), 1)
+
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
     for i, split in enumerate(splits):
         op_reindex = OperationResetIndices()
         split = op_reindex(split)
-        UnderfolderWriter(
-            output_folder / f"{output_prefix}_{str(i).zfill(zfill)}",
-            root_files_keys=template.root_files_keys,
-            extensions_map=template.extensions_map,
+        UnderfolderWriterV2(
+            Path(output_folder) / f"{output_prefix}_{str(i).zfill(zfill)}",
+            copy_mode=copy_mode,
+            reader_template=template,
         )(split)
 
 
 @click.command("group_by", help="Group input underfolder by key")
-@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
-@click.option("-k", "--key", required=True, type=str, help="Grouping keys")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
+@click.option(
+    "-k",
+    "--key",
+    required=True,
+    type=str,
+    help="Grouping keys",
+)
 @click.option(
     "-o",
     "--output_folder",
     required=True,
-    type=str,
-    help="Output Underfolder for positive matches",
+    type=click.Path(),
+    help="Output Underfolder for grouped dataset",
 )
-def operation_groupby(input_folder, key, output_folder):
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_groupby(
+    input_folder: str,
+    key: str,
+    output_folder: str,
+    hardlink: bool,
+):
 
     from pipelime.sequences.operations import OperationGroupBy, OperationResetIndices
     from pipelime.sequences.readers.filesystem import UnderfolderReader
-    from pipelime.sequences.writers.filesystem import UnderfolderWriter
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
 
     dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
-    template = dataset.get_reader_template()
 
     # operations
     op_filterbyquery = OperationGroupBy(field=key)
     op_reindex = OperationResetIndices()
     output_dataset = op_reindex(op_filterbyquery(dataset))
 
-    writer = UnderfolderWriter(
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
         folder=output_folder,
-        copy_files=True,
+        copy_mode=copy_mode,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
     )
     writer(output_dataset)
 
 
 @click.command("summary", help="Prints the summary of an underfolder dataset")
-@click.option("-i", "--input", "path", type=Path, required=True, help="Input dataset")
+@click.option(
+    "-i",
+    "--input",
+    "path",
+    type=click.Path(exists=True),
+    required=True,
+    help="Input dataset",
+)
 @click.option(
     "-o",
     "--order_by",
@@ -496,7 +831,13 @@ def operation_groupby(input_folder, key, output_folder):
     default="name",
     help="Sort by column value",
 )
-@click.option("-R", "--reversed", "reversed_", is_flag=True, help="Reverse sorting")
+@click.option(
+    "-R",
+    "--reversed",
+    "reversed_",
+    is_flag=True,
+    help="Reverse sorting",
+)
 @click.option(
     "-k",
     "--max_samples",
@@ -504,7 +845,12 @@ def operation_groupby(input_folder, key, output_folder):
     type=int,
     help="Maximum number of samples to read from the dataset",
 )
-def summary(path, order_by, reversed_, max_samples):
+def summary(
+    path: str,
+    order_by: str,
+    reversed_: str,
+    max_samples: str,
+):
 
     from pipelime.cli.summary import print_summary
     from pipelime.sequences.readers.filesystem import UnderfolderReader
@@ -515,5 +861,336 @@ def summary(path, order_by, reversed_, max_samples):
     )
 
 
-if __name__ == "__main__":
-    operation_filterbyquery()
+@click.command("remap_keys", help="Remap sample keys")
+@click.option(
+    "-i",
+    "--input_folder",
+    required=True,
+    type=click.Path(exists=True),
+    help="Input Underfolder",
+)
+@click.option(
+    "-k",
+    "--keys",
+    required=True,
+    type=str,
+    multiple=True,
+    nargs=2,
+    help="Mapping (source key, remapped key)",
+)
+@click.option(
+    "-R",
+    "--remove",
+    "remove_",
+    is_flag=True,
+    help="TRUE to remove not mapped keys",
+)
+@click.option(
+    "-o",
+    "--output_folder",
+    required=True,
+    type=click.Path(),
+    help="Output Underfolder for positive matches",
+)
+@hardlink_option
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def operation_remap_keys(
+    input_folder: str,
+    keys: Sequence[str],
+    remove_: bool,
+    output_folder: str,
+    hardlink: bool,
+):
+
+    from pipelime.sequences.operations import OperationRemapKeys
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    PiperCommand.instance
+
+    dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
+    template = dataset.get_reader_template()
+
+    # operations
+    keys_map = {k0: k1 for k0, k1 in keys}
+
+    # update root files and extensions map
+    for k0, k1 in keys_map.items():
+        if k0 in template.root_files_keys:
+            template.root_files_keys.append(k1)
+        if k0 in template.extensions_map:
+            template.extensions_map[k1] = template.extensions_map[k0]
+
+    op = OperationRemapKeys(remap=keys_map, remove_missing=remove_)
+    output_dataset = op(dataset)
+
+    copy_mode = (
+        UnderfolderWriterV2.CopyMode.HARD_LINK
+        if hardlink
+        else UnderfolderWriterV2.CopyMode.DEEP_COPY
+    )
+    writer = UnderfolderWriterV2(
+        folder=output_folder,
+        copy_mode=copy_mode,
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
+    )
+    writer(output_dataset)
+
+
+@click.command("remote_add", help="Upload underfolder data to remote")
+@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
+@click.option(
+    "-r",
+    "--remote",
+    required=True,
+    multiple=True,
+    help="For each remote you must provide "
+    "'<scheme>://<netloc>/<base_path>"
+    "[?<init-kw>=<init-value>:<init-kw>=<init-value>...]'"
+    " (repeat for each remote)",
+)
+@click.option(
+    "-k",
+    "--key",
+    required=True,
+    multiple=True,
+    help="Keys to upload (repeat for each key)",
+)
+@click.option(
+    "--copy",
+    "copy_mode",
+    flag_value="copy",
+    help="Always deep copy source data (default)",
+    default=True,
+)
+@click.option(
+    "--symlink", "copy_mode", flag_value="symlink", help="Sym-link source data"
+)
+@click.option(
+    "--hardlink", "copy_mode", flag_value="hardlink", help="Hard-link source data"
+)
+@click.option(
+    "-o", "--output_folder", required=True, type=str, help="Output Underfolder"
+)
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def upload_to_remote(input_folder, remote, key, copy_mode, output_folder):
+    from urllib.parse import unquote_plus, urlparse
+
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+    from pipelime.sequences.samples import SamplesSequence
+    from pipelime.sequences.stages import RemoteParams, StageUploadToRemote
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    def _convert_val(val: str):
+        if val == "True":
+            return True
+        if val == "False":
+            return False
+        try:
+            num = int(val)
+            return num
+        except ValueError:
+            pass
+        try:
+            num = float(val)
+            return num
+        except ValueError:
+            pass
+        return val
+
+    remote_params = [urlparse(rm) for rm in remote]
+    remote_params = [
+        RemoteParams(
+            scheme=rm.scheme,
+            netloc=rm.netloc,
+            base_path=unquote_plus(rm.path)[1:],
+            init_args={
+                kw.split("=", 1)[0]: _convert_val(kw.split("=", 1)[1])
+                for kw in rm.query.split(":")
+                if len(kw) >= 3 and "=" in kw
+            },
+        )
+        for rm in remote_params
+    ]
+
+    dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
+    template = dataset.get_reader_template()
+
+    sseq = SamplesSequence(
+        dataset,
+        stage=StageUploadToRemote(
+            remotes=remote_params,
+            key_ext_map={k: template.extensions_map.get(k) for k in key},
+        ),
+    )
+
+    # save uploaded keys with .remote extension
+    for k in key:
+        template.extensions_map[k] = "remote"
+
+    writer = UnderfolderWriterV2(
+        folder=output_folder,
+        copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK
+        if copy_mode == "hardlink"
+        else (
+            UnderfolderWriterV2.CopyMode.SYM_LINK
+            if copy_mode == "symlink"
+            else UnderfolderWriterV2.CopyMode.DEEP_COPY
+        ),
+        reader_template=template,
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
+    )
+    writer(sseq)
+
+
+@click.command("remote_remove", help="Remove remote from list")
+@click.option("-i", "--input_folder", required=True, type=str, help="Input Underfolder")
+@click.option(
+    "-r",
+    "--remote",
+    required=True,
+    multiple=True,
+    help="For each remote you must provide "
+    "'<scheme>://<netloc>/<base_path>' (repeat for each remote)",
+)
+@click.option(
+    "-k",
+    "--key",
+    required=True,
+    multiple=True,
+    help="Keys to upload (repeat for each key)",
+)
+@click.option(
+    "--copy",
+    "copy_mode",
+    flag_value="copy",
+    help="Always deep copy source data (default)",
+    default=True,
+)
+@click.option(
+    "--symlink", "copy_mode", flag_value="symlink", help="Sym-link source data"
+)
+@click.option(
+    "--hardlink", "copy_mode", flag_value="hardlink", help="Hard-link source data"
+)
+@click.option(
+    "-o", "--output_folder", required=True, type=str, help="Output Underfolder"
+)
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def remove_remote(input_folder, remote, key, copy_mode, output_folder):
+    from urllib.parse import unquote_plus, urlparse
+
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+    from pipelime.sequences.samples import SamplesSequence
+    from pipelime.sequences.stages import RemoteParams, StageRemoveRemote
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+
+    remote_params = [urlparse(rm) for rm in remote]
+    remote_params = [
+        RemoteParams(
+            scheme=rm.scheme,
+            netloc=rm.netloc,
+            base_path=unquote_plus(rm.path)[1:],
+        )
+        for rm in remote_params
+    ]
+
+    dataset = UnderfolderReader(folder=input_folder, lazy_samples=True)
+    sseq = SamplesSequence(
+        dataset,
+        stage=StageRemoveRemote(
+            remotes=remote_params,
+            key_list=key,
+        ),
+    )
+
+    writer = UnderfolderWriterV2(
+        folder=output_folder,
+        copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK
+        if copy_mode == "hardlink"
+        else (
+            UnderfolderWriterV2.CopyMode.SYM_LINK
+            if copy_mode == "symlink"
+            else UnderfolderWriterV2.CopyMode.DEEP_COPY
+        ),
+        reader_template=dataset.get_reader_template(),
+        progress_callback=PiperCommand.instance.generate_progress_callback(),
+    )
+    writer(sseq)
+
+
+@click.command("dump", help="Dumps an underfolder dataset to CSV or Orange Tab file.")
+@click.option(
+    "-i", "--input_folder", type=Path, required=True, help="Input Underfolder"
+)
+@click.option(
+    "-o", "--output_folder", type=Path, required=True, help="Output Underfolder"
+)
+@click.option("-k", "--keys", type=str, multiple=True, help="Filtering keys [multiple]")
+@click.option(
+    "--negated",
+    is_flag=True,
+    help="If present, filtering keys are removed from samples, instead of extracted",
+)
+@click.option(
+    "--start",
+    default=None,
+    type=int,
+    help="First sample index (negative values are allowed)",
+)
+@click.option(
+    "--stop",
+    default=None,
+    type=int,
+    help="Last sample index, excluded (negative values are allowed)",
+)
+@click.option(
+    "--step", default=None, type=int, help="Range step (negative values are allowed)"
+)
+@click.option(
+    "-f",
+    "--format",
+    type=click.Choice(["csv", "orange"]),
+    default="orange",
+    help="Output format [csv|orange]",
+    show_default=True,
+)
+@click.option(
+    "--link_type",
+    type=click.Choice(["copy", "soft", "hard"]),
+    default="hard",
+    help="if soft/hard links should be used whenever possible when writing assets",
+    show_default=True,
+)
+@Piper.command(inputs=["input_folder"], outputs=["output_folder"])
+def dump(
+    input_folder, output_folder, keys, negated, start, stop, step, format, link_type
+):
+    from pipelime.cli.dump import LinkType, dump_data
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+
+    reader = UnderfolderReader(input_folder)
+    if keys:
+        from pipelime.sequences.samples import SamplesSequence
+        from pipelime.sequences.stages import StageKeysFilter
+
+        reader = SamplesSequence(reader, StageKeysFilter(keys, negated))
+
+    file_ext = {"csv": ".csv", "orange": ".tab"}
+
+    output_folder = Path(output_folder)
+    output_folder.mkdir(parents=True, exist_ok=True)
+    with open(
+        output_folder / (Path(input_folder).name + file_ext[format]), "w"
+    ) as outfile:
+        dump_data(
+            samples=reader,
+            output_assets_path=output_folder / "assets",
+            start=start,
+            stop=stop,
+            step=step,
+            format=format,
+            link_type=LinkType[link_type],
+            file=outfile,
+        )
diff --git a/pipelime/cli/underfolder/underfolder.py b/pipelime/cli/underfolder/underfolder.py
index b4b8c25..f124943 100644
--- a/pipelime/cli/underfolder/underfolder.py
+++ b/pipelime/cli/underfolder/underfolder.py
@@ -7,6 +7,7 @@
     operation_groupby,
     operation_mix,
     operation_orderby,
+    operation_remap_keys,
     operation_shuffle,
     operation_split,
     operation_split_by_value,
@@ -14,6 +15,9 @@
     operation_subsample,
     operation_sum,
     summary,
+    upload_to_remote,
+    remove_remote,
+    dump,
 )
 
 
@@ -34,4 +38,8 @@ def underfolder():
 underfolder.add_command(operation_split_by_value)
 underfolder.add_command(operation_groupby)
 underfolder.add_command(operation_mix)
+underfolder.add_command(operation_remap_keys)
 underfolder.add_command(summary)
+underfolder.add_command(upload_to_remote)
+underfolder.add_command(remove_remote)
+underfolder.add_command(dump)
diff --git a/pipelime/converters/__init__.py b/pipelime/converters/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pipelime/converters/base.py b/pipelime/converters/base.py
new file mode 100644
index 0000000..50b6eab
--- /dev/null
+++ b/pipelime/converters/base.py
@@ -0,0 +1,7 @@
+from abc import abstractmethod
+
+
+class UnderfolderConverter:
+    @abstractmethod
+    def convert(self, output_folder: str):
+        raise NotImplementedError
diff --git a/pipelime/converters/subfolders2underfolder.py b/pipelime/converters/subfolders2underfolder.py
new file mode 100644
index 0000000..c17adf6
--- /dev/null
+++ b/pipelime/converters/subfolders2underfolder.py
@@ -0,0 +1,175 @@
+import os
+from typing import Callable, Dict, Optional, Sequence
+from pathlib import Path
+from pipelime.converters.base import UnderfolderConverter
+from pipelime.sequences.readers.base import ReaderTemplate
+from pipelime.sequences.samples import FileSystemSample, SamplesSequence
+from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+import re
+
+
+class Subfolders2Underfolder(UnderfolderConverter):
+    CHAR_TO_REPLACE = ["/", "\\", ":", "*", "?", '"', "<", ">", "|", "-"]
+
+    def __init__(
+        self,
+        folder: str,
+        images_extension: str = "png",
+        use_symlinks: bool = False,
+        num_workers: int = 0,
+        progress_callback: Optional[Callable[[dict], None]] = None,
+    ) -> None:
+        """Converts a subfolder tree structure, containing images, to a single Underfolder.
+        Subfolder structure should be like
+
+        root
+        - subfolder1
+            - subfolder2
+                - subfolder3
+                    - image1.png
+                    - image2.png
+                - image3.png
+            - image4.png
+
+        Category for image2 will be 'subfolder1_subfolder2_subfolder3'. An so on...
+
+        :param folder: root folder
+        :type folder: str
+        :param images_extension: image extension to include in conversion, defaults to "png"
+        :type images_extension: str, optional
+        :param use_symlinks: use symlinks instead of copying files, defaults to False
+        :type use_symlinks: bool, optional
+        :param num_workers: number of workers to use, defaults to 0
+        :type num_workers: int, optional
+        :param progress_callback: callback to report progress, defaults to None
+        :type progress_callback: Optional[Callable[[dict], None]], optional
+        """
+        self._folder = folder
+        self._use_symlinks = use_symlinks
+        self._num_workers = num_workers
+        self._progress_callback = progress_callback
+        self._images_extension = images_extension
+        out = self.remap(self.extract_subfolders_and_files(folder))
+        self._items = out["items"]
+        self._classmap = out["classmap"]
+
+    def extract_subfolders_and_files(self, folder: str) -> Sequence[Dict]:
+        """extract subfolders and files from a folder recursively
+
+        :param folder: root folder
+        :type folder: str
+        :return: list of all subfolders (recursive) with associated files
+        :rtype: Sequence[Dict]
+        """
+        subfolders = []
+        for root, dirs, files in os.walk(folder):
+            for dir in dirs:
+                subfolder = os.path.join(root, dir)
+                subfolder_files = [
+                    f
+                    for f in os.listdir(subfolder)
+                    if os.path.isfile(os.path.join(subfolder, f))
+                ]
+                relative_path = os.path.relpath(subfolder, folder)
+                subfolders.append(
+                    {
+                        "folder": os.path.join(root, dir),
+                        "relative": relative_path,
+                        "files": subfolder_files,
+                    }
+                )
+        return subfolders
+
+    def purge_string(self, s: str, replace_char: str = "_") -> str:
+        """Purge input string of all special characters
+
+        :param s: input string
+        :type s: str
+        :param replace_char: replacement, defaults to "_"
+        :type replace_char: str, optional
+        :return: purged string
+        :rtype: str
+        """
+        s = replace_char.join(s.split())
+        for ctr in self.CHAR_TO_REPLACE:
+            s = s.replace(ctr, replace_char)
+
+        rx = re.compile(r"_{2,}")
+        s = rx.sub("", s)
+        return s
+
+    def remap(self, subfolder: Sequence[Dict]) -> Dict:
+        """remap subfolders to underfolder
+
+        :param subfolder: list of subfolders
+        :type subfolder: Sequence[Dict]
+        :return: list of single items files with metadata
+        :rtype: Sequence[Dict]
+        """
+        items = []
+        classmap = {}
+        for sub_index, sub in enumerate(subfolder):
+            for file in sub["files"]:
+                file_path = os.path.join(sub["folder"], file)
+
+                # ignore hidden files
+                if Path(file_path).name.startswith("."):
+                    continue
+
+                category = self.purge_string(sub["relative"])
+                data = {
+                    "filename": file,
+                    "filepath": file_path,
+                    "category": category,
+                }
+                category_item = {
+                    "name": category,
+                    "class_id": sub_index,
+                    "color": "#ff0000",
+                }
+                classmap[category] = category_item
+                items.append(data)
+
+        return {
+            "items": items,
+            "classmap": {"classes": [v for k, v in classmap.items()]},
+        }
+
+    def convert(self, output_folder: str):
+        """
+
+        :param output_folder: [description]
+        :type output_folder: str
+        """
+
+        output_folder = Path(output_folder)
+        if not output_folder.exists():
+            output_folder.mkdir(parents=True, exist_ok=False)
+
+        samples = []
+        for index, item in enumerate(self._items):
+            sample = FileSystemSample(data_map={}, id=index)
+            sample.filesmap["image"] = item["filepath"]
+            sample["metadata"] = {
+                "category": item["category"],
+                "filename": item["filename"],
+            }
+            sample["classmap"] = self._classmap
+            samples.append(sample)
+
+        writer = UnderfolderWriterV2(
+            folder=output_folder,
+            file_handling=UnderfolderWriterV2.FileHandling.COPY_IF_NOT_CACHED,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+            reader_template=ReaderTemplate(
+                extensions_map={
+                    "image": self._images_extension,
+                    "metadata": "yml",
+                    "classmap": "yml",
+                },
+                root_files_keys=["classmap"],
+            ),
+            num_workers=self._num_workers,
+            progress_callback=self._progress_callback,
+        )
+        writer(SamplesSequence(samples=samples))
diff --git a/pipelime/filesystem/remotes.py b/pipelime/filesystem/remotes.py
new file mode 100644
index 0000000..f92a320
--- /dev/null
+++ b/pipelime/filesystem/remotes.py
@@ -0,0 +1,704 @@
+from abc import ABCMeta, abstractmethod
+from pathlib import Path, WindowsPath
+from typing import Type, Mapping, MutableMapping, Any, Tuple, Union, Optional, BinaryIO
+import hashlib
+from urllib.parse import urlparse, unquote_plus, ParseResult
+
+import os
+import shutil
+
+from loguru import logger
+
+
+class RemoteRegister(ABCMeta):
+    """A factory class managing a single remote instance
+    for any (scheme, netloc) pair.
+    """
+
+    REMOTE_CLASSES: MutableMapping[str, Type["BaseRemote"]] = {}
+    REMOTE_INSTANCES: MutableMapping[Tuple[str, str], "BaseRemote"] = {}
+
+    def __init__(cls, name, bases, dct):
+        cls.REMOTE_CLASSES[cls.scheme()] = cls  # type: ignore
+        super().__init__(name, bases, dct)
+
+    @classmethod
+    def get_instance(cls, scheme: str, netloc: str, **kwargs) -> Optional["BaseRemote"]:
+        """Return a remote instance for this scheme and netloc.
+
+        :param scheme: the protocol name, eg, 's3' or 'file'.
+        :type scheme: str
+        :param netloc: the address of the endpoint.
+        :type netloc: str
+        :param **kwargs: arguments forwarded to the remote instance `__init__()`
+        :return: the remote instance or None if the scheme is unknown.
+        :rtype: Optional[BaseRemote]
+        """
+        remote_instance = cls.REMOTE_INSTANCES.get((scheme, netloc))
+        if remote_instance is None:
+            remote_class = cls.REMOTE_CLASSES.get(scheme)
+            if remote_class is not None:
+                remote_instance = remote_class(netloc, **kwargs)
+                cls.REMOTE_INSTANCES[(scheme, netloc)] = remote_instance
+            else:
+                logger.warning(f"Unknown remote scheme '{scheme}'.")
+        return remote_instance
+
+
+def create_remote(scheme: str, netloc: str, **kwargs) -> Optional["BaseRemote"]:
+    """Return a remote instance for this scheme and netloc.
+
+    :param scheme: the protocol name, eg, 's3' or 'file'.
+    :type scheme: str
+    :param netloc: the address of the endpoint.
+    :type netloc: str
+    :param **kwargs: arguments forwarded to the remote instance's `__init__()`
+    :return: the remote instance or None if the scheme is unknown.
+    :rtype: Optional[BaseRemote]
+    """
+    return RemoteRegister.get_instance(scheme, netloc, **kwargs)
+
+
+def parse_url(url: str) -> Union[Tuple[str, str], Tuple[None, None]]:
+    """Extract base path and target name from the path component of a given url.
+
+    :param url: the url to split.
+    :type url: str
+    :return: the base path and the target name.
+    :rtype: Union[Tuple[str, str], Tuple[None, None]]
+    """
+    parsed_url = urlparse(url)
+    if len(parsed_url.path) > 1:
+        file_full_path = Path(unquote_plus(parsed_url.path)[1:])
+        return str(file_full_path.parent.as_posix()), str(file_full_path.name)
+    else:
+        return None, None
+
+
+def get_remote_and_paths(
+    url: str, remotes_kwargs: Mapping[str, Mapping[str, Any]] = {}
+) -> Tuple[Optional["BaseRemote"], Optional[str], Optional[str]]:
+    """Create a remote and extract base and target path from a given url.
+
+    :param url: the url to split.
+    :type url: str
+    :param remotes_kwargs: keys are the scheme names and values are the arguments
+        that must be forwarded to the remote instance's `__init__()`.
+    :type remotes_kwargs: Mapping[str, Mapping[str, Any]]
+    :return: (remote, base path, target name)
+    :rtype: Tuple[Optional["BaseRemote"], Optional[str], Optional[str]]
+    """
+    parsed_url = urlparse(url)
+    return (
+        create_remote(
+            parsed_url.scheme,
+            parsed_url.netloc,
+            **(remotes_kwargs.get(parsed_url.scheme, {})),
+        ),
+    ) + parse_url(url)
+
+
+class BaseRemote(metaclass=RemoteRegister):  # type: ignore
+    """Base class for any remote."""
+
+    def __init__(self, netloc: str):
+        """Set the network address.
+
+        :param netloc: the network address.
+        :type netloc: str
+        """
+        self._netloc = netloc
+
+    def upload_file(
+        self, local_file: Union[Path, str], target_base_path: str
+    ) -> Optional[str]:
+        """Upload an existing file to remote storage.
+
+        :param local_file: local file path.
+        :type local_file: Union[Path, str]
+        :param target_base_path: the remote base path, eg, the bucket name.
+        :type target_base_path: str
+        :return: a url to get back the file.
+        :rtype: Optional[str]
+        """
+        local_file = Path(local_file)
+        file_size = local_file.stat().st_size
+        with open(local_file, "rb") as file_data:
+            return self.upload_stream(
+                file_data, file_size, target_base_path, "".join(local_file.suffixes)
+            )
+
+    def upload_stream(
+        self,
+        local_stream: BinaryIO,
+        local_stream_size: int,
+        target_base_path: str,
+        target_suffix: str,
+    ) -> Optional[str]:
+        """Upload data from a readable stream.
+
+        :param local_stream: a binary data stream.
+        :type local_stream: BinaryIO
+        :param local_stream_size: the byte size to upload.
+        :type local_stream_size: int
+        :param target_base_path: the remote base path, eg, the bucket name.
+        :type target_base_path: str
+        :param target_suffix: the remote file suffix.
+        :type target_suffix: str
+        :return: a url to get back the file.
+        :rtype: Optional[str]
+        """
+        hash_name = self._compute_hash(
+            local_stream, self._get_hash_fn(target_base_path)
+        )
+        target_name = hash_name + target_suffix
+        if self.target_exists(target_base_path, target_name) or self._upload(
+            local_stream, local_stream_size, target_base_path, target_name
+        ):
+            return self._make_url(f"{target_base_path}/{target_name}")
+        return None
+
+    def download_file(
+        self,
+        local_file: Union[Path, str],
+        source_base_path: str,
+        source_name: str,
+    ) -> bool:
+        """Download and write data to a local file.
+
+        :param local_file: the file to write to.
+        :type local_file: Union[Path, str]
+        :param source_base_path: the remote base path, eg, the bucket name.
+        :type source_base_path: str
+        :param source_name: the remote object name.
+        :type source_name: str
+        :return: True if no error occurred.
+        :rtype: bool
+        """
+        local_file = Path(local_file)
+        source_name_path = Path(source_name)
+        if local_file.suffixes != source_name_path.suffixes:
+            local_file = local_file.with_suffix("".join(source_name_path.suffixes))
+        if local_file.is_file():
+            with open(local_file, "rb") as local_stream:
+                hash_name = self._compute_hash(
+                    local_stream, self._get_hash_fn(source_base_path)
+                )
+                if (
+                    hash_name
+                    == source_name_path.name[: -len("".join(source_name_path.suffixes))]
+                ):
+                    # local file is up-to-date
+                    return True
+
+        local_file.parent.mkdir(parents=True, exist_ok=True)
+        part_file = local_file.with_suffix(local_file.suffix + ".part")
+        offset: int = 0
+        try:
+            offset = part_file.stat().st_size
+        except IOError:
+            pass
+
+        ok = False
+        with open(part_file, "ab") as part_stream:
+            ok = self.download_stream(
+                part_stream, source_base_path, source_name, offset
+            )
+
+        if ok:
+            local_file.unlink(missing_ok=True)
+            part_file.rename(local_file)
+
+        return ok
+
+    def download_stream(
+        self,
+        local_stream: BinaryIO,
+        source_base_path: str,
+        source_name: str,
+        source_offset: int = 0,
+    ) -> bool:
+        """Download and write data to a writable stream.
+
+        :param local_stream: a writable stream.
+        :type local_stream: BinaryIO
+        :param source_base_path: the remote base path, eg, the bucket name.
+        :type source_base_path: str
+        :param source_name: the remote object name.
+        :type source_name: str
+        :param source_offset: where to start reading the remote data, defaults to 0
+        :type source_offset: int, optional
+        :return: True if no error occurred.
+        :rtype: bool
+        """
+        return self._download(
+            local_stream, source_base_path, source_name, source_offset
+        )
+
+    @abstractmethod
+    def target_exists(self, target_base_path: str, target_name: str) -> bool:
+        """Check if a remote object exists.
+
+        :param target_base_path: the remote base path, eg, the bucket name.
+        :type target_base_path: str
+        :param target_name: the remote object name.
+        :type target_name: str
+        :return: True if <target_base_path>/<target_name> exists on this remote.
+        :rtype: bool
+        """
+        ...
+
+    def _compute_hash(self, stream: BinaryIO, hash_fn: Any = None) -> str:
+        """Compute a hash value based on the content of a readable stream.
+
+        :param stream: the stream to hash.
+        :type stream: BinaryIO
+        :param hash_fn: the hash funtion, if None `hashlib.sha256()` will be used,
+            defaults to None
+        :type hash_fn: Any, optional
+        :return: the computed hash
+        :rtype: str
+        """
+        if hash_fn is None:
+            hash_fn = hashlib.sha256()
+        b = bytearray(1024 * 1024)
+        mv = memoryview(b)
+        fpos = stream.tell()
+        for n in iter(lambda: stream.readinto(mv), 0):  # type: ignore
+            hash_fn.update(mv[:n])
+        stream.seek(fpos)
+        return hash_fn.hexdigest()
+
+    def _make_url(self, target_full_path: str) -> str:
+        """Make a complete url for this remote path.
+
+        :param target_full_path: the remote path.
+        :type target_full_path: str
+        :return: the url
+        :rtype: str
+        """
+        return ParseResult(
+            scheme=self.scheme(),
+            netloc=self.netloc,
+            path=Path(target_full_path).as_posix(),
+            params="",
+            query="",
+            fragment="",
+        ).geturl()
+
+    @abstractmethod
+    def _get_hash_fn(self, target_base_path: str) -> Any:
+        """Return the hash function used for the object in the give remote base path.
+
+        :param target_base_path: the remote base path, eg, the bucket name.
+        :type target_base_path: str
+        :return: the hash function, eg, `hashlib.sha256()`.
+        :rtype: Any
+        """
+        ...
+
+    @abstractmethod
+    def _upload(
+        self,
+        local_stream: BinaryIO,
+        local_stream_size: int,
+        target_base_path: str,
+        target_name: str,
+    ) -> bool:
+        """Upload data from a readable stream.
+
+        :param local_stream: a binary data stream.
+        :type local_stream: BinaryIO
+        :param local_stream_size: the byte size to upload.
+        :type local_stream_size: int
+        :param target_base_path: the remote base path, eg, the bucket name.
+        :type target_base_path: str
+        :param target_name: the remote object name.
+        :type target_name: str
+        :return: True if no error occurred.
+        :rtype: bool
+        """
+        ...
+
+    @abstractmethod
+    def _download(
+        self,
+        local_stream: BinaryIO,
+        source_base_path: str,
+        source_name: str,
+        source_offset: int,
+    ) -> bool:
+        """Download and write data to a writable stream.
+
+        :param local_stream: a writable stream.
+        :type local_stream: BinaryIO
+        :param source_base_path: the remote base path, eg, the bucket name.
+        :type source_base_path: str
+        :param source_name: the remote object name.
+        :type source_name: str
+        :param source_offset: where to start reading the remote data, defaults to 0
+        :type source_offset: int, optional
+        :return: True if no error occurred.
+        :rtype: bool
+        """
+        ...
+
+    @classmethod
+    @abstractmethod
+    def scheme(cls) -> str:
+        """Return the scheme name, eg, 's3' or 'file'.
+
+        :return: the scheme name.
+        :rtype: str
+        """
+        pass
+
+    @property
+    def netloc(self) -> str:
+        """Return the remote network address.
+
+        :return: the remote network address.
+        :rtype: str
+        """
+        return self._netloc
+
+    @property
+    @abstractmethod
+    def is_valid(self) -> bool:
+        """Check if this remote instance is valid and usable.
+
+        :return: True if this instance is valid.
+        :rtype: bool
+        """
+        pass
+
+
+class S3Remote(BaseRemote):
+    _HASH_FN_KEY_ = "__HASH_FN__"
+    _DEFAULT_HASH_FN_ = "sha256"
+
+    def __init__(
+        self,
+        endpoint: str,
+        access_key: Optional[str] = None,
+        secret_key: Optional[str] = None,
+        session_token: Optional[str] = None,
+        secure_connection: bool = True,
+        region: Optional[str] = None,
+    ):
+        """S3-compatible remote. Credential can be passed or retrieved from:
+            * env var:
+                ** access key: AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY, MINIO_ACCESS_KEY
+                ** secret key: AWS_SECRET_ACCESS_KEY, AWS_SECRET_KEY, MINIO_SECRET_KEY
+                ** session token: AWS_SESSION_TOKEN
+            * config files:
+                ** ~/.aws/credentials
+                ** ~/[.]mc/config.json
+
+        :param endpoint: the endpoint address
+        :type endpoint: str
+        :param access_key: optional user id, defaults to None
+        :type access_key: Optional[str], optional
+        :param secret_key: optional user password, defaults to None
+        :type secret_key: Optional[str], optional
+        :param session_token: optional session token, defaults to None
+        :type session_token: Optional[str], optional
+        :param secure_connection: should use a secure connection, defaults to True
+        :type secure_connection: bool, optional
+        :param region: optional region, defaults to None
+        :type region: Optional[str], optional
+        """
+        super().__init__(endpoint)
+
+        try:
+            from minio import Minio
+
+            from minio.credentials import (
+                ChainedProvider,
+                EnvAWSProvider,
+                EnvMinioProvider,
+                AWSConfigProvider,
+                MinioClientConfigProvider,
+            )
+
+            self._client = Minio(
+                endpoint=endpoint,
+                access_key=access_key,
+                secret_key=secret_key,
+                session_token=session_token,
+                secure=secure_connection,
+                region=region,
+                credentials=ChainedProvider(
+                    [
+                        # AWS_ACCESS_KEY_ID|AWS_ACCESS_KEY,
+                        # AWS_SECRET_ACCESS_KEY|AWS_SECRET_KEY,
+                        # AWS_SESSION_TOKEN
+                        EnvAWSProvider(),
+                        # MINIO_ACCESS_KEY, MINIO_SECRET_KEY
+                        EnvMinioProvider(),
+                        # ~/.aws/credentials
+                        AWSConfigProvider(),
+                        # ~/[.]mc/config.json
+                        MinioClientConfigProvider(),
+                    ]
+                ),
+            )
+        except ModuleNotFoundError:  # pragma: no cover
+            logger.error("S3 remote needs `minio` python package.")
+            self._client = None
+
+    @property
+    def client(self):
+        return self._client
+
+    def _maybe_create_bucket(self, target_base_path: str):
+        if not self._client.bucket_exists(target_base_path):  # type: ignore
+            logger.info(
+                f"Creating bucket '{target_base_path}' on S3 remote {self.netloc}."
+            )
+            self._client.make_bucket(target_base_path)  # type: ignore
+
+    def _get_hash_fn(self, target_base_path: str) -> Any:
+        if self.is_valid:
+            try:
+                self._maybe_create_bucket(target_base_path)
+                tags = self._client.get_bucket_tags(target_base_path)  # type: ignore
+                if tags is None:
+                    from minio.commonconfig import Tags
+
+                    tags = Tags()
+                hash_fn_name = tags.get(self._HASH_FN_KEY_)
+
+                # try-get
+                if isinstance(hash_fn_name, str) and len(hash_fn_name) > 0:
+                    try:
+                        hash_fn = getattr(hashlib, hash_fn_name)
+                        return hash_fn()
+                    except AttributeError:
+                        pass
+
+                tags[self._HASH_FN_KEY_] = self._DEFAULT_HASH_FN_
+                self._client.set_bucket_tags(target_base_path, tags)  # type: ignore
+
+                hash_fn = getattr(hashlib, self._DEFAULT_HASH_FN_)
+                return hash_fn()
+            except Exception as exc:
+                logger.debug(str(exc))
+        return None
+
+    def target_exists(self, target_base_path: str, target_name: str) -> bool:
+        if self.is_valid:
+            try:
+                objlist = self._client.list_objects(  # type: ignore
+                    target_base_path, prefix=target_name
+                )
+                _ = next(objlist)
+                return True
+            except StopIteration:
+                return False
+            except Exception as exc:
+                logger.debug(str(exc))
+        return False
+
+    def _upload(
+        self,
+        local_stream: BinaryIO,
+        local_stream_size: int,
+        target_base_path: str,
+        target_name: str,
+    ) -> bool:
+        if self.is_valid:
+            try:
+                self._maybe_create_bucket(target_base_path)
+                self._client.put_object(  # type: ignore
+                    bucket_name=target_base_path,
+                    object_name=target_name,
+                    data=local_stream,
+                    length=local_stream_size,
+                )
+                return True
+            except Exception as exc:
+                logger.debug(str(exc))
+                return False
+
+        return False
+
+    def _download(
+        self,
+        local_stream: BinaryIO,
+        source_base_path: str,
+        source_name: str,
+        source_offset: int,
+    ) -> bool:
+        if self.is_valid:
+            if not self._client.bucket_exists(source_base_path):  # type: ignore
+                logger.debug(
+                    f"Bucket '{source_base_path}' does not exist "
+                    f"on S3 remote '{self.netloc}'."
+                )
+                return False
+
+            response = None
+            ok = False
+            try:
+                response = self._client.get_object(  # type: ignore
+                    bucket_name=source_base_path,
+                    object_name=source_name,
+                    offset=source_offset,
+                )
+                for data in response.stream(amt=1024 * 1024):
+                    local_stream.write(data)
+                ok = True
+            except Exception as exc:
+                logger.debug(str(exc))
+                return False
+            finally:
+                if response:
+                    response.close()
+                    response.release_conn()
+                return ok
+
+        return False
+
+    @classmethod
+    def scheme(cls) -> str:
+        return "s3"
+
+    @property
+    def is_valid(self) -> bool:
+        return self._client is not None
+
+
+class FileRemote(BaseRemote):
+    _PL_FOLDER_ = ".pl"
+    _TAGS_FILE_ = "tags.json"
+    _HASH_FN_KEY_ = "__HASH_FN__"
+    _DEFAULT_HASH_FN_ = "sha256"
+
+    def __init__(self, netloc: str):
+        """Filesystem-based remote.
+
+        :param netloc: the ip address, can be empty.
+        :type netloc: str
+        """
+        if netloc == "localhost" or netloc == "127.0.0.1":
+            netloc = ""
+        super().__init__(netloc)
+
+    def _maybe_create_root(self, target_base_path: Path):
+        if not target_base_path.exists():
+            logger.info(f"Creating folder tree '{target_base_path}'.")
+            target_base_path.mkdir(parents=True, exist_ok=True)
+        pldir = target_base_path / self._PL_FOLDER_
+        if not pldir.is_dir():
+            pldir.mkdir(parents=True, exist_ok=True)
+
+    def _get_hash_fn(self, target_base_path: str) -> Any:
+        if self.is_valid:
+            try:
+                import json
+
+                target_root = self._make_file_path(target_base_path, "")
+                self._maybe_create_root(target_root)
+
+                tags = {}
+                try:
+                    with open(
+                        target_root / self._PL_FOLDER_ / self._TAGS_FILE_, "r"
+                    ) as jtags:
+                        tags = json.load(jtags)
+                except Exception:
+                    pass
+
+                hash_fn_name = tags.get(self._HASH_FN_KEY_)
+
+                # try-get
+                if isinstance(hash_fn_name, str) and len(hash_fn_name) > 0:
+                    try:
+                        hash_fn = getattr(hashlib, hash_fn_name)
+                        return hash_fn()
+                    except AttributeError:
+                        pass
+
+                tags[self._HASH_FN_KEY_] = self._DEFAULT_HASH_FN_
+                with open(
+                    target_root / self._PL_FOLDER_ / self._TAGS_FILE_, "w"
+                ) as jtags:
+                    json.dump(tags, jtags)
+
+                hash_fn = getattr(hashlib, self._DEFAULT_HASH_FN_)
+                return hash_fn()
+            except Exception as exc:
+                logger.debug(str(exc))
+        return None
+
+    def target_exists(self, target_base_path: str, target_name: str) -> bool:
+        if self.is_valid:
+            return self._make_file_path(target_base_path, target_name).exists()
+        return False
+
+    def _upload(
+        self,
+        local_stream: BinaryIO,
+        local_stream_size: int,
+        target_base_path: str,
+        target_name: str,
+    ) -> bool:
+        if self.is_valid:
+            try:
+                target_full_path = self._make_file_path(target_base_path, target_name)
+                self._maybe_create_root(target_full_path.parent)
+
+                with target_full_path.open("wb") as target:
+                    shutil.copyfileobj(local_stream, target)
+
+                return True
+            except Exception as exc:
+                logger.debug(str(exc))
+                return False
+
+        return False
+
+    def _download(
+        self,
+        local_stream: BinaryIO,
+        source_base_path: str,
+        source_name: str,
+        source_offset: int,
+    ) -> bool:
+        if self.is_valid:
+            try:
+                source_full_path = self._make_file_path(source_base_path, source_name)
+                if not source_full_path.is_file():
+                    logger.debug(f"File '{source_full_path}' does not exist.")
+                    return False
+
+                with source_full_path.open("rb") as source:
+                    source.seek(source_offset)
+                    shutil.copyfileobj(source, local_stream)
+
+                return True
+            except Exception as exc:
+                logger.debug(str(exc))
+                return False
+
+        return False
+
+    def _make_file_path(self, file_path: str, file_name: str) -> Path:
+        full_path = Path(file_path) / Path(file_name)
+        if self.netloc:
+            return Path(
+                "{0}{0}{1}{0}{2}".format(os.path.sep, self.netloc, str(full_path))
+            )
+        elif isinstance(full_path, WindowsPath):
+            return full_path
+        else:
+            return Path(os.path.sep + str(full_path))
+
+    @classmethod
+    def scheme(cls) -> str:
+        return "file"
+
+    @property
+    def is_valid(self) -> bool:
+        return True
diff --git a/pipelime/filesystem/toolkit.py b/pipelime/filesystem/toolkit.py
index 8483eee..22ed40c 100644
--- a/pipelime/filesystem/toolkit.py
+++ b/pipelime/filesystem/toolkit.py
@@ -4,19 +4,39 @@
 import warnings
 from collections import defaultdict
 from pathlib import Path
-from typing import Union
+from typing import Any, Union, Iterable, Sequence, BinaryIO, TextIO, Tuple, List
 
 import imageio
 import numpy as np
 import yaml
+import toml
+from io import BytesIO, TextIOWrapper
+
+from loguru import logger
 
 from pipelime.tools.bytes import DataCoding
+import pipelime.filesystem.remotes as plr
 
 
 class FSToolkit(object):
 
     # Default imageio options for each image format
-    OPTIONS = {"png": {"compress_level": 4}}
+    IMG_SAVE_OPTIONS = {"png": {"compress_level": 4}}
+
+    YAML_EXT = ("yaml", "yml")
+    JSON_EXT = ("json",)
+    TOML_EXT = ("toml", "tml")
+    METADATA_EXT = YAML_EXT + JSON_EXT + TOML_EXT
+
+    NUMPY_TXT_EXT = ("txt", "data")
+    NUMPY_NATIVE_EXT = ("npy", "npz")
+    NUMPY_EXT = NUMPY_TXT_EXT + NUMPY_NATIVE_EXT
+
+    PICKLE_EXT = ("pkl", "pickle")
+
+    BINARY_EXT = ("bin",)
+
+    REMOTE_EXT = ("plr", "rmt", "remote")
 
     # Declare TREE structure
     @classmethod
@@ -24,10 +44,10 @@ def tree(cls):
         return defaultdict(cls.tree)
 
     @classmethod
-    def tree_from_underscore_notation_files(cls, folder: str):
+    def tree_from_underscore_notation_files(cls, folder: Union[str, Path]):
         """Walk through files in folder generating a tree based on Underscore notation.
-        Leafs of abovementioned tree are string representing filenames, inner nodes represent
-        keys hierarchy.
+        Leafs of abovementioned tree are string representing filenames, inner nodes
+        represent keys hierarchy.
 
         :param folder: target folder
         :type folder: str
@@ -67,21 +87,24 @@ def tree_from_underscore_notation_files(cls, folder: str):
     @classmethod
     def get_file_extension(cls, filename, with_dot=False):
         ext = Path(filename).suffix.lower()
-        if not with_dot and len(ext) > 0:
-            ext = ext[1:]
-        return ext
+        return ext if with_dot else ext.lstrip(".")
 
     @classmethod
     def is_metadata_file(cls, filename: str) -> bool:
         ext = cls.get_file_extension(filename)
-        return ext in ["yml", "json", "toml", "tml"]
+        return ext in cls.METADATA_EXT
 
     @classmethod
-    def is_image_file(cls, filename: str) -> bool:
-        return imghdr.what(filename) is not None
+    def is_remote_file(cls, filename: str) -> bool:
+        ext = cls.get_file_extension(filename)
+        return ext in cls.REMOTE_EXT
+
+    @classmethod
+    def is_image_file(cls, filename: Union[str, BinaryIO]) -> bool:
+        return imghdr.what(filename) is not None  # type: ignore
 
     @classmethod
-    def _numpy_load_txt(cls, filename: str) -> np.ndarray:
+    def _numpy_load_txt(cls, filename: Union[str, TextIO, BinaryIO]) -> np.ndarray:
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", "loadtxt")
             data = np.loadtxt(filename)
@@ -90,13 +113,13 @@ def _numpy_load_txt(cls, filename: str) -> np.ndarray:
     @classmethod
     def is_numpy_array_file(cls, filename: str) -> bool:
         ext = cls.get_file_extension(filename)
-        if ext in ["txt", "data"]:
+        if ext in cls.NUMPY_TXT_EXT:
             try:
                 cls._numpy_load_txt(filename)
                 return True
             except Exception:
                 return False
-        if ext in ["npy", "npz"]:
+        elif ext in cls.NUMPY_NATIVE_EXT:
             try:
                 np.load(filename)
                 return True
@@ -105,12 +128,128 @@ def is_numpy_array_file(cls, filename: str) -> bool:
         return False
 
     @classmethod
-    def is_picke_file(cls, filename: str):
+    def is_pickle_file(cls, filename: str):
+        ext = cls.get_file_extension(filename)
+        return ext in cls.PICKLE_EXT
+
+    @classmethod
+    def is_binary_file(cls, filename: str):
         ext = cls.get_file_extension(filename)
-        return ext == "pkl"
+        return ext in cls.BINARY_EXT
+
+    @classmethod
+    def _download_from_remote_list(
+        cls, remote_list: Union[Sequence[str], TextIO]
+    ) -> Tuple[str, BytesIO]:
+        extension = ""
+        data_stream = None
+        for line in remote_list:  # download from first available remote
+            line = line.strip()
+            remote, rm_base, rm_name = plr.get_remote_and_paths(line)
+            if remote and rm_base and rm_name:
+                extension = cls.get_file_extension(rm_name)
+                data_stream = BytesIO()
+                if not remote.download_stream(data_stream, rm_base, rm_name):
+                    data_stream = None
+            if data_stream is None:
+                logger.debug(f"unknown or unreachable remote: {line}")
+            else:
+                data_stream.seek(0)
+                break
+        else:  # no remote loaded
+            raise Exception("remote loading error")
+        return extension, data_stream
+
+    @classmethod
+    def _download_from_remote(cls, filename: str) -> Tuple[str, BytesIO]:
+        with open(filename, "r") as fd:
+            return cls._download_from_remote_list(fd)
+
+    @classmethod
+    def load_remote_list(cls, filename: str) -> List[str]:
+        url_list = []
+        with open(filename, "r") as fd:
+            for line in fd:
+                url_list.append(line.strip())
+        return url_list
+
+    @classmethod
+    def is_remote_list(cls, data: Any) -> bool:
+        from urllib.parse import urlparse
+
+        if not isinstance(data, Sequence):
+            return False
+        for d in data:
+            if not isinstance(d, str):
+                return False
+            try:
+                parsed_url = urlparse(d)
+                if not parsed_url.scheme:
+                    return False
+            except ValueError:
+                return False
+        return True
+
+    @classmethod
+    def load_data_from_stream(
+        cls, data_stream: BinaryIO, extension: str
+    ) -> Union[None, np.ndarray, dict, bytes]:
+        switches = (
+            (
+                lambda: extension in cls.YAML_EXT,
+                lambda: yaml.safe_load(TextIOWrapper(data_stream)),
+            ),
+            (
+                lambda: extension in cls.JSON_EXT,
+                lambda: json.load(TextIOWrapper(data_stream)),
+            ),
+            (
+                lambda: extension in cls.TOML_EXT,
+                lambda: dict(toml.load(TextIOWrapper(data_stream))),
+            ),
+            (
+                lambda: extension in cls.PICKLE_EXT,
+                lambda: pickle.load(data_stream),
+            ),
+            (
+                lambda: extension in cls.BINARY_EXT,
+                lambda: data_stream.read(),
+            ),
+            (
+                lambda: extension in cls.NUMPY_TXT_EXT,
+                lambda: np.atleast_2d(cls._numpy_load_txt(data_stream)),
+            ),
+            (
+                lambda: extension in cls.NUMPY_NATIVE_EXT,
+                lambda: np.atleast_2d(np.load(data_stream)),
+            ),
+            # image must be tested last, since it looks at the binary content
+            (
+                lambda: cls.is_image_file(data_stream),
+                lambda: np.array(imageio.imread(data_stream)),
+            ),
+        )
+
+        for cond, fn in switches:
+            if cond():
+                return fn()
+        return None
 
     @classmethod
-    def load_data(cls, filename: str) -> Union[None, np.ndarray, dict]:
+    def load_remote_data(
+        cls, remote_list: Sequence[str]
+    ) -> Union[None, np.ndarray, dict, bytes]:
+        try:
+            extension, data_stream = cls._download_from_remote_list(remote_list)
+            data = cls.load_data_from_stream(data_stream, extension)
+            if data is not None:
+                return data
+        except Exception as e:
+            raise Exception(f"Loading data error: {e}") from e
+        raise NotImplementedError(f"Unknown data extension: {extension}")
+
+    @classmethod
+    def load_data(cls, filename: str) -> Union[None, np.ndarray, dict, bytes]:
         """Load data from file based on its extension
 
         :param filename: target filename
@@ -118,49 +257,114 @@ def load_data(cls, filename: str) -> Union[None, np.ndarray, dict]:
         :return: Loaded data as array or dict. May return NONE
         :rtype: Union[None, np.ndarray, dict]
         """
+        data_stream = None
+        try:
+            extension = cls.get_file_extension(filename)
+            if extension in cls.REMOTE_EXT:
+                extension, data_stream = cls._download_from_remote(filename)
 
-        extension = cls.get_file_extension(filename)
-        data = None
-
-        if cls.is_image_file(filename):
-            data = np.array(imageio.imread(filename))
+            if data_stream is None:
+                data_stream = open(filename, "rb")
 
-        elif cls.is_numpy_array_file(filename):
-            if extension in ["txt"]:
-                data = cls._numpy_load_txt(filename)
-            elif extension in ["npy", "npz"]:
-                data = np.load(filename)
+            data = cls.load_data_from_stream(data_stream, extension)
             if data is not None:
-                data = np.atleast_2d(data)
-
-        elif cls.is_metadata_file(filename):
-            if extension in ["yml"]:
-                data = yaml.safe_load(open(filename, "r"))
-            elif extension in ["json"]:
-                data = json.load(open(filename))
-        elif cls.is_picke_file(filename):
-            data = pickle.load(open(filename, "rb"))
-        else:
-            raise NotImplementedError(f"Unknown file extension: {filename}")
-        return data
+                return data
+        except Exception as e:
+            raise Exception(f"Loading data error: {e}") from e
+        finally:
+            if data_stream is not None:
+                data_stream.close()
+
+        raise NotImplementedError(f"Unknown file extension: {filename}")
 
     @classmethod
-    def store_data(cls, filename: str, data: any):
-
-        extension = cls.get_file_extension(filename)
-        if DataCoding.is_image_extension(extension):
-            options = cls.OPTIONS.get(extension, {})
-            imageio.imwrite(filename, data, **options)
-        elif DataCoding.is_text_extension(extension):
-            np.savetxt(filename, data)
-        elif DataCoding.is_numpy_extension(extension):
-            np.save(filename, data)
-        elif DataCoding.is_metadata_extension(extension):
-            if extension in ["yml"]:
-                yaml.safe_dump(data, open(filename, "w"))
-            elif extension in ["json"]:
-                json.dump(data, open(filename, "w"))
-        elif DataCoding.is_pickle_extension(extension):
-            pickle.dump(data, open(filename, "wb"))
+    def store_data_to_stream(cls, data_stream: BinaryIO, extension: str, data: Any):
+        try:
+
+            def _write_remote_file(filestream, data):
+                if isinstance(data, Iterable):
+                    data = "\n".join(data)
+                filestream.write(data)
+
+            extension = extension.lstrip(".")
+
+            switches = (
+                (
+                    lambda: DataCoding.is_image_extension(extension),
+                    lambda: imageio.imwrite(
+                        data_stream,
+                        data,
+                        format=f".{extension}",
+                        **(cls.IMG_SAVE_OPTIONS.get(extension, {})),
+                    ),
+                ),
+                (
+                    lambda: DataCoding.is_text_extension(extension),
+                    lambda: np.savetxt(data_stream, data),
+                ),
+                (
+                    lambda: DataCoding.is_numpy_extension(extension),
+                    lambda: np.save(data_stream, data),
+                ),
+                (
+                    lambda: extension in cls.YAML_EXT,
+                    lambda: yaml.safe_dump(data, TextIOWrapper(data_stream)),
+                ),
+                (
+                    lambda: extension in cls.JSON_EXT,
+                    lambda: json.dump(data, TextIOWrapper(data_stream)),
+                ),
+                (
+                    lambda: extension in cls.TOML_EXT,
+                    lambda: toml.dump(data, TextIOWrapper(data_stream)),
+                ),
+                (
+                    lambda: DataCoding.is_pickle_extension(extension),
+                    lambda: pickle.dump(data, data_stream),
+                ),
+                (
+                    lambda: extension in cls.BINARY_EXT,
+                    lambda: data_stream.write(data),
+                ),
+                (
+                    lambda: extension in cls.REMOTE_EXT,
+                    lambda: _write_remote_file(TextIOWrapper(data_stream), data),
+                ),
+            )
+
+            for cond, fn in switches:
+                if cond():
+                    fn()
+                    return
+        except Exception as e:
+            raise Exception(f"Loading data error: {e}") from e
+
+        # extension not found in switches
+        raise NotImplementedError(f"Unknown file extension: {extension}")
+
+    @classmethod
+    def store_data(cls, filename: str, data: Any):
+        data_stream = None
+        try:
+            extension = cls.get_file_extension(filename)
+            data_stream = open(filename, "wb")
+        except Exception as e:
+            raise Exception(f"Loading data error: {e}") from e
         else:
-            raise NotImplementedError(f"Unknown file extension: {filename}")
+            return cls.store_data_to_stream(data_stream, extension, data)
+        finally:
+            if data_stream is not None:
+                data_stream.close()
+
+    @classmethod
+    def start_file(cls, filename: str):
+        import subprocess
+        import os
+        import platform
+
+        if platform.system() == "Darwin":  # macOS
+            subprocess.call(("open", filename))
+        elif platform.system() == "Windows":  # Windows
+            os.startfile(filename)
+        else:  # linux variants #TODO: verify!
+            subprocess.call(("xdg-open", filename))
diff --git a/pipelime/h5/toolkit.py b/pipelime/h5/toolkit.py
index 884b918..3228a79 100644
--- a/pipelime/h5/toolkit.py
+++ b/pipelime/h5/toolkit.py
@@ -162,7 +162,7 @@ class H5ToolKit:
     ENCODING_STRING = "_encoding"
     ENCODING_BINARY = "pkl"
     OPTIONS = {"png": {"compress_level": 4}}
-    ALLOWED_ENCODINGS = DataCoding.IMAGE_CODECS + [ENCODING_BINARY]
+    ALLOWED_ENCODINGS = DataCoding.IMAGE_CODECS + (ENCODING_BINARY,)
 
     @classmethod
     def get_encoding(cls, dataset: h5py.Dataset) -> Optional[str]:
diff --git a/pipelime/pipes/__init__.py b/pipelime/pipes/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pipelime/pipes/communication.py b/pipelime/pipes/communication.py
new file mode 100644
index 0000000..34e2358
--- /dev/null
+++ b/pipelime/pipes/communication.py
@@ -0,0 +1,195 @@
+import json
+from abc import ABC, abstractmethod
+import os
+from typing import Callable
+
+import redis
+from choixe.bulletins import Bulletin, BulletinBoard
+from loguru import logger
+
+
+class PiperCommunicationChannel(ABC):
+    def __init__(self, token: str) -> None:
+        """Generic Piper communication channel.
+
+        Args:
+            token (str): The token to use for the communication.
+        """
+        self._token = token
+
+    @property
+    def token(self) -> str:
+        return self._token
+
+    @abstractmethod
+    def send(self, id: str, payload: any) -> bool:
+        """Send a value to the communication channel.
+
+        Args:
+            id (str): sender id
+            payload (any): payload to send
+
+        Returns:
+            bool: True if send was successful
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def register_callback(self, callback: Callable[[dict], None]) -> bool:
+        """Register a callback to call every time a message is received"""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def listen(self) -> None:
+        """Wait for new messages"""
+        raise NotImplementedError()
+
+
+class PiperCommunicationChannelBulletinBoard(PiperCommunicationChannel):
+    def __init__(self, token: str) -> None:
+        """Create a new communication channel based on a bulletin board (choixe)
+        It uses a MQTT client to send messages to the bulletin board which is a MQTT
+        subscriber. This channel needs an active MQTT broker to work. If no broker is
+        available, the channel will not work and an error message will be logged.
+
+        Args:
+            token (str): token to use for the communication
+        """
+
+        super().__init__(token)
+
+        self._client = None
+
+        try:
+            self._client = BulletinBoard(session_id=token)
+        except Exception as e:
+            logger.error(f"{self.__class__.__name__} No Bulletin server!|{e}")
+
+    @property
+    def valid(self) -> bool:
+        return self._client is not None
+
+    def send(self, id: str, payload: any) -> bool:
+        """Send a value to the communication channel.
+
+        Args:
+            id (str): sender id
+            payload (any): payload to send
+
+        Returns:
+            bool: True if the value was sent, False otherwise
+        """
+
+        if self.valid:
+            self._client.hang(
+                Bulletin(
+                    metadata={
+                        "id": id,
+                        "token": self._token,
+                        "payload": payload,
+                    }
+                )
+            )
+            return True
+        return False
+
+    def register_callback(self, callback: Callable[[dict], None]) -> bool:
+        def bulletin_helper(bulletin: Bulletin) -> None:
+            callback(bulletin.metadata)
+
+        if self.valid:
+            self._client.register_callback(bulletin_helper)
+            return True
+        return False
+
+    def listen(self) -> None:
+        self._client.wait_for_bulletins()
+
+
+class PiperCommunicationChannelRedis(PiperCommunicationChannel):
+    """`PiperCommunicationChannel` implementation for Redis backend.
+
+    It uses a redis client to send/receive messages using redis pub/sub implementation.
+    This channel needs an active redis instance. If no redis server is
+    available, the channel will not work and an error message will be logged.
+    """
+
+    def __init__(self, token: str) -> None:
+        """Constructor for `PiperCommunicationChannelRedis`
+
+        Args:
+            token (str): The token to use for the communication.
+        """
+        super().__init__(token)
+
+        self._db = None
+
+        try:
+            self._db = redis.Redis()
+            self._pubsub = self._db.pubsub()
+        except Exception as e:
+            logger.error(f"{self.__class__.__name__} No Redis server!|{e}")
+
+    @property
+    def valid(self) -> bool:
+        return self._db is not None
+
+    def send(self, id: str, payload: any) -> bool:
+        if self.valid:
+            data = {"id": id, "token": self._token, "payload": payload}
+            msg = json.dumps(data).encode("utf-8")
+            self._db.publish(self._token, msg)
+            return True
+        return False
+
+    def listen(self) -> None:
+        for _ in self._pubsub.listen():
+            pass
+
+    def register_callback(self, callback: Callable[[dict], None]) -> bool:
+
+        # Callback helper that converts bytes to dictionary before call
+        def redis_helper(msg: dict):
+            json_string = msg["data"].decode("utf-8")
+            data = json.loads(json_string)
+            callback(data)
+
+        if self.valid:
+            self._pubsub.subscribe(**{self._token: redis_helper})
+            return True
+        return False
+
+
+class PiperCommunicationChannelFactory:
+    """Factory for `PiperCommunicationChannel` objects"""
+
+    CHANNEL_TYPE_VARNAME = "PIPELINE_PIPER_CHANNEL_TYPE"
+    """Name of the environment variable with the channel type"""
+
+    BULLETIN_BOARD = "BULLETIN"
+    """Channel type env value for Choixe BulletinBoard"""
+    REDIS = "REDIS"
+    """Channel type env value for Redis"""
+
+    _default_channel_cls = PiperCommunicationChannelBulletinBoard
+
+    _cls_map = {
+        BULLETIN_BOARD: PiperCommunicationChannelBulletinBoard,
+        REDIS: PiperCommunicationChannelRedis,
+    }
+
+    @classmethod
+    def create_channel(cls, token: str) -> PiperCommunicationChannel:
+        """Instantiates a `PiperCommunicationChannel` with a given token.
+        The type of the returned channel can be controlled by setting an environment
+        variable.
+
+        Args:
+            token (str): The token used for communication
+
+        Returns:
+            PiperCommunicationChannel: The instanced communication channel
+        """
+        channel_type = os.getenv(cls.CHANNEL_TYPE_VARNAME)
+        channel_cls = cls._cls_map.get(channel_type, cls._default_channel_cls)
+        return channel_cls(token)
diff --git a/pipelime/pipes/drawing/__init__.py b/pipelime/pipes/drawing/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pipelime/pipes/drawing/assets/NodeUnderfolder.png b/pipelime/pipes/drawing/assets/NodeUnderfolder.png
new file mode 100644
index 0000000..d876d3e
Binary files /dev/null and b/pipelime/pipes/drawing/assets/NodeUnderfolder.png differ
diff --git a/pipelime/pipes/drawing/base.py b/pipelime/pipes/drawing/base.py
new file mode 100644
index 0000000..d70c326
--- /dev/null
+++ b/pipelime/pipes/drawing/base.py
@@ -0,0 +1,31 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Sequence
+from pathlib import Path
+
+import numpy as np
+
+from pipelime.pipes.graph import DAGNodesGraph
+
+
+class NodesGraphDrawer(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def exportable_formats(self) -> Sequence[str]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def draw(self, graph: DAGNodesGraph) -> np.ndarray:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def representation(self, graph: DAGNodesGraph) -> str:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def export(self, graph: DAGNodesGraph, filename: str, format: Optional[str] = None):
+        if format is None:
+            format = Path(filename).suffix[1:]
+        if format not in self.exportable_formats():
+            raise ValueError(f"Format {format} not supported")
diff --git a/pipelime/pipes/drawing/factory.py b/pipelime/pipes/drawing/factory.py
new file mode 100644
index 0000000..e217e12
--- /dev/null
+++ b/pipelime/pipes/drawing/factory.py
@@ -0,0 +1,22 @@
+from typing import Sequence
+from pipelime.pipes.drawing.graphviz import GraphvizNodesGraphDrawer
+from pipelime.pipes.drawing.mermaid import MermaidNodesGraphDrawer
+from pipelime.pipes.drawing.base import NodesGraphDrawer
+
+
+class NodesGraphDrawerFactory:
+
+    FACTORY_MAP = {
+        "graphviz": GraphvizNodesGraphDrawer,
+        "mermaid": MermaidNodesGraphDrawer,
+    }
+
+    @classmethod
+    def create(cls, backend: str) -> NodesGraphDrawer:
+        if backend not in cls.FACTORY_MAP:
+            raise ValueError(f"Backend {backend} not supported")
+        return cls.FACTORY_MAP[backend]()
+
+    @classmethod
+    def available_backends(cls) -> Sequence[str]:
+        return list(cls.FACTORY_MAP.keys())
diff --git a/pipelime/pipes/drawing/graphviz.py b/pipelime/pipes/drawing/graphviz.py
new file mode 100644
index 0000000..ae7900e
--- /dev/null
+++ b/pipelime/pipes/drawing/graphviz.py
@@ -0,0 +1,349 @@
+from typing import Optional, Sequence
+import pygraphviz as pgv
+import numpy as np
+from pipelime.pipes.drawing.base import NodesGraphDrawer
+from pipelime.pipes.graph import (
+    GraphNode,
+    GraphNodeData,
+    GraphNodeOperation,
+    DAGNodesGraph,
+)
+from tempfile import NamedTemporaryFile
+from pathlib import Path
+
+
+class GraphvizNodesGraphDrawer(NodesGraphDrawer):
+    def _is_node_file(self, node: GraphNodeData) -> bool:
+        """Returns True if the node path is (or could be) a file
+
+        Args:
+            node (GraphNodeData):  input node
+
+        Returns:
+            bool:  True if the node path is (or could be) a file
+        """
+        return len(Path(node.path).suffix) > 0
+
+    def _operation_shape(self, node: GraphNodeOperation) -> str:
+        """Returns the shape string of an operation node
+
+        Args:
+            node (GraphNodeOperation): input node
+
+        Returns:
+            str: shape of the node
+        """
+        return "box3d"
+
+    def _data_shape(self, node: GraphNodeData) -> str:
+        """Returns the shape string of a data node
+
+        Args:
+            node (GraphNodeData): input node
+
+        Returns:
+            str: shape of the node
+        """
+        return "note" if self._is_node_file(node) else "cylinder"
+
+    def _operation_styles(self, node: GraphNodeOperation) -> dict:
+        """Returns the style attributes for a generic Operation node
+
+        Args:
+            node (GraphNodeOperation): input node
+
+        Returns:
+            dict: style attributes for the node
+        """
+        return {
+            "style": "filled",
+            "fillcolor": "#03A9F4",
+            "fontcolor": "#fafafa",
+        }
+
+    def _data_styles(self, node: GraphNodeData) -> dict:
+        """Returns the style attributes for a generic Data node
+
+        Args:
+            node (GraphNodeData): input node
+
+        Returns:
+            dict: style attributes for the node
+        """
+        if self._is_node_file(node):
+            return {
+                "style": "filled",
+                "fillcolor": "#fafafa",
+                "fontcolor": "#222222",
+            }
+        else:
+            return {
+                "style": "filled",
+                "fillcolor": "#009688",
+                "fontcolor": "#fafafa",
+            }
+
+    def _operation_attributes(self, node: GraphNodeOperation) -> dict:
+        """Returns the whole attributes kwargs for a generic Operation node
+
+        Args:
+            node (GraphNodeOperation): input node
+
+        Returns:
+            dict: attributes for the node
+        """
+        return {
+            **(self._operation_styles(node)),
+            "shape": self._operation_shape(node),
+            "label": self._operation_node_content(node),
+            # Easter egg
+            "URL": "https://c.tenor.com/2jvzfQQxDFcAAAAC/think-smart.gif",
+        }
+
+    def _data_attributes(self, node: GraphNodeData) -> dict:
+        """Returns the whole attributes kwargs for a generic Data node
+
+        Args:
+            node (GraphNodeData): input node
+
+        Returns:
+            dict: attributes for the node
+        """
+        return {
+            **(self._data_styles(node)),
+            "shape": self._data_shape(node),
+            "label": self._data_node_content(node),
+        }
+
+    def _operation_node_content(self, node: GraphNodeOperation) -> str:
+        """Returns the textual representation of an Operation node in Graphviz TABLE
+        format
+
+        Args:
+            node (GraphNodeOperation): input node
+
+        Returns:
+            str: textual representation of the node
+        """
+        content = '<<TABLE BORDER="0" CELLBORDER="0" CELLPADDING="20">'
+        content += "<TR>"
+        content += f"<TD>{node.node_model.command}</TD>"
+        content += "</TR>"
+        content += "</TABLE>>"
+        return content
+
+    def _data_node_content(self, node: GraphNodeData) -> str:
+        """Returns the textual representation of a Data node in Graphviz TABLE format
+
+        Args:
+            node (GraphNodeData): input node
+
+        Returns:
+            str: textual representation of the node
+        """
+        content = '<<TABLE BORDER="0" CELLBORDER="0" CELLPADDING="20">'
+        content += "<TR>"
+        content += f"<TD>{node.path}</TD>"
+        content += "</TR>"
+        content += "</TABLE>>"
+        return content
+
+    def _add_node_to_agraph(self, agraph: pgv.AGraph, node: GraphNode):
+        """Adds a node to the graph distinguising between operation and data nodes
+
+        Args:
+            agraph (pgv.AGraph): graph to add the node to
+            node (GraphNode): node to add
+        """
+        if isinstance(node, GraphNodeOperation):
+            agraph.add_node(node, **(self._operation_attributes(node)))
+        elif isinstance(node, GraphNodeData):
+            agraph.add_node(node, **(self._data_attributes(node)))
+
+    def _data_edge_style(self, node: GraphNodeData, edge_attrs: dict) -> dict:
+        """Returns the style attributes for a data edge (an edge starting from a Data
+        node and ending in an Operation node)
+
+        Args:
+            node (GraphNodeData): input node
+            edge_attrs (dict): edge attributes
+
+        Returns:
+            dict: style attributes for the edge
+        """
+        return {
+            "color": "#1DE9B6",
+            "arrowsize": 2.0,
+            "style": "dashed",
+        }
+
+    def _operation_edge_style(self, node: GraphNodeOperation, edge_attrs: dict) -> dict:
+        """Returns the style attributes for an operation edge (an edge starting from an
+        Operation node and ending in a Data node)
+
+        Args:
+            node (GraphNodeOperation): input node
+            edge_attrs (dict): edge attributes
+
+        Returns:
+            dict: style attributes for the edge
+        """
+        return {
+            "color": "#00B0FF",
+            "arrowsize": 1.0,
+        }
+
+    def _add_edge_op2data(
+        self,
+        agraph: pgv.AGraph,
+        n0: GraphNodeOperation,
+        n1: GraphNodeData,
+        edge_attrs: dict,
+    ):
+        """Adds an edge from an Operation node to a Data node
+
+        Args:
+            agraph (pgv.AGraph): graph to add the edge to
+            n0 (GraphNodeOperation): starting node
+            n1 (GraphNodeData): ending node
+            edge_attrs (dict): edge attributes
+        """
+
+        edge_name = edge_attrs.get(DAGNodesGraph.GraphAttrs.OUTPUT_PORT, "")
+        agraph.add_edge(
+            n0,
+            n1,
+            label=f"{edge_name}",
+            **(self._data_edge_style(n1, edge_attrs)),
+        )
+
+    def _add_edge_data2op(
+        self,
+        agraph: pgv.AGraph,
+        n0: GraphNodeData,
+        n1: GraphNodeOperation,
+        edge_attrs: dict,
+    ):
+        """Adds an edge from a Data node to an Operation node
+
+        Args:
+            agraph (pgv.AGraph): graph to add the edge to
+            n0 (GraphNodeData): starting node
+            n1 (GraphNodeOperation): ending node
+            edge_attrs (dict): edge attributes
+        """
+
+        edge_name = edge_attrs.get(DAGNodesGraph.GraphAttrs.INPUT_PORT, "")
+        agraph.add_edge(
+            n0,
+            n1,
+            label=f"{edge_name}",
+            **(self._operation_edge_style(n1, edge_attrs)),
+        )
+
+    def _add_edge_to_agraph(
+        self,
+        agraph: pgv.AGraph,
+        n0: GraphNode,
+        n1: GraphNode,
+        edge_attrs: dict,
+    ):
+        """Adds an edge to the graph
+
+        Args:
+            agraph (pgv.AGraph): graph to add the edge to
+            n0 (GraphNode): starting node
+            n1 (GraphNode): ending node
+            edge_attrs (dict): edge attributes
+
+        Raises:
+            ValueError: if the edge is not managed
+        """
+
+        if isinstance(n0, GraphNodeOperation) and isinstance(n1, GraphNodeData):
+            self._add_edge_op2data(agraph, n0, n1, edge_attrs)
+        elif isinstance(n0, GraphNodeData) and isinstance(n1, GraphNodeOperation):
+            self._add_edge_data2op(agraph, n0, n1, edge_attrs)
+        else:
+            raise ValueError(f"Unknown edge type: {type(n0)}, {type(n1)}")
+
+    def _build_agraph(self, graph: DAGNodesGraph) -> pgv.AGraph:
+        """Builds the graphviz graph from a DAGNodesGraph
+
+        Args:
+            graph (DAGNodesGraph): input graph
+
+        Returns:
+            pgv.AGraph: graphviz graph
+        """
+        agraph = pgv.AGraph(directed=True)
+
+        for edge in graph.raw_graph.edges():
+            n0: GraphNode = edge[0]
+            n1: GraphNode = edge[1]
+
+            self._add_node_to_agraph(agraph, n0)
+            self._add_node_to_agraph(agraph, n1)
+            self._add_edge_to_agraph(agraph, n0, n1, graph.raw_graph.edges[n0, n1])
+
+        return agraph
+
+    def draw(self, graph: DAGNodesGraph) -> np.ndarray:
+        """Draws a graph and returns the image as a numpy array
+
+        Args:
+            graph (DAGNodesGraph): graph to draw
+
+        Returns:
+            np.ndarray: image as a numpy array
+        """
+        import imageio
+
+        agraph: pgv.AGraph = self._build_agraph(graph)
+        agraph.layout("dot")
+        f = NamedTemporaryFile(suffix=".png")
+        agraph.draw(f.name)
+        return imageio.imread(f.name)
+
+    def representation(self, graph: DAGNodesGraph) -> str:
+        """Returns a representation of the graph as a DOT string
+
+        Args:
+            graph (DAGNodesGraph): graph to represent
+
+        Returns:
+            str: DOT string
+        """
+
+        agraph = self._build_agraph(graph)
+
+        return agraph.string()
+
+    def exportable_formats(self) -> Sequence[str]:
+        """Returns the exportable formats. They are mixed formats (png, pdf, ...) and
+        the dot format.
+
+        Returns:
+            Sequence[str]: exportable formats
+        """
+        return ["png", "dot", "pdf", "svg"]
+
+    def export(self, graph: DAGNodesGraph, filename: str, format: Optional[str] = None):
+        """Exports the graph to a file
+
+        Args:
+            graph (DAGNodesGraph): graph to export
+            filename (str): filename
+            format (str): format
+        """
+        super().export(graph, filename, format)
+
+        if format is None:
+            format = Path(filename).suffix[1:]
+
+        agraph = self._build_agraph(graph)
+        if format in ["dot"]:
+            agraph.write(filename)
+        else:
+            agraph.layout("dot")
+            agraph.draw(filename)
diff --git a/pipelime/pipes/drawing/mermaid.py b/pipelime/pipes/drawing/mermaid.py
new file mode 100644
index 0000000..c72a300
--- /dev/null
+++ b/pipelime/pipes/drawing/mermaid.py
@@ -0,0 +1,236 @@
+from typing import Optional, Sequence
+from pathlib import Path
+import numpy as np
+from pipelime.pipes.drawing.base import NodesGraphDrawer
+from pipelime.pipes.graph import (
+    GraphNode,
+    GraphNodeData,
+    GraphNodeOperation,
+    DAGNodesGraph,
+)
+import base64
+from PIL import Image
+import requests
+from io import BytesIO
+
+
+class MermaidNodesGraphDrawer(NodesGraphDrawer):
+    PRIVATE_CHARS = ["@"]
+    SUBSTITUTION_CHAR = "_"
+    OPERATION_NAME = "operation"
+    DATA_NAME = "data"
+    MERMAID_SERVICE_URL = "https://mermaid.ink/img/{}"
+
+    def _is_node_file(self, node: GraphNodeData) -> bool:
+        """Returns True if the node path is (or could be) a file
+
+        Args:
+            node (GraphNodeData):  input node
+
+        Returns:
+            bool:  True if the node path is (or could be) a file
+        """
+        return len(Path(node.path).suffix) > 0
+
+    def _data_node_shape(self, node: GraphNodeData) -> str:
+        """Returns the shape string of a data node
+
+        Args:
+            node (GraphNodeData): input node
+
+        Returns:
+            str: shape of the node
+        """
+
+        return (
+            f"{node.name}[[{node.name}]]"
+            if self._is_node_file(node)
+            else f"{node.name}[({node.name})]"
+        )
+
+    def _node_representation(self, node: GraphNode) -> str:
+        """Returns the string mermaid representation of a node
+
+        Args:
+            node (GraphNode): node to represent
+
+        Raises:
+            ValueError: if node is not a valid node
+
+        Returns:
+            str: mermaid textual representation of the node
+        """
+
+        out = ""
+        if isinstance(node, GraphNodeOperation):
+            out = f"{node.name}:::{MermaidNodesGraphDrawer.OPERATION_NAME}"
+        elif isinstance(node, GraphNodeData):
+            out = f"{self._data_node_shape(node)}:::{MermaidNodesGraphDrawer.DATA_NAME}"
+        else:
+            raise ValueError(f"Unknown node type: {type(node)}")
+
+        for c in MermaidNodesGraphDrawer.PRIVATE_CHARS:
+            out = out.replace(c, MermaidNodesGraphDrawer.SUBSTITUTION_CHAR)
+        return out
+
+    def _style_attrs_operation(self) -> dict:
+        """
+        Returns the style attributes for an operation node
+        """
+        return {"fill": "#03A9F4", "stroke": "#fafafa", "color": "#fff"}
+
+    def _style_attrs_data(self) -> dict:
+        """
+        Returns the style attributes for a data node
+        """
+        return {"fill": "#009688", "stroke": "#fafafa", "color": "#fff"}
+
+    def _classdef_row(self, name: str, style_attrs: dict) -> str:
+        """Returns the string representation of a class (style) definition
+
+        Args:
+            name (str): name of the class
+            style_attrs (dict): style attributes of the class
+
+        Returns:
+            str: string representation of the class definition
+        """
+
+        def style_string(x):
+            return ",".join([f"{k}:{v}" for k, v in x.items()])
+
+        return f"classDef {name} {style_string(style_attrs)};"
+
+    def _header_row(self, orientation: str = "TB") -> str:
+        """Returns the string representation of the header row of the mermaid graph
+
+        Args:
+            orientation (str, optional): TB - top to bottom, TD - top-down/ same as top
+            to bottom, BT - bottom to top, RL - right to left, LR - left to right .
+            Defaults to "TB".
+
+        Returns:
+            str: string representation of the header row
+        """
+        return f"flowchart {orientation}"
+
+    def _edge_row(self, graph: DAGNodesGraph, n0: GraphNode, n1: GraphNode) -> str:
+        """Returns the string representation of an edge row in the mermaid graph
+
+        Args:
+            graph (DAGNodesGraph): graph to draw
+            n0 (GraphNode): first node of the edge
+            n1 (GraphNode): second node of the edge
+
+        Returns:
+            str: string representation of the edge row
+        """
+
+        edge_attrs = graph.raw_graph.edges[n0, n1]
+
+        edge_name = ""
+        if DAGNodesGraph.GraphAttrs.INPUT_PORT in edge_attrs:
+            edge_name = f"|{edge_attrs[DAGNodesGraph.GraphAttrs.INPUT_PORT]}|"
+        if DAGNodesGraph.GraphAttrs.OUTPUT_PORT in edge_attrs:
+            edge_name += f"|{edge_attrs[DAGNodesGraph.GraphAttrs.OUTPUT_PORT]}|"
+
+        return "{}-->{}{}".format(
+            self._node_representation(n0),
+            edge_name,
+            self._node_representation(n1),
+        )
+
+    def _add_row(self, representation: str, row: str, header: bool = False):
+        """Append a row to the representation with correct indentation and newline
+
+        Args:
+            representation (str): representation to append to
+            row (str): row to append
+            header (bool, optional): if TRUE the row is a header row. Defaults to False.
+
+        Returns:
+            _type_: representation with the row appended
+        """
+        representation = representation + (f"{row}\n" if header else f"\t{row}\n")
+        return representation
+
+    def draw(self, graph: DAGNodesGraph) -> np.ndarray:
+        """Draws a graph and returns the image as a numpy array
+
+        Args:
+            graph (DAGNodesGraph): graph to draw
+
+        Returns:
+            np.ndarray: image as a numpy array
+        """
+
+        representation = self.representation(graph)
+        graphbytes = representation.encode("ascii")
+        base64_bytes = base64.b64encode(graphbytes)
+        base64_string = base64_bytes.decode("ascii")
+        url = MermaidNodesGraphDrawer.MERMAID_SERVICE_URL.format(base64_string)
+        response = requests.get(url)
+        return np.array(Image.open(BytesIO(response.content)))
+
+    def representation(self, graph: DAGNodesGraph) -> str:
+        """Returns the string representation of the graph in the mermaid language
+
+        Args:
+            graph (DAGNodesGraph): graph to draw
+
+        Returns:
+            str: string representation of the graph in the mermaid language
+        """
+
+        # execution_stack = graph.build_execution_stack()
+
+        # initialize representation
+        repr = ""
+
+        # flochart header
+        repr = self._add_row(repr, self._header_row(), header=True)
+
+        # style calss for operations
+        repr = self._add_row(
+            repr,
+            self._classdef_row(self.OPERATION_NAME, self._style_attrs_operation()),
+        )
+
+        # style class for data
+        repr = self._add_row(
+            repr,
+            self._classdef_row(self.DATA_NAME, self._style_attrs_data()),
+        )
+
+        for edge in graph.raw_graph.edges():
+            n0: GraphNode = edge[0]
+            n1: GraphNode = edge[1]
+
+            repr = self._add_row(repr, self._edge_row(graph, n0, n1))
+
+        return repr
+
+    def exportable_formats(self) -> Sequence[str]:
+        """Returns the exportable formats of the graph drawer.
+
+        Returns:
+            Sequence[str]: exportable formats of the graph drawer
+        """
+        return ["png", "markdown", "md"]
+
+    def export(self, graph: DAGNodesGraph, filename: str, format: Optional[str] = None):
+        super().export(graph, filename, format)
+
+        if format is None:
+            format = Path(filename).suffix[1:]
+
+        if format == "png":
+            import imageio
+
+            img = self.draw(graph)
+            imageio.imwrite(filename, img)
+        elif format in ["markdown", "md"]:
+            representation = self.representation(graph)
+            markdown = f"\n```mermaid\n{representation}\n```\n"
+            with open(filename, "w") as f:
+                f.write(markdown)
diff --git a/pipelime/pipes/executors/__init__.py b/pipelime/pipes/executors/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pipelime/pipes/executors/base.py b/pipelime/pipes/executors/base.py
new file mode 100644
index 0000000..dc0ab34
--- /dev/null
+++ b/pipelime/pipes/executors/base.py
@@ -0,0 +1,60 @@
+from abc import ABC, abstractmethod
+from typing import Any, Sequence
+from pipelime.pipes.graph import DAGNodesGraph
+from pipelime.pipes.model import NodeModel
+
+
+class NodeModelExecutionParser(ABC):
+    """A NodeModelExecutionParser is an abstract class that should implement the parsing
+    of a NodeModel into a bash command and arguments.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    @abstractmethod
+    def build_command_chunks(self, node_model: NodeModel) -> Sequence[Any]:
+        """Builds the command chunks for the given node model. Chunks are pieces of the command
+        that usually are separated by spaces.
+
+        Args:
+            node_model (NodeModel): input node model
+
+        Returns:
+            Sequence[Any]: command chunks
+        """
+        raise NotImplementedError()
+
+    def build_plain_command(self, node_model: NodeModel) -> str:
+        """Builds the plain command for the given node model. It joins the command chunks with
+        spaces.
+
+        Args:
+            node_model (NodeModel): input node model
+
+        Returns:
+            str: plain bash command
+        """
+        return " ".join(self.build_command_chunks(node_model=node_model))
+
+
+class NodesGraphExecutor(ABC):
+    """An NodesGraphExecutor is an abstract class that should implement the execution of a
+    NodesGraph made of Nodes.
+
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    @abstractmethod
+    def exec(self, graph: DAGNodesGraph) -> bool:
+        """Executes the given NodesGraph.
+
+        Args:
+            graph (DAGNodesGraph): input DAGNodesGraph
+
+        Returns:
+            bool: TRUE if the execution was successful, FALSE otherwise
+        """
+        raise NotImplementedError()
diff --git a/pipelime/pipes/executors/naive.py b/pipelime/pipes/executors/naive.py
new file mode 100644
index 0000000..5495866
--- /dev/null
+++ b/pipelime/pipes/executors/naive.py
@@ -0,0 +1,254 @@
+from pathlib import Path
+import subprocess
+from typing import Any, Dict, List, Sequence, Tuple
+from pipelime.pipes.executors.base import NodeModelExecutionParser, NodesGraphExecutor
+from pipelime.pipes.graph import GraphNodeOperation, DAGNodesGraph
+from pipelime.pipes.model import NodeModel
+from pipelime.pipes.piper import PiperNamespace
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.validation import OperationValidate, SampleSchema, SchemaLoader
+from loguru import logger
+
+
+class NaiveNodeModelExecutionParser(NodeModelExecutionParser):
+    def _append_argument_to_chunks(
+        self,
+        chunks: Sequence[str],
+        argument_name: str,
+        value: Any,
+    ):
+        """Appends the given argument to the given chunks list. It manages different
+        types of values.
+
+        Args:
+            chunks (Sequence[str]): input chunks list [in/out]
+            argument_name (str): input argument name
+            value (Any): input argument value
+        """
+        if isinstance(value, List):
+            for x in value:
+                if isinstance(x, List):
+                    self._append_argument_to_chunks(chunks, argument_name, tuple(x))
+                else:
+                    self._append_argument_to_chunks(chunks, argument_name, x)
+        elif isinstance(value, Tuple):
+            chunks.append(f"--{argument_name}")
+            for t in value:
+                chunks.append(str(t))
+        elif isinstance(value, Dict):
+            chunks.append(f"--{argument_name}")
+            for k, v in value.items():
+                chunks.append(str(k))
+                chunks.append(str(v))
+        else:
+            chunks.append(f"--{argument_name}")
+            chunks.append(str(value))
+
+    def build_command_chunks(self, node_model: NodeModel) -> Sequence[Any]:
+        """Builds the command chunks from the given node model parsing inputs, outputs
+        and arguments.
+
+        Args:
+            node_model (NodeModel): input node model
+
+        Returns:
+            Sequence[Any]: the command chunks (as a ' ' saperated string)
+        """
+        chunks = node_model.command.split(" ")
+
+        if node_model.inputs is not None:
+            for k, v in node_model.inputs.items():
+                self._append_argument_to_chunks(chunks, k, v)
+
+        if node_model.outputs is not None:
+            for k, v in node_model.outputs.items():
+                self._append_argument_to_chunks(chunks, k, v)
+
+        if node_model.args is not None:
+            for k, v in node_model.args.items():
+                self._append_argument_to_chunks(chunks, k, v)
+
+        return chunks
+
+
+class NaiveGraphExecutor(NodesGraphExecutor):
+    def __init__(self) -> None:
+        super().__init__()
+        self._validated_paths = set()
+
+    def _validate_path(
+        self,
+        node_model: NodeModel,
+        name: str,
+        value: any,
+        schema_file: str,
+    ) -> bool:
+        """Validates the given path and given schema.
+
+        Args:
+            node_model (NodeModel): input node model
+            name (str): input/output name
+            value (any): the port value, could be a path or a list of paths
+            schema_file (str): provided schema file
+
+        Raises:
+            NotImplementedError: if value is not a path or a list of paths
+            SampleSchema.ValidationError: if the validation fails
+
+        Returns:
+            bool: True if the validation succeeds
+        """
+
+        # Iterate over list of values if any
+        if isinstance(value, list):
+            [
+                self._validate_path(
+                    node_model,
+                    name=name,
+                    value=x,
+                    schema_file=schema_file,
+                )
+                for x in value
+            ]
+            return
+
+        # Check if value is a string
+        if isinstance(value, str):
+            path = value
+        else:
+            raise NotImplementedError(f"{type(value)} is not supported")
+
+        # Avoid validating the same path twice
+        if path in self._validated_paths:
+            return
+
+        if Path(path).is_dir():
+
+            try:
+                reader = UnderfolderReader(folder=path)
+                if schema_file is not None:
+
+                    if not Path(schema_file).exists():
+                        raise SampleSchema.ValidationError(
+                            f'Schema file "{schema_file}" not found'
+                        )
+
+                    schema_file = SchemaLoader.load(schema_file)
+                    try:
+
+                        op = OperationValidate(sample_schema=schema_file)
+                        op(reader)
+                    except SampleSchema.ValidationError as e:
+                        logger.error(
+                            f"Validation error on node: {node_model.command}:{name}:{path} -> {e}"
+                        )
+                        raise SampleSchema.ValidationError
+
+                    # Add path to validated paths to avoid validating it twice
+                    self._validated_paths.add(path)
+
+            except FileNotFoundError:
+                pass
+
+        return True
+
+    def _validate_node_inputs(self, node_model: NodeModel) -> bool:
+        """Validates the given node inputs.
+
+        Args:
+            node_model (NodeModel): input node model
+
+        Returns:
+            bool: True if the validation succeeds
+        """
+
+        if node_model.inputs is not None:
+            for input_name, value in node_model.inputs.items():
+                self._validate_path(
+                    node_model,
+                    name=input_name,
+                    value=value,
+                    schema_file=node_model.get_input_schema(input_name),
+                )
+
+    def _validate_node_outputs(self, node_model: NodeModel) -> bool:
+        """Validates the given node outputs.
+
+        Args:
+            node_model (NodeModel): input node model
+
+        Returns:
+            bool: True if the validation succeeds
+        """
+
+        if node_model.outputs is not None:
+            for output_name, value in node_model.outputs.items():
+                self._validate_path(
+                    node_model,
+                    name=output_name,
+                    value=value,
+                    schema_file=node_model.get_output_schema(output_name),
+                )
+
+    def exec(self, graph: DAGNodesGraph, token: str = "") -> bool:
+        """Executes the given graph.
+
+        Args:
+            graph (DAGNodesGraph): target graph
+            token (str, optional): execution token shared among nodes. Defaults to "".
+
+        Raises:
+            RuntimeError: if some node fails to execute
+
+        Returns:
+            bool: True if the execution succeeds
+        """
+
+        parser = NaiveNodeModelExecutionParser()
+        self._validated_paths.clear()
+        # channel: PiperCommunicationChannel = (
+        #     PiperCommunicationChannelFactory.create_channel(token=token)
+        # )
+
+        for layer in graph.build_execution_stack():
+            for node in layer:
+                node: GraphNodeOperation
+
+                command_chunks: List = parser.build_command_chunks(
+                    node_model=node.node_model
+                )
+
+                if len(token) > 0:
+                    command_chunks.append(PiperNamespace.ARGUMENT_NAME_TOKEN)
+                    command_chunks.append(token)
+
+                command = " ".join(command_chunks)
+
+                logger.debug(f"Executing command: {command}")
+
+                # Validate inputs before call
+                try:
+                    self._validate_node_inputs(node.node_model)
+                except SampleSchema.ValidationError:
+                    logger.error("Execution aborted")
+                    raise SampleSchema.ValidationError
+
+                pipe = subprocess.Popen(
+                    command_chunks,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                )
+
+                stdout, stderr = pipe.communicate()
+                if pipe.returncode == 0:
+
+                    # validate produced outputs
+                    try:
+                        self._validate_node_outputs(node.node_model)
+                    except SampleSchema.ValidationError:
+                        logger.error("Execution aborted")
+                        raise SampleSchema.ValidationError
+
+                else:
+                    logger.error(f"Node {str(node)} failed -> {stderr}")
+                    raise RuntimeError(f"{stderr}")
diff --git a/pipelime/pipes/graph.py b/pipelime/pipes/graph.py
new file mode 100644
index 0000000..fca732e
--- /dev/null
+++ b/pipelime/pipes/graph.py
@@ -0,0 +1,319 @@
+from typing import Sequence, Set
+from pipelime.pipes.model import NodeModel, DAGModel
+import networkx as nx
+import itertools
+
+
+class GraphNode:
+    GRAPH_NODE_TYPE_OPERATION = "operation"
+    GRAPH_NODE_TYPE_DATA = "data"
+
+    def __init__(self, name: str, type: str):
+        self.name = name
+        self.type = type
+
+    @property
+    def id(self):
+        return f"{self.type}({self.name})"
+
+    def __hash__(self) -> int:
+        return hash(f"{self.type}({self.name})")
+
+    def __repr__(self) -> str:
+        return f"{self.type}({self.name})"
+
+    def __eq__(self, o: object) -> bool:
+        return self.id == o.id
+
+
+class GraphNodeOperation(GraphNode):
+    def __init__(self, name: str, node_model: NodeModel):
+        super().__init__(name, GraphNode.GRAPH_NODE_TYPE_OPERATION)
+        self._node_model = node_model
+
+    @property
+    def node_model(self) -> NodeModel:
+        return self._node_model
+
+
+class GraphNodeData(GraphNode):
+    def __init__(self, name: str, path: str):
+        super().__init__(name, GraphNode.GRAPH_NODE_TYPE_DATA)
+        self._path = path
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+
+class DAGNodesGraph:
+    class GraphAttrs:
+        INPUT_PORT = "input_port"
+        OUTPUT_PORT = "output_port"
+        EDGE_TYPE = "edge_type"
+
+    def __init__(self, raw_graph: nx.DiGraph):
+        """Initialize the DAG graph starting from a raw directed graph (networkx).
+
+        Args:
+            raw_graph (nx.DiGraph): The raw graph to be converted to a DAG graph.
+        """
+        super().__init__()
+        self._raw_graph = raw_graph
+
+    @property
+    def raw_graph(self) -> nx.DiGraph:
+        """The raw graph (networkx) of the DAG.
+
+        Returns:
+            nx.DiGraph: The raw graph (networkx) of the DAG.
+        """
+        return self._raw_graph
+
+    @property
+    def operations_graph(self) -> "DAGNodesGraph":
+        """The operations graph (networkx) of the DAG. It is a filtered version of the
+        raw graph with operations nodes only.
+
+        Returns:
+            DAGNodesGraph: The operations graph (networkx) of the DAG.
+        """
+        return DAGNodesGraph.filter_node_graph(
+            self, [GraphNode.GRAPH_NODE_TYPE_OPERATION]
+        )
+
+    @property
+    def data_graph(self) -> "DAGNodesGraph":
+        """The data graph (networkx) of the DAG. It is a filtered version of the
+        raw graph with data nodes only.
+
+        Returns:
+            DAGNodesGraph: The data graph (networkx) of the DAG.
+        """
+        return DAGNodesGraph.filter_node_graph(self, [GraphNode.GRAPH_NODE_TYPE_DATA])
+
+    @property
+    def root_nodes(self) -> Sequence[GraphNode]:
+        """The root nodes of the DAG. They are the nodes that have no predecessors.
+
+        Returns:
+            Sequence[GraphNode]: The root nodes of the DAG.
+        """
+        return [
+            node
+            for node in self.raw_graph.nodes
+            if len(list(self.raw_graph.predecessors(node))) == 0
+        ]
+
+    def consumable_operations(
+        self,
+        produced_data: Set[GraphNodeData],
+    ) -> Set[GraphNodeOperation]:
+        """Given a set of produced data, returns the set of operations that can be
+        consumed given the produced data, i.e. the operations that have inputs available
+
+        Args:
+            produced_data (Set[GraphNodeData]): The set of produced data.
+
+        Returns:
+            Set[GraphNodeOperation]: The set of operations that can be consumed given
+            the produced data.
+        """
+
+        consumables = set()
+        for node in self.operations_graph.raw_graph.nodes:
+            in_data = [
+                x
+                for x in self.raw_graph.predecessors(node)
+                if isinstance(x, GraphNodeData)
+            ]
+            if all(x in produced_data for x in in_data):
+                consumables.add(node)
+        return consumables
+
+    def consume(
+        self,
+        operation_nodes: Sequence[GraphNodeOperation],
+    ) -> Set[GraphNodeData]:
+        """Given a set of operations, consume them and return the set of produced data,
+        i.e. the data are the outputs of the operations.
+
+        Args:
+            operation_nodes (Sequence[GraphNodeOperation]): The set of operations to be
+            consumed.
+
+        Returns:
+            Set[GraphNodeData]: The set of produced data.
+        """
+        consumed_data = set()
+        for node in operation_nodes:
+            out_data = [
+                x
+                for x in self.raw_graph.successors(node)
+                if isinstance(x, GraphNodeData)
+            ]
+            consumed_data.update(out_data)
+        return consumed_data
+
+    def build_execution_stack(self) -> Sequence[Sequence[GraphNodeOperation]]:
+        """Builds the execution stack of the DAG. The execution stack is a list of lists
+        of operations. Each list of operations is a virtual layer of operations that
+        can be executed in parallel because they have no dependencies.
+
+        Returns:
+            Sequence[Sequence[GraphNodeOperation]]: The execution stack of the DAG.
+        """
+
+        # the execution stack is a list of lists of nodes. Each list represents a
+        execution_stack: Sequence[Sequence[GraphNodeOperation]] = []
+
+        # initalize global produced data with the root nodes
+        global_produced_data = set()
+        global_produced_data.update(
+            [x for x in self.root_nodes if isinstance(x, GraphNodeData)]
+        )
+
+        # set of operations that have been consumed
+        consumed_operations = set()
+
+        while True:
+            # which operations can be consumed given the produced data?
+            consumable: set = self.consumable_operations(global_produced_data)
+
+            # Remove from consumable operations the ones that have already been consumed
+            consumable = consumable.difference(consumed_operations)
+
+            # No consumable operations? We are done!
+            if len(consumable) == 0:
+                break
+
+            # If not empty, append consumable operations to the execution stack
+            execution_stack.append(consumable)
+
+            # The set of produced data is the union of all the consumed operations
+            produced_data: set = self.consume(consumable)
+
+            # Add the consumed operations to the consumed operations set
+            consumed_operations.update(consumable)
+
+            # Add the produced data to the global produced data
+            global_produced_data.update(produced_data)
+
+        return execution_stack
+
+    @classmethod
+    def build_nodes_graph(
+        cls,
+        dag_model: DAGModel,
+    ) -> "DAGNodesGraph":
+        """Builds the nodes graph of the DAG starting from a plain DAG model.
+
+        Returns:
+            DAGNodesGraph: The nodes graph of the DAG.
+        """
+
+        g = nx.DiGraph()
+
+        for node_name, node in dag_model.nodes.items():
+            node: NodeModel
+
+            inputs = node.inputs
+            outputs = node.outputs
+
+            if inputs is not None:
+                for input_name, input_value in inputs.items():
+                    if isinstance(input_value, str):
+                        input_value = [input_value]
+
+                    attrs = {}
+                    for x in input_value:
+                        n0 = GraphNodeData(str(x), str(x))
+                        n1 = GraphNodeOperation(node_name, node)
+                        g.add_edge(n0, n1)
+                        attrs.update(
+                            {
+                                (n0, n1): {
+                                    DAGNodesGraph.GraphAttrs.INPUT_PORT: input_name,
+                                    DAGNodesGraph.GraphAttrs.EDGE_TYPE: "DATA_2_OPERATION",
+                                }
+                            }
+                        )
+                    nx.set_edge_attributes(g, attrs)
+
+            if outputs is not None:
+                for output_name, output_value in outputs.items():
+                    if isinstance(output_value, str):
+                        output_value = [output_value]
+
+                    attrs = {}
+                    for x in output_value:
+                        n0 = GraphNodeOperation(node_name, node)
+                        n1 = GraphNodeData(str(x), str(x))
+                        g.add_edge(n0, n1)
+                        attrs.update(
+                            {
+                                (n0, n1): {
+                                    DAGNodesGraph.GraphAttrs.OUTPUT_PORT: output_name,
+                                    DAGNodesGraph.GraphAttrs.EDGE_TYPE: "OPERATION_2_DATA",
+                                }
+                            }
+                        )
+
+                    nx.set_edge_attributes(g, attrs)
+
+        return DAGNodesGraph(raw_graph=g)
+
+    @classmethod
+    def filter_raw_graph(
+        cls,
+        full_graph: nx.DiGraph,
+        types: Sequence[str],
+    ) -> nx.DiGraph:
+        """Filters the raw graph of the DAG to keep only the nodes of the given types.
+
+        Args:
+            full_graph (nx.DiGraph): The raw graph of the DAG.
+            types (Sequence[str]): The types of the nodes to keep.
+
+        Returns:
+            nx.DiGraph: The filtered raw graph of the DAG.
+        """
+
+        filtered_graph: nx.DiGraph = nx.DiGraph()
+
+        for node in full_graph.nodes:
+            if node.type not in types:
+                predecessors = list(full_graph.predecessors(node))
+                successors = list(full_graph.successors(node))
+                pairs = list(itertools.product(predecessors, successors))
+                for pair in pairs:
+                    filtered_graph.add_edge(pair[0], pair[1])
+
+        # If filtered graph is empty could be a single layer graph without connections
+        # in this case all nodes are kept
+        if len(filtered_graph.nodes()) == 0:
+            for node in full_graph.nodes:
+                if node.type in types:
+                    filtered_graph.add_node(node)
+
+        return filtered_graph
+
+    @classmethod
+    def filter_node_graph(
+        cls,
+        nodes_graph: "DAGNodesGraph",
+        types: Sequence[str],
+    ) -> "DAGNodesGraph":
+        """Filters the nodes graph of the DAG to keep only the nodes of the given types.
+        It wraps the raw graph into a DAGNodesGraph.
+
+        Args:
+            nodes_graph (DAGNodesGraph): The nodes graph of the DAG.
+            types (Sequence[str]): The types of the nodes to keep.
+
+        Returns:
+            DAGNodesGraph: The filtered nodes graph of the DAG.
+        """
+        return DAGNodesGraph(
+            raw_graph=cls.filter_raw_graph(nodes_graph.raw_graph, types)
+        )
diff --git a/pipelime/pipes/model.py b/pipelime/pipes/model.py
new file mode 100644
index 0000000..5dc9dae
--- /dev/null
+++ b/pipelime/pipes/model.py
@@ -0,0 +1,31 @@
+from typing import Dict, Optional
+from pydantic import BaseModel
+
+
+class NodeModel(BaseModel):
+    command: str
+    args: Optional[dict] = None
+    inputs: Optional[dict] = None
+    outputs: Optional[dict] = None
+    outputs_schema: Optional[dict] = None
+    inputs_schema: Optional[dict] = None
+
+    def get_output_schema(self, name: str) -> Optional[str]:
+        if self.outputs_schema is not None:
+            return self.outputs_schema.get(name, None)
+        return None
+
+    def get_input_schema(self, name: str) -> Optional[str]:
+        if self.inputs_schema is not None:
+            return self.inputs_schema.get(name, None)
+        return None
+
+
+class DAGModel(BaseModel):
+    nodes: Dict[str, NodeModel]
+
+    def purged_dict(self):
+        return self.dict(
+            exclude_unset=True,
+            exclude_none=True,
+        )
diff --git a/pipelime/pipes/parsers/__init__.py b/pipelime/pipes/parsers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pipelime/pipes/parsers/base.py b/pipelime/pipes/parsers/base.py
new file mode 100644
index 0000000..3a99a9d
--- /dev/null
+++ b/pipelime/pipes/parsers/base.py
@@ -0,0 +1,62 @@
+from abc import ABC, abstractclassmethod
+from typing import Optional
+from pipelime.pipes.model import DAGModel
+from choixe.configurations import XConfig
+
+
+class DAGConfigParser(ABC):
+    PARAMS_NAMESPACE = "params"
+    NODES_NAMESPACE = "nodes"
+
+    @abstractclassmethod
+    def parse_cfg(
+        self,
+        cfg: dict,
+        global_data: Optional[dict] = None,
+    ) -> DAGModel:
+        """Parses the given configuration into a DAGModel.
+
+        Args:
+            cfg (dict): input configuration as dictionary
+            global_data (Optional[dict], optional): the global data dictionary. Defaults to None.
+
+        Returns:
+            DAGModel: resulting DAGModel
+        """
+        pass
+
+    def parse_file(
+        self,
+        cfg_file: str,
+        params_file: Optional[str] = None,
+        additional_args: Optional[dict] = None,
+    ) -> DAGModel:
+        """Parses the given configuration file into a DAGModel.
+
+        Args:
+            cfg_file (str): input configuration file
+            params_file (Optional[str], optional): the global data file. Defaults to None.
+
+        Returns:
+            DAGModel: resulting DAGModel
+        """
+
+        cfg = XConfig(cfg_file).to_dict()
+        global_data = {}
+        if params_file:
+            global_data = XConfig(params_file)
+
+            if additional_args is not None:
+
+                # Replace placeholders filling default values if not provided
+                global_data.replace_variables_map(
+                    additional_args,
+                    replace_defaults=True,
+                )
+
+            # Checks for placeholders, if any close the execution!
+            global_data.check_available_placeholders(close_app=True)
+
+            global_data = global_data.to_dict()
+
+        return self.parse_cfg(cfg=cfg, global_data=global_data)
diff --git a/pipelime/pipes/parsers/factory.py b/pipelime/pipes/parsers/factory.py
new file mode 100644
index 0000000..f4e6aff
--- /dev/null
+++ b/pipelime/pipes/parsers/factory.py
@@ -0,0 +1,59 @@
+from typing import Optional, Sequence
+from pydantic import BaseModel
+from pipelime.pipes.parsers.base import DAGConfigParser
+from pipelime.pipes.parsers.simple import DAGSimpleParser
+from pipelime.pipes.model import DAGModel
+from choixe.configurations import XConfig
+
+
+class DAGConfigParserFactoryConfigurationModel(BaseModel):
+    parser_name: str = "DAGSimpleParser"
+
+
+class DAGConfigParserFactory:
+
+    FACTORY_MAP = {
+        "DAGSimpleParser": DAGSimpleParser,
+    }
+
+    @classmethod
+    def parse_file(
+        cls,
+        cfg_file: str,
+        params_file: Optional[str] = None,
+        additional_args: Optional[dict] = None,
+    ) -> DAGModel:
+        """Parse the given configuration file into a DAGModel.
+
+        Args:
+            cfg_file (str): input configuration file
+            params_file (Optional[str], optional): additional params file. Defaults to None.
+            additional_args (Optional[dict], optional): _description_. Defaults to None.
+
+        Raises:
+            NotImplementedError: _description_
+
+        Returns:
+            DAGModel: _description_
+        """
+
+        # builds/validate the generic parser configuration
+        cfg = DAGConfigParserFactoryConfigurationModel(**(XConfig(cfg_file).to_dict()))
+
+        # is parser available?
+        if cfg.parser_name in cls.FACTORY_MAP:
+
+            # retrieve the parser
+            parser: DAGConfigParser = cls.FACTORY_MAP[cfg.parser_name]()
+
+            return parser.parse_file(
+                cfg_file=cfg_file,
+                params_file=params_file,
+                additional_args=additional_args,
+            )
+        else:
+            raise NotImplementedError(f"parser {cfg.parser_name} not implemented")
+
+    @classmethod
+    def available_parsers(cls) -> Sequence[str]:
+        return list(DAGConfigParserFactory.FACTORY_MAP.keys())
diff --git a/pipelime/pipes/parsers/simple.py b/pipelime/pipes/parsers/simple.py
new file mode 100644
index 0000000..9818cf6
--- /dev/null
+++ b/pipelime/pipes/parsers/simple.py
@@ -0,0 +1,500 @@
+import copy
+from typing import Callable, Optional, Sequence, Tuple
+import re
+import pydash
+from pipelime.pipes.parsers.base import DAGConfigParser
+from pipelime.tools.dictionaries import DictionaryUtils
+from pipelime.pipes.model import DAGModel
+
+
+class DAGSimpleParser(DAGConfigParser):
+
+    PLACEHOLDER_CHAR = "\$"
+    PLACEHOLDER_REGEX = r = f"{PLACEHOLDER_CHAR}(\\w*)\\(([^)]+)\\)"  # @methodName(arg)
+    PLACEHOLDER_VARIABLE_NAME = "var"
+    PLACEHOLDER_ITERATION_NAME = "iter"
+    PLACEHOLDER_ARG_ITERATION_NAME = "argiter"
+    PLACEHOLDER_FOREACH_NAME = "foreach"
+    PLACEHOLDER_FOREACHARG_NAME = "foreach_arg"
+    PLACEHOLDER_FOREACH_DO_NAME = "do"
+    PLACEHOLDER_FOREACH_ITEMS_NAME = "items"
+    PLACEHOLDER_ARG_SPLIT_CHAR = "@"
+
+    def __init__(self) -> None:
+        self._regex = re.compile(self.PLACEHOLDER_REGEX)
+        self._global_data = {}
+
+    def clear_global_data(self):
+        self._global_data.clear()
+
+    def set_global_data(self, key: str, value: any):
+        """Set a global data key/value pair.
+
+        :param key: key
+        :type key: str
+        :param value: value
+        :type value: any
+        """
+        self._global_data[key] = value
+
+    def _get_value(self, data: dict, dotted_key: str) -> any:
+        """Get a value from a dict.
+
+        :param data: dict
+        :type data: dict
+        :param key: key
+        :type key: str
+        :return: value
+        :rtype: any
+        """
+        if not pydash.has(data, dotted_key):
+            raise KeyError(f"Key {dotted_key} not found among provided params/data")
+        else:
+            return pydash.get(data, dotted_key)
+
+    def _convert_match_to_variable(self, m: re.Match) -> any:
+        """This is a callable, used in the _parse_string function. It will be called on
+        every "value" found in the configuration dict. It converts the regex match to the
+        corresponding value in the global data.
+
+        :param m: re.Match object
+        :type m: re.Match
+        :return: parsed value
+        :rtype: any
+        """
+        command, content = m.groups()
+        if command.lower() == DAGSimpleParser.PLACEHOLDER_VARIABLE_NAME:
+            value = self._get_value(self._global_data, content)
+            return value
+        else:
+            return m.group()
+
+    def _convert_match_to_item_data(self, m: re.Match, index: int, item: any) -> any:
+        """This is a callable, used in the _parse_string function. It will be called on
+        nodes in a foreach loop. It converts the regex match to the corresponding value/s
+        of the current item.
+
+        :param m: re.Match object
+        :type m: re.Match
+        :param index: current index in the foreach loop
+        :type index: int
+        :param item: current item in the foreach loop
+        :type item: any
+        :return: parsed value
+        :rtype: any
+        """
+        local_data = {"item": item, "index": index}
+        command, content = m.groups()
+        if command.lower() == DAGSimpleParser.PLACEHOLDER_ITERATION_NAME:
+            value = self._get_value(local_data, content)
+            return value
+        else:
+            return m.group()
+
+    def _convert_match_to_argitem_data(self, m: re.Match, index: int, item: any) -> any:
+        """This is a callable, used in the _parse_string function. It will be called on
+        nodes in a foreach loop FOR ARGUMENTS only. It converts the regex match to the corresponding value/s
+        of the current item.
+
+        :param m: re.Match object
+        :type m: re.Match
+        :param index: current index in the foreach loop
+        :type index: int
+        :param item: current item in the foreach loop
+        :type item: any
+        :return: parsed value
+        :rtype: any
+        """
+        local_data = {"item": item, "index": index}
+        command, content = m.groups()
+        if command.lower() == DAGSimpleParser.PLACEHOLDER_ARG_ITERATION_NAME:
+            value = self._get_value(local_data, content)
+            return value
+        else:
+            return m.group()
+
+    def _parse_foreach_node(self, node: dict, item: any, index: int) -> dict:
+        """Parse a foreach node (a dict). This is a node containing some @iter(index) and @iter(item)
+        placeholder which will be replaced by the current item and index values in a foreach
+        loop. For example a node could be a random dict like:
+
+        {
+            'data0': '@iter(item)',
+            'data1': {
+                'index': '@iter(index)',
+            }
+        }
+
+        :param node: node to parse
+        :type node: dict
+        :param item: current item in the foreach loop
+        :type item: any
+        :param index: current index in the foreach loop
+        :type index: int
+        :return: parsed node
+        :rtype: dict
+        """
+        empty = copy.deepcopy(node)
+        flat_node = DictionaryUtils.flatten(node)
+        for key, value in flat_node.items():
+            flat_node[key] = self._parse_string(
+                value, lambda m: self._convert_match_to_item_data(m, index, item)
+            )
+            pydash.set_(empty, key, flat_node[key])
+
+        return empty
+
+    def _parse_string(
+        self, s: str, replace_cb: Callable[[re.Match], any] = _convert_match_to_variable
+    ) -> any:
+        """Parse a string searching for placeholders. For example a string could be:
+
+        'this is a string @var(params.input_folders) with a placeholder'
+
+        The placeholder will be replaced by the value of the global data 'params.input_folders'.
+        If the string contains multiple placeholders, the function will be called multiple times.
+
+        If the string IS the placeholder itself (e.g. '@var(params.input_folders)'),
+        the value of the global data will be returned and could be any type of data.
+
+        :param s: string to parse
+        :type s: str
+        :param replace_cb: callable function responsible to build the repelcamente value
+        given the input re.Match , defaults to lambdax:x.group()
+        :type replace_cb: Callable[[re.Match], any], optional
+        :return: parsed string. Could be a string or another type.
+        :rtype: any
+        """
+
+        # How many matches?
+        try:
+            occurrences = len(self._regex.findall(s))
+        except Exception:
+            occurrences = 0
+
+        # If one match and the match is the same as the string, return the value
+        # this is used to replace values with any type other than a string. This is
+        # possibile only if the whole input string is a placeholder.
+        if occurrences == 1:
+            match = next(self._regex.finditer(s))
+            if match.group() == s:
+                return replace_cb(match)
+
+        # One or more matches among other random characters. This means that each match
+        # can be replaced only with a string (a cast to string is done).
+        if occurrences >= 1:
+            matches = self._regex.finditer(s)
+            repl_map = {}
+            for match in matches:
+                repl_map[match.group()] = str(replace_cb(match))
+            return self._regex.sub(lambda m: repl_map[m.group()], s)
+
+        # Nothing found. Return the original string.
+        else:
+            return s
+
+    def _extract_foreach_data(
+        self,
+        node: dict,
+    ) -> Tuple[Optional[dict], Optional[dict]]:
+        """Extract the foreach data from the node if any. A foreach node is a dict like:
+
+        {
+            'foreach': {
+                'items': '@var(params.input_folders)',
+                'do': {
+                    'data0': '@iter(item)',
+                    'data1': {
+                        'index': '@iter(index)',
+                    }
+                }
+            }
+        }
+
+        :param node: source node
+        :type node: dict
+        :raises Exception: If needed keys are missing ('do' and 'items')
+        :return: (pseudo node, foreach data). The pseudo node is a dict within the 'do' key.
+        'foreach data' is the data contained in the 'items' key. If no foreach data is found,
+        the pseudo node and the foreach data are None.
+        :rtype: Tuple[Optional[dict], Optional[dict]]
+        """
+        if DAGSimpleParser.PLACEHOLDER_FOREACH_NAME in node:
+            foreach_node = node[DAGSimpleParser.PLACEHOLDER_FOREACH_NAME]
+            pseudo_node = foreach_node.get(
+                DAGSimpleParser.PLACEHOLDER_FOREACH_DO_NAME, None
+            )
+            foreach_data = foreach_node.get(
+                DAGSimpleParser.PLACEHOLDER_FOREACH_ITEMS_NAME, None
+            )
+            if pseudo_node is None or foreach_data is None:
+                raise KeyError("Invalid foreach node. Missing 'do' or 'items' keys.")
+
+            return pseudo_node, foreach_data
+        else:
+            return None, None
+
+    def _expand_dict_values(self, branch_cfg: dict) -> dict:
+        """Parse a generic dict where values could be a foreach node. This is used to
+        parse foreach(s) inside nodes arguments, like input or output ports. A branch config
+        could be something like:
+
+        {
+            'inputA': {
+                'foreach': {
+                    'items': ['a', 'b', 'c'],
+                    'do': '@var(params.base_folder)/@iter(item)'
+                    }
+                }
+            },
+            'inputB': {
+                'foreach': {
+                    'items': ['a', 'b', 'c'],
+                    'do': '@var(params.base_folder)/@iter(index)'
+                    }
+                }
+            }
+        }
+
+        :param branch_cfg: branch to parse
+        :type branch_cfg: dict
+        :raises Exception: If needed keys are missing ('do' and 'items')
+        :raises Exception: If the 'do' item is not a string
+        :raises Exception: If the 'items' item is not a Sequence
+        :return: parsed branch
+        :rtype: dict
+        """
+
+        branch_cfg = copy.deepcopy(branch_cfg)
+
+        to_replace_data = {}
+        for key, value in branch_cfg.items():
+            if isinstance(value, dict):
+                if DAGSimpleParser.PLACEHOLDER_FOREACH_NAME in value:
+                    pseudo_node, foreach_data = self._extract_foreach_data(value)
+                    if pseudo_node is not None:
+                        # if not isinstance(pseudo_node, str):
+                        #     raise Exception(
+                        #         "Invalid foreach node. 'do' key must be a sequence."
+                        #     )
+                        if not isinstance(foreach_data, Sequence):
+                            raise TypeError(
+                                "Invalid foreach node. 'items' key must be a Sequence."
+                            )
+                        parsed_list = []
+                        for value_index, value in enumerate(foreach_data):
+                            parsed_list.append(
+                                self._parse_string(
+                                    pseudo_node,
+                                    lambda m: self._convert_match_to_argitem_data(
+                                        m, value_index, value
+                                    ),
+                                )
+                            )
+                        to_replace_data[key] = parsed_list
+
+        for key, value in to_replace_data.items():
+            branch_cfg[key] = value
+
+        return branch_cfg
+
+    def _replace_variables_deep(self, cfg: dict) -> dict:
+        """Deep replace all variables placeholders in the config.
+
+        :param cfg: config to parse
+        :type cfg: dict
+        :return: parsed config
+        :rtype: dict
+        """
+
+        new_cfg = copy.deepcopy(cfg)
+
+        # Parse variables. This is done first because the variables could be used in
+        # other nodes/foreach nodes. Iterates over the whole configuration as a flat list
+        # of pydash(ed) keys and values
+        for key, value in DictionaryUtils.flatten(cfg).items():
+            if isinstance(value, str):
+                pydash.set_(
+                    new_cfg,
+                    key,
+                    self._parse_string(
+                        value, replace_cb=self._convert_match_to_variable
+                    ),
+                )
+
+        return new_cfg
+
+    def _expand_nodes(self, nodes_cfg: dict) -> dict:
+        """Iterates through the nodes and expand foreach-nodes.
+
+        :param nodes_cfg: nodes config {node_name: node_config}
+        :type nodes_cfg: dict
+        :return: expanded nodes config {node_name: node_config}
+        :rtype: dict
+        """
+
+        nodes_cfg = copy.deepcopy(nodes_cfg)
+
+        to_add_nodes = {}  # used to not modify original dict while iterating
+        to_delete_nodes = set()  # used to not modify original dict while iterating
+        for node_name in nodes_cfg:
+            node = nodes_cfg[node_name]
+            pseudo_node, foreach_data = self._extract_foreach_data(node)
+
+            # if pseudo_node is not None, then the node is a foreach node!
+            if pseudo_node is not None:
+
+                if not isinstance(pseudo_node, dict):
+                    raise TypeError("Invalid foreach node. 'do' key must be a dict.")
+
+                # Iterate the foreach data and create a new node for each item
+                for index, data in enumerate(foreach_data):
+                    new_node_key = str(index)
+                    if isinstance(data, str):
+                        new_node_key = data
+
+                    # Create a new node with a name based on original name and the index
+                    to_add_nodes[
+                        f"{node_name}@{new_node_key}"
+                    ] = self._parse_foreach_node(pseudo_node, data, index)
+
+                # forwards the node to be deleted
+                to_delete_nodes.add(node_name)
+
+        # remove the foreach nodes
+        for node_name in to_delete_nodes:
+            del nodes_cfg[node_name]
+
+        # Add the new nodes to the original config
+        for node_name in to_add_nodes:
+            nodes_cfg[node_name] = to_add_nodes[node_name]
+
+        return nodes_cfg
+
+    def _expand_nodes_arguments(self, nodes_cfg: dict) -> dict:
+        """Iterate through all nodes and expand the foreach branches in arguments.
+
+        :param nodes_cfg: nodes config {node_name: node_config}
+        :type nodes_cfg: dict
+        :return: expanded config {node_name: node_config}
+        :rtype: dict
+        """
+
+        nodes_cfg = copy.deepcopy(nodes_cfg)
+
+        for node_name in nodes_cfg:
+            node = nodes_cfg[node_name]
+
+            to_expand = {}
+            for key, value in node.items():
+                if isinstance(value, dict):
+                    to_expand[key] = self._expand_dict_values(value)
+
+            for key, value in to_expand.items():
+                nodes_cfg[node_name][key] = value
+
+        return nodes_cfg
+
+    def _merge_multiple_arguments(self, nodes_cfg: dict) -> dict:
+        """Merge node input/output/arguments with a name containing a '@' inside.
+        For example:
+
+        {
+            'args': {
+                'par@0': [1,2,3],
+                'par@1': [4,5,6]
+            }
+        }
+
+        becomes:
+
+        {
+            'args': {
+                'par': [(1,2), (3,4), (5,6)]],
+            }
+        }
+
+        Args:
+            nodes_cfg (dict): nodes config {node_name: node_config}
+
+        Returns:
+            dict: merged config {node_name: node_config}
+        """
+        nodes_cfg = copy.deepcopy(nodes_cfg)
+
+        for node_name in nodes_cfg:
+            node = nodes_cfg[node_name]
+
+            to_replace = {}
+            for key, subnode in node.items():
+                if isinstance(subnode, dict):
+                    rephrased_args = {}
+                    for arg_name, value in subnode.items():
+                        if DAGSimpleParser.PLACEHOLDER_ARG_SPLIT_CHAR in arg_name:
+                            arg_name, arg_index = arg_name.split(
+                                DAGSimpleParser.PLACEHOLDER_ARG_SPLIT_CHAR
+                            )
+                            if arg_name not in rephrased_args:
+                                rephrased_args[arg_name] = {}
+                            rephrased_args[arg_name][arg_index] = value
+
+                        else:
+                            rephrased_args[arg_name] = value
+
+                    for arg_name, value in list(rephrased_args.items()):
+                        if isinstance(value, dict):
+                            raw_values = list(value.values())
+                            raw_size = len(raw_values)
+                            row_size = -1
+                            if raw_size > 0:
+                                row_size = len(raw_values[0])
+
+                            if row_size > 0:
+                                remapped = []
+                                for r in range(row_size):
+                                    row = [raw_values[i][r] for i in range(raw_size)]
+                                    remapped.append(row)
+                                rephrased_args[arg_name] = remapped
+                    to_replace[key] = rephrased_args
+
+            for k, v in to_replace.items():
+                node[k] = v
+
+        return nodes_cfg
+
+    def parse_cfg(self, cfg: dict, global_data: Optional[dict] = None) -> DAGModel:
+        """Parse the configuration
+
+        :param cfg: configuration to parse
+        :type cfg: dict
+        :param global_variables: global variables
+        :type global_variables: Optional[dict], default None
+        :return: nodes model
+        :rtype: NodesModel
+        """
+
+        # sets global variables
+        if global_data is not None:
+            self.clear_global_data()
+            [self.set_global_data(k, v) for k, v in global_data.items()]
+
+        # replace all variables
+        parsed = self._replace_variables_deep(cfg)
+
+        # Parse foreach nodes. Each configuration node could contain a foreach node. This
+        # means that the node generates multiple nodes based on a list of values.
+        parsed[DAGConfigParser.NODES_NAMESPACE] = self._expand_nodes(
+            parsed[DAGConfigParser.NODES_NAMESPACE]
+        )
+
+        # Parse the branches. AKA foreach nodes inside nodes inputs/outputs values.
+        parsed[DAGConfigParser.NODES_NAMESPACE] = self._expand_nodes_arguments(
+            parsed[DAGConfigParser.NODES_NAMESPACE]
+        )
+
+        # Merge multiple arguments as tuples
+        parsed[DAGConfigParser.NODES_NAMESPACE] = self._merge_multiple_arguments(
+            parsed[DAGConfigParser.NODES_NAMESPACE]
+        )
+
+        return DAGModel(**parsed)
diff --git a/pipelime/pipes/piper.py b/pipelime/pipes/piper.py
new file mode 100644
index 0000000..cc3ee4e
--- /dev/null
+++ b/pipelime/pipes/piper.py
@@ -0,0 +1,321 @@
+from __future__ import annotations
+
+import functools
+import io
+import subprocess
+import uuid
+from typing import Any, Callable, Dict, Optional, Sequence, Union
+
+import click
+import yaml
+from loguru import logger
+from yaml.scanner import ScannerError
+
+from pipelime.pipes.communication import PiperCommunicationChannelFactory
+
+
+class PiperNamespace:
+    """Namespace constants for Piper ecosystem."""
+
+    PIPER_PREFIX = "piper_"
+    PRIVATE_ARGUMENT_PREFIX = "---"
+    PRIVATE_OPTION_PREFIX = "_"
+    NAME_INPUTS = "inputs"
+    NAME_OUTPUTS = "outputs"
+    NAME_TOKEN = "token"
+    NAME_INFO = "info"
+    NAME_ARGS = "args"
+    COMMAND_KWARGS_NAME = "piper_kwargs"
+
+    """ Names of kwargs variable """
+    OPTION_NAME_INPUTS = f"{PRIVATE_OPTION_PREFIX}{PIPER_PREFIX}{NAME_INPUTS}"
+    OPTION_NAME_OUTPUTS = f"{PRIVATE_OPTION_PREFIX}{PIPER_PREFIX}{NAME_OUTPUTS}"
+    OPTION_NAME_TOKEN = f"{PRIVATE_OPTION_PREFIX}{PIPER_PREFIX}{NAME_TOKEN}"
+    OPTION_NAME_INFO = f"{PRIVATE_OPTION_PREFIX}{PIPER_PREFIX}{NAME_INFO}"
+
+    """ Names of click arguments """
+    ARGUMENT_NAME_INPUTS = f"{PRIVATE_ARGUMENT_PREFIX}{PIPER_PREFIX}{NAME_INPUTS}"
+    ARGUMENT_NAME_OUTPUTS = f"{PRIVATE_ARGUMENT_PREFIX}{PIPER_PREFIX}{NAME_OUTPUTS}"
+    ARGUMENT_NAME_TOKEN = f"{PRIVATE_ARGUMENT_PREFIX}{PIPER_PREFIX}{NAME_TOKEN}"
+    ARGUMENT_NAME_INFO = f"{PRIVATE_ARGUMENT_PREFIX}{PIPER_PREFIX}{NAME_INFO}"
+
+
+class PiperCommand:
+    instance: PiperCommand = None
+
+    def __init__(
+        self,
+        fn: Callable,
+        inputs: Sequence[str],
+        outputs: Sequence[str],
+    ) -> None:
+        # Extract default values from the kwargs
+        self._fn = fn
+        self.__name__ = fn.__name__
+        self._caller_name = f"{fn.__module__}:{fn.__name__}"
+        self._inputs = inputs
+        self._outputs = outputs
+
+        # progress callbacks list
+        self._progress_callbacks = []
+
+    def _progress_callback(self, chunk_index: int, total_chunks: int, payload: dict):
+        self.log(
+            "_progress",
+            {
+                "chunk_index": chunk_index,
+                "total_chunks": total_chunks,
+                "progress_data": payload,
+            },
+        )
+
+    def generate_progress_callback(
+        self,
+        chunk_index: int = 0,
+        total_chunks: int = 1,
+    ) -> Callable[[dict], None]:
+        """Generates a progress callback function to send back to the caller. The callback
+        should be called every time the external progress is updated. Internally the callback
+        will log the progress (also in the communication channel).
+
+        Args:
+            chunk_index (int, optional): index for multiple user tasks. Defaults to 0.
+            total_chunks (int, optional): total number of chunks. Defaults to 1.
+
+        Returns:
+            Callable[[dict], None]: the callback function
+        """
+        callback = functools.partial(self._progress_callback, chunk_index, total_chunks)
+        self._progress_callbacks.append(callback)
+        return callback
+
+    def clear_progress_callbacks(self):
+        self._progress_callbacks = []
+
+    @property
+    def active(self) -> bool:
+        return self._active
+
+    @property
+    def _log_header(self) -> str:
+        return f"{self._id}|"
+
+    def log(self, key: str, value: any):
+        """Logs a key/value pair into the communication channel.
+
+        Args:
+            key (str): The key to log.
+            value (any): The value to log. Can be any picklable object.
+        """
+        if self.active:
+            logger.debug(f"{self._log_header}Logging {key}={value}")
+            self._channel.send(self._id, {key: value})
+
+    def _init_state(self, token: str) -> None:
+        self.__class__.instance = self
+        self._token = token if len(token) > 0 else None
+
+        # Token check, if not provided, the command is disabled
+        self._active = self._token is not None
+
+        if self._active:
+            # Builds an unique id for the command, multiple instances of same command
+            # are allowed, for this reason an unique id is required
+            self._unique_identifier = str(uuid.uuid1())
+            self._id = f"{self._caller_name}:{self._unique_identifier}"
+
+            # Creates the communication channel among commands
+            self._channel = PiperCommunicationChannelFactory.create_channel(self._token)
+
+    def _log_state(self) -> None:
+        if self._active:
+            # Logs the command creation
+            logger.debug(f"{self._log_header}New Piper created from: {self._id}")
+            logger.debug(f"{self._log_header}\tPiper inputs: {self._inputs}")
+            logger.debug(f"{self._log_header}\tPiper outputs: {self._outputs}")
+            logger.debug(f"{self._log_header}\tPiper token: {self._token}")
+
+    def _filter_kwargs(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        to_remove = {
+            PiperNamespace.OPTION_NAME_TOKEN,
+            PiperNamespace.OPTION_NAME_INFO,
+            PiperNamespace.OPTION_NAME_OUTPUTS,
+            PiperNamespace.OPTION_NAME_INPUTS,
+        }
+        return {k: v for k, v in kwargs.items() if k not in to_remove}
+
+    def __call__(self, *args, **kwargs):
+        token = kwargs.get(PiperNamespace.OPTION_NAME_TOKEN, "")
+        kwargs = self._filter_kwargs(kwargs)
+        self._init_state(token)
+        self._log_state()
+        self._fn(*args, **kwargs)
+        self.__class__.instance = None
+
+
+class Piper:
+    @staticmethod
+    def _piper_info_callback(
+        ctx: click.core.Context, param: click.core.Option, value: bool
+    ):
+        """Callback for the click eager corresponding option. If the eager option is
+        set to True, the command will exit printing the Piper info into the pipe.
+
+        Args:
+            ctx (click.core.Context): The click context.
+            param (click.core.Option): The click option.
+            value (bool): The value of the eager option.
+        """
+        if value:
+            stream = io.StringIO()
+            yaml.safe_dump(ctx.command.to_info_dict(ctx), stream)
+            click.echo(stream.getvalue())  # (ctx.command.to_info_dict(ctx))
+            ctx.exit()
+
+    @staticmethod
+    def command(
+        inputs: Optional[Sequence[str]] = None,
+        outputs: Optional[Sequence[str]] = None,
+    ):
+        """This is the special decorator for the Piper command. It is used to add hidden
+        options to the command used to manager the Piper ecosystem.
+
+        Args:
+            inputs (Optional[Sequence[str]], optional): List of click command options
+            treated as inputs. Defaults to None.
+            outputs (Optional[Sequence[str]], optional): List of click command options
+            treated as output . Defaults to None.
+        """
+
+        def _wrapped(func):
+            cmd = PiperCommand(func, inputs, outputs)
+            cmd = click.option(
+                PiperNamespace.ARGUMENT_NAME_TOKEN, default="", hidden=True
+            )(cmd)
+            cmd = click.option(
+                PiperNamespace.ARGUMENT_NAME_INPUTS, default=inputs, hidden=True
+            )(cmd)
+            cmd = click.option(
+                PiperNamespace.ARGUMENT_NAME_OUTPUTS, default=outputs, hidden=True
+            )(cmd)
+            cmd = click.option(
+                PiperNamespace.ARGUMENT_NAME_INFO,
+                is_flag=True,
+                is_eager=True,
+                expose_value=False,
+                callback=Piper._piper_info_callback,
+                hidden=True,
+            )(cmd)
+            return cmd
+
+        return _wrapped
+
+    @classmethod
+    def piper_command_raw_info(cls, command: str) -> Union[None, dict]:
+        """Retrieves the Piper info from a generic bash command. If the bash command
+        is a valid Piper command, the Piper info will be returned as a dict. If the
+        bash command is not a valid Piper command, None will be returned.
+
+        Args:
+            command (str): The bash command to execute. The command should be the raw bash
+            command, without any arguments provided. Automatically the eager option will
+            be added to the command to force the Piper to manifest itself.
+
+        Returns:
+            Union[None, dict]: The Piper info as a dict or None if the bash command
+        """
+
+        # Append the piper eager option to the command
+        command += f" {PiperNamespace.ARGUMENT_NAME_INFO}"
+        info = None
+
+        pipe = subprocess.Popen(
+            command.split(" "),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = pipe.communicate()
+        if pipe.returncode == 0:
+            try:
+                info = yaml.safe_load(stdout)
+            except ScannerError:
+                logger.error(f"{command} is not a valid Piper command!")
+                info = None
+        else:
+            raise TypeError(stderr)
+
+        return info
+
+    @classmethod
+    def piper_command_description(cls, command: str) -> dict:
+        """Retrieves the Piper structured description from a generic bash command.
+        If the bash command is a valid Piper command, the Piper description will be returned
+        as a dict. If the bash command is not a valid Piper command, None will be returned.
+
+        Args:
+            command (str): The bash command to execute. The command should be the raw bash
+            command, without any arguments provided. Automatically the eager option will
+            be added to the command to force the Piper to manifest itself.
+
+        Raises:
+            RuntimeError: If the Piper command is not a valid Piper command.
+
+        Returns:
+            dict: The Piper description as a dict or None if the bash command is not a
+            valid Piper command.
+        """
+
+        raw_info = cls.piper_command_raw_info(command)
+        if raw_info is None:
+            raise TypeError(f"Command '{command}' is not a piper!")
+
+        commands_map = {x["name"]: x for x in raw_info["params"]}
+
+        piper_inputs = commands_map[PiperNamespace.OPTION_NAME_INPUTS]
+        piper_outputs = commands_map[PiperNamespace.OPTION_NAME_OUTPUTS]
+
+        # Remove the piper options from the commands map
+        del commands_map[PiperNamespace.OPTION_NAME_INPUTS]
+        del commands_map[PiperNamespace.OPTION_NAME_OUTPUTS]
+        del commands_map[PiperNamespace.OPTION_NAME_TOKEN]
+        del commands_map[PiperNamespace.OPTION_NAME_INFO]
+
+        # Retrieves inputs/outputs fields
+        inputs_list = piper_inputs["default"]
+        outputs_list = piper_outputs["default"]
+
+        # initialize the description
+        description = {
+            PiperNamespace.NAME_INPUTS: {},
+            PiperNamespace.NAME_OUTPUTS: {},
+            PiperNamespace.NAME_ARGS: {},
+        }
+
+        # For each inputs check if a corresponding click command option is present
+        if inputs_list is not None:
+            for i in inputs_list:
+                if i not in commands_map:
+                    raise KeyError(f"Input '{i}' is not a valid Piper command option!")
+                description[PiperNamespace.NAME_INPUTS][i] = commands_map[i]
+                del commands_map[i]
+
+        # For each outputs check if a corresponding click command option is present
+        if outputs_list is not None:
+            for o in outputs_list:
+                if o not in commands_map:
+                    raise KeyError(f"Output '{o}' is not a valid Piper command option!")
+                description[PiperNamespace.NAME_OUTPUTS][o] = commands_map[o]
+                del commands_map[o]
+
+        # Adds remaining options as generic arguments
+        description[PiperNamespace.NAME_ARGS] = commands_map
+
+        return description
+
+    @classmethod
+    def piper_info_argument(cls):
+        return PiperNamespace.ARGUMENT_NAME_INFO
+
+    @classmethod
+    def piper_token_argument(cls):
+        return PiperNamespace.ARGUMENT_NAME_TOKEN
diff --git a/pipelime/sequences/api/__init__.py b/pipelime/sequences/api/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pipelime/sequences/api/authentication.py b/pipelime/sequences/api/authentication.py
new file mode 100644
index 0000000..c35a8f0
--- /dev/null
+++ b/pipelime/sequences/api/authentication.py
@@ -0,0 +1,78 @@
+from typing import Callable, Dict, Optional
+from fastapi import HTTPException
+from fastapi.openapi.models import OAuthFlows as OAuthFlowsModel
+from fastapi.security.oauth2 import OAuth2
+from fastapi.security.utils import get_authorization_scheme_param
+from starlette.requests import Request
+from starlette.status import HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN
+
+
+class CustomAPIAuthentication(OAuth2):
+    def __init__(
+        self,
+        token_callback: Callable[[str], bool],
+        tokenUrl: str = "token",
+        scheme_name: Optional[str] = None,
+        scopes: Optional[Dict[str, str]] = None,
+        description: Optional[str] = None,
+        auto_error: bool = True,
+    ):
+        """This is a Custom authentication class passed as Depends into the API endpoints.
+        Classical authentication Depends just analyse the Authorization header and return the token.
+        This class, instead, is used to verify the token by means of a callback function
+        provided by the user. In this way the user can implement a custom authentication
+        scheme external to the API.
+
+        :param token_callback: A callback function that takes a token and returns a boolean
+        :type token_callback: Callable[[str], bool]
+        :param tokenUrl: the default token key , defaults to "token"
+        :type tokenUrl: str, optional
+        :param scheme_name: authentication scheme name, defaults to None
+        :type scheme_name: Optional[str], optional
+        :param scopes: authentication scopes, defaults to None
+        :type scopes: Optional[Dict[str, str]], optional
+        :param description: description , defaults to None
+        :type description: Optional[str], optional
+        :param auto_error: TRUE to auto-raise authentication error defaults to True
+        :type auto_error: bool, optional
+        """
+        if not scopes:
+            scopes = {}
+        flows = OAuthFlowsModel(password={"tokenUrl": tokenUrl, "scopes": scopes})
+        self._token_callback = token_callback
+        self._token_key = tokenUrl
+        super().__init__(
+            flows=flows,
+            scheme_name=scheme_name,
+            description=description,
+            auto_error=auto_error,
+        )
+
+    async def __call__(self, request: Request) -> Optional[str]:
+
+        # Get the token from the Authorization header and check presence
+        # If token not present the authentication fails automatically
+        authorization: str = request.headers.get("Authorization")
+        scheme, token = get_authorization_scheme_param(authorization)
+        if not authorization or scheme.lower() != "bearer":
+            if self._token_key in request.query_params:
+                token = request.query_params[self._token_key]
+            else:
+                if self.auto_error:
+                    raise HTTPException(
+                        status_code=HTTP_401_UNAUTHORIZED,
+                        detail="Not authenticated",
+                        headers={"WWW-Authenticate": "Bearer"},
+                    )
+                else:
+                    return None
+
+        # If the token is present:
+        # Verify the token by means of external callback function provided by the user
+        if not self._token_callback(token):
+            raise HTTPException(
+                status_code=HTTP_403_FORBIDDEN,
+                detail="Invalid Token",
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+        return token
diff --git a/pipelime/sequences/api/base.py b/pipelime/sequences/api/base.py
new file mode 100644
index 0000000..2b1dfc7
--- /dev/null
+++ b/pipelime/sequences/api/base.py
@@ -0,0 +1,34 @@
+from abc import abstractmethod
+from typing import Hashable, Sequence
+from pipelime.sequences.api.entities import EntityDataset, EntitySample
+import io
+
+
+class SequenceInterface:
+    @abstractmethod
+    def list_datasets(self) -> Sequence[EntityDataset]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_dataset(self, dataset_name: str) -> EntityDataset:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_sample(self, dataset_name: str, sample_id: Hashable) -> EntitySample:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_sample_data(
+        self, sample_id: int, item_name: str, format: str = None
+    ) -> io.BytesIO:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def put_sample(
+        self, dataset_name: str, sample_id: Hashable, sample_entity: EntitySample
+    ) -> EntitySample:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def search_samples(self, proto_sample: EntitySample) -> Sequence[EntitySample]:
+        raise NotImplementedError()
diff --git a/pipelime/sequences/api/entities.py b/pipelime/sequences/api/entities.py
new file mode 100644
index 0000000..d6cda7a
--- /dev/null
+++ b/pipelime/sequences/api/entities.py
@@ -0,0 +1,52 @@
+from pydantic.main import BaseModel
+from typing import Dict, Optional, Sequence
+
+
+class EntitySampleData(BaseModel):
+    type: str
+    encoding: str
+
+
+class EntitySample(BaseModel):
+    id: int
+    metadata: dict
+    data: Optional[Dict[str, EntitySampleData]]
+
+
+class EntityDatasetManifest(BaseModel):
+    size: int
+    keys: Sequence[str]
+    sample_ids: Sequence[int]
+
+
+class EntityDataset(BaseModel):
+    name: str
+    manifest: EntityDatasetManifest
+
+
+class EntityPagination(BaseModel):
+    offset: Optional[int] = 0
+    limit: Optional[int] = 50
+    total_count: Optional[int] = None
+
+    @classmethod
+    def create_from_sequence(self, Sequence: Sequence[any]) -> None:
+        pagination = EntityPagination()
+        pagination.offset = 0
+        pagination.limit = len(Sequence)
+        pagination.total_count = pagination.limit
+        return pagination
+
+    def filter(self, s: Sequence[any]) -> Sequence[any]:
+        return s[self.offset : self.offset + self.limit]
+
+
+class EntitySampleSearchRequest(BaseModel):
+    proto_sample: Optional[EntitySample]
+    only_pagination: Optional[bool] = False
+    pagination: Optional[EntityPagination] = None
+
+
+class EntitySampleSearchResponse(BaseModel):
+    samples: Sequence[EntitySample] = []
+    pagination: EntityPagination
diff --git a/pipelime/sequences/api/underfolder.py b/pipelime/sequences/api/underfolder.py
new file mode 100644
index 0000000..8b41363
--- /dev/null
+++ b/pipelime/sequences/api/underfolder.py
@@ -0,0 +1,562 @@
+from pipelime.sequences.api.authentication import CustomAPIAuthentication
+from pipelime.sequences.operations import OperationFilterByScript
+from pipelime.sequences.api.entities import (
+    EntityDataset,
+    EntityPagination,
+    EntitySample,
+    EntitySampleData,
+    EntitySampleSearchRequest,
+    EntitySampleSearchResponse,
+)
+from pipelime.sequences.api.base import (
+    SequenceInterface,
+)
+from pipelime.sequences.samples import FileSystemSample, Sample
+from pipelime.sequences.streams.underfolder import UnderfolderStream
+from pipelime.filesystem.toolkit import FSToolkit
+from pipelime.tools.bytes import DataCoding
+
+from typing import Any, Dict, Callable, Optional, Sequence
+from fastapi import HTTPException
+from fastapi.param_functions import Depends
+from fastapi.routing import APIRouter
+from starlette.responses import StreamingResponse
+from starlette.requests import Request
+import io
+from loguru import logger
+from pipelime.tools.dictionaries import DictSearch
+
+
+class UnderfolderInterface(SequenceInterface):
+    def __init__(
+        self, name: str, folder: str, allowed_keys: Optional[Sequence[str]] = None
+    ) -> None:
+        """The UnderfolderInterface is a SequenceInterface that uses Entities to
+        'communicate' with the underlying UnderfolderStream (i.e. an UnderfolderReader
+        and UnderfolderWriter wrapper).
+
+
+        :param name: name of the interface (should be unique among all interfaces)
+        :type name: str
+        :param folder: the Underfolder path
+        :type folder: str
+        :param allowed_keys: the list of allowed keys, if NONE all keys are allowed,
+        default None
+        :type allowed_keys: Optional[Sequence[str]], optional
+        """
+        self._name = name
+        self._stream = UnderfolderStream(folder=folder, allowed_keys=allowed_keys)
+
+    def _raw_sample_to_entity(self, raw_sample: Sample) -> EntitySample:
+        """Converts a raw sample into an entity sample.
+
+        :param raw_sample: the raw sample
+        :type raw_sample: Sample
+        :return: the entity sample
+        :rtype: EntitySample
+        """
+
+        # create entity
+        entity = EntitySample(id=raw_sample.id, metadata={}, data={})
+
+        # fill metadata and urls
+        for key in raw_sample:
+            filename = raw_sample.filesmap[key]
+            extension = FSToolkit.get_file_extension(filename)
+            if DataCoding.is_metadata_extension(extension):
+                entity.metadata[key] = raw_sample[key]
+            else:
+                # TODO: move this kind of "File Description" in unique proxy
+                if DataCoding.is_image_extension(extension):
+                    entity.data[key] = EntitySampleData(
+                        type="image", encoding=extension
+                    )
+                elif DataCoding.is_numpy_extension(extension):
+                    entity.data[key] = EntitySampleData(
+                        type="numpy", encoding=extension
+                    )
+                elif DataCoding.is_pickle_extension(extension):
+                    entity.data[key] = EntitySampleData(
+                        type="pickle", encoding=extension
+                    )
+                elif DataCoding.is_text_extension(extension):
+                    entity.data[key] = EntitySampleData(type="text", encoding=extension)
+
+        return entity
+
+    def _get_sample_entity(self, sample_id: int) -> EntitySample:
+        """Returns the sample entity for the given sample id. Wraps all raw metadata
+        into the EntitySample. For binary data just the item names and encoding will
+        be provided through the entity 'data' field
+
+        :param sample_id: the sample id
+        :type sample_id: int
+        :return: the sample entity built
+        :rtype: EntitySample
+        """
+
+        # get raw sample
+        raw_sample: FileSystemSample = self._stream.get_sample(sample_id)
+
+        return self._raw_sample_to_entity(raw_sample)
+
+    def _put_sample_entity(self, sample_id: int, entity: EntitySample) -> None:
+        """Updates the raw sample with the given entity.
+
+        :param sample_id: the sample id
+        :type sample_id: int
+        :param entity: the sample entity to update
+        :type entity: EntitySample
+        :raises PermissionError: if some metadata is not allowed to be updated
+        :raises KeyError: if sample_id is not found
+        """
+
+        try:
+            for key in entity.metadata:
+                self._stream.set_data(
+                    sample_id, key, entity.metadata[key], format="dict"
+                )
+
+        except PermissionError as e:
+            raise PermissionError(e)
+
+        except KeyError as e:
+            raise KeyError(e)
+
+    def get_dataset(self) -> EntityDataset:
+        """Returns the dataset entity. Wraps dataset info into the EntityDataset.
+
+        :return: the dataset entity
+        :rtype: EntityDataset
+        """
+
+        return EntityDataset(name=self._name, manifest=self._stream.manifest())
+
+    def get_sample(self, sample_id: int) -> EntitySample:
+        """Returns the sample entity for the given sample id. Wraps all raw metadata
+
+        :param sample_id: the sample id
+        :type sample_id: int
+        :raises KeyError: if sample_id is not found
+        :return: the sample entity built
+        :rtype: EntitySample
+        """
+
+        if sample_id in self._stream.get_sample_ids():
+            return self._get_sample_entity(sample_id)
+        else:
+            raise KeyError(f"Sample {sample_id} not found")
+
+    def get_sample_data(
+        self, sample_id: int, item_name: str, format: str = None
+    ) -> io.BytesIO:
+        """Returns the raw binary data for the given sample id and item name.
+
+        :param sample_id: the sample id
+        :type sample_id: int
+        :param item_name: the item name
+        :type item_name: str
+        :param format: binary data forma (e.g. jpg, png) , defaults to None
+        :type format: str, optional
+        :raises KeyError: if sample_id is not found
+        :raises KeyError: if item_name is not found
+        :return: io.BytesIO buffer
+        :rtype: io.BytesIO
+        """
+
+        if sample_id in self._stream.get_sample_ids():
+            entity = self._get_sample_entity(sample_id)
+
+            if item_name in entity.data:
+                if format is None:
+                    return self._stream.get_bytes(sample_id, item_name)
+                else:
+                    raise NotImplementedError(
+                        "Custom format not implemented yet, leave NONE for auto encoding"
+                        "based on filename extension!"
+                    )
+                    # item = entity.data[item_name]
+                    # data_format = format if format is not None else item["encoding"]
+                    # return self._stream.get_data(sample_id, item_name, format=data_format)
+            else:
+                raise KeyError(f"Item {item_name} not found")
+        else:
+            raise KeyError(f"Sample {sample_id} not found")
+
+    def put_sample(self, sample_id: int, sample_entity: EntitySample) -> EntitySample:
+        """Updates the raw sample with the given entity.
+
+        :param sample_id: the sample id
+        :type sample_id: int
+        :param sample_entity: the sample entity to update
+        :type sample_entity: EntitySample
+        :raises PermissionError: if some metadata is not allowed to be updated
+        :raises KeyError: if sample_id is not found
+        :raises KeyError: if item_name is not found
+        :return: the sample entity built
+        :rtype: EntitySample
+        """
+
+        if sample_id in self._stream.get_sample_ids():
+
+            try:
+
+                self._put_sample_entity(sample_id, sample_entity)
+                return self._get_sample_entity(sample_id)
+
+            except PermissionError as e:
+                raise PermissionError(e)
+
+        else:
+
+            raise KeyError(f"Sample {sample_id} not found")
+
+    def search_samples(self, proto_sample: EntitySample) -> Sequence[EntitySample]:
+        """Search samples based on the given sample entity.
+
+        :param proto_sample: the sample entity with query values
+        :type proto_sample: EntitySample
+        :return: the list of samples matching the query
+        :rtype: Sequence[EntitySample]
+        """
+
+        def filter_sample(sample: FileSystemSample, sequence) -> bool:
+            return DictSearch.match_queries(proto_sample.metadata, sample)
+
+        filtered_reader = OperationFilterByScript(path_or_func=filter_sample)(
+            self._stream.reader
+        )
+
+        entities = [self._raw_sample_to_entity(sample) for sample in filtered_reader]
+        return sorted(entities, key=lambda x: x.id)
+
+
+class UnderfolderAPI(APIRouter):
+    def __init__(
+        self,
+        underfolders_map: Dict[str, str],
+        allowed_keys_map: Optional[Dict[str, Sequence[str]]] = None,
+        auth_callback: Callable[[str], bool] = None,
+    ):
+        """Initialize the API router with a map of underfolder names to underfolder paths.
+
+        :param underfolders_map: A map of underfolder names to underfolder paths.
+        :type underfolders_map: Dict[str, str]
+        :param allowed_keys_map: A map of underfolder names to allowed keys.
+        :type allowed_keys_map: Dict[str, Sequence[str]]
+        :param auth_callback: A callback function that takes a token and returns a boolean
+            indicating whether the token is valid.
+        :type auth_callback: Callable[[str], bool]
+        """
+
+        super().__init__()
+
+        self._underfolders_map = underfolders_map
+
+        # Authentication callback verification
+        self._auth_callback: Optional[Callable[[str], bool]] = auth_callback
+
+        # Single underfodlers interfaces
+        self._interfaces_map: Dict[str, UnderfolderInterface] = {}
+        for name, folder in self._underfolders_map.items():
+            allowed_keys = (
+                allowed_keys_map[name]
+                if (allowed_keys_map is not None and name in allowed_keys_map)
+                else None
+            )
+            interface = UnderfolderInterface(name, folder, allowed_keys=allowed_keys)
+            self._interfaces_map[name] = interface
+
+            logger.info(
+                f"New API Interface created: (name={name}, folder={str(folder)})"
+            )
+
+        # .------------------.
+        # | API list_dataset |
+        # '------------------'
+        # generated with: https://texteditor.com/ascii-frames/
+
+        self.add_api_route(
+            "/datasets",
+            response_model=Dict[str, EntityDataset],
+            endpoint=self.list_datasets,
+            methods=["GET"],
+            dependencies=self._get_dependencies(),
+        )
+
+        # .-----------------.
+        # | API get_dataset |
+        # '-----------------'
+        self.add_api_route(
+            "/dataset/{dataset_name}",
+            response_model=EntityDataset,
+            endpoint=self.get_dataset,
+            methods=["GET"],
+            dependencies=self._get_dependencies(),
+        )
+
+        # .----------------.
+        # | API get_sample |
+        # '----------------'
+        self.add_api_route(
+            "/dataset/{dataset_name}/{sample_id}",
+            response_model=EntitySample,
+            endpoint=self.get_sample,
+            methods=["GET"],
+            dependencies=self._get_dependencies(),
+        )
+
+        # .---------------------.
+        # | API get_sample_data |
+        # '---------------------'
+        self.add_api_route(
+            "/dataset/{dataset_name}/{sample_id}/{item_name}",
+            response_model=Any,
+            endpoint=self.get_sample_data,
+            methods=["GET"],
+            dependencies=self._get_dependencies(),
+        )
+
+        # .----------------.
+        # | API put_sample |
+        # '----------------'
+        self.add_api_route(
+            "/dataset/{dataset_name}/{sample_id}",
+            response_model=EntitySample,
+            endpoint=self.put_sample,
+            methods=["PUT"],
+            dependencies=self._get_dependencies(),
+        )
+
+        # .-------------------.
+        # | API search_sample |
+        # '-------------------'
+        self.add_api_route(
+            "/search/{dataset_name}",
+            response_model=EntitySampleSearchResponse,
+            endpoint=self.search_samples,
+            methods=["POST"],
+            dependencies=self._get_dependencies(),
+        )
+
+    def _log_api(
+        self,
+        request: Request,
+        message: str = "",
+    ):
+        import inspect
+
+        message_part = f" [{message}] " if message else ""
+        curframe = inspect.currentframe()
+        calframe = inspect.getouterframes(curframe, 2)
+        logger.info(
+            f"[{request.method}] {calframe[1][3]}{message_part}<- {request.client}"
+        )
+        logger.debug(
+            f"[{request.method}] {calframe[1][3]} path_params: {request.path_params}"
+        )
+        logger.debug(
+            f"[{request.method}] {calframe[1][3]} query_params: {request.query_params}"
+        )
+
+    def _get_dependencies(self):
+        dependencies: Sequence[Depends] = []
+
+        # Adds authentication if callback is provided
+        if self._auth_callback is not None:
+            dependencies.append(
+                Depends(CustomAPIAuthentication(token_callback=self._auth_callback))
+            )
+
+        return dependencies
+
+    async def list_datasets(self, request: Request) -> Dict[str, EntityDataset]:
+        """List all datasets as a dictionary of dataset names to dataset entity.
+
+        :param request: The request object.
+        :type request: Request
+        :return: A dictionary of dataset names to dataset entity.
+        :rtype: Dict[str, EntityDataset]
+        """
+
+        self._log_api(request=request)
+
+        return {
+            name: interface.get_dataset()
+            for name, interface in self._interfaces_map.items()
+        }
+
+    async def get_dataset(self, dataset_name: str, request: Request) -> EntityDataset:
+        """Get a dataset entity by name.
+
+        :param dataset_name: The name of the dataset.
+        :type dataset_name: str
+        :param request: The request object.
+        :type request: Request
+        :raises HTTPException: If the dataset is not found.
+        :return: The retreived dataset entity.
+        :rtype: EntityDataset
+        """
+
+        self._log_api(request=request)
+
+        if dataset_name in self._interfaces_map:
+
+            return self._interfaces_map[dataset_name].get_dataset()
+
+        else:
+
+            raise HTTPException(
+                status_code=404, detail=f"Dataset {dataset_name} not found"
+            )
+
+    async def get_sample(
+        self, dataset_name: str, sample_id: int, request: Request
+    ) -> EntitySample:
+        """Get a sample entity by id and dataset name.
+
+        :param dataset_name: The name of the dataset.
+        :type dataset_name: str
+        :param sample_id: The id of the sample.
+        :type sample_id: int
+        :param request: The request object.
+        :type request: Request
+        :raises HTTPException: If the sample is not found.
+        :raises HTTPException: If the dataset is not found.
+        :return: The retreived sample entity.
+        :rtype: EntitySample
+        """
+
+        self._log_api(request=request)
+
+        if dataset_name in self._interfaces_map:
+            try:
+
+                return self._interfaces_map[dataset_name].get_sample(sample_id)
+
+            except KeyError as e:
+
+                raise HTTPException(status_code=404, detail=str(e))
+        else:
+
+            raise HTTPException(
+                status_code=404, detail=f"Dataset {dataset_name} not found"
+            )
+
+    async def get_sample_data(
+        self,
+        dataset_name: str,
+        sample_id: int,
+        item_name: str,
+        request: Request,
+        format: str = None,
+    ) -> StreamingResponse:
+        """Get a sample binary data by id and dataset name as a StreamingResponse.
+
+        :param dataset_name: The name of the dataset.
+        :type dataset_name: str
+        :param sample_id: The id of the sample.
+        :type sample_id: int
+        :param item_name:  The name of the item.
+        :type item_name: str
+        :param request: The request object.
+        :type request: Request
+        :param format: binary data encoding (e.g. jpg, png, pkl) , defaults to None
+        :type format: str, optional
+        :raises HTTPException: If the sample is not found.
+        :raises HTTPException: If the dataset is not found.
+        :raises HTTPException: If the item is not found.
+        :return: binary data stream with selected mimetype
+        :rtype: StreamingResponse
+        """
+
+        self._log_api(request=request)
+
+        if dataset_name in self._interfaces_map:
+            try:
+
+                data, mimetype = self._interfaces_map[dataset_name].get_sample_data(
+                    sample_id=sample_id, item_name=item_name, format=format
+                )
+
+                return StreamingResponse(data, media_type=mimetype)
+
+            except KeyError as e:
+
+                raise HTTPException(status_code=404, detail=str(e))
+
+        else:
+
+            raise HTTPException(
+                status_code=404, detail=f"Dataset {dataset_name} not found"
+            )
+
+    async def put_sample(
+        self,
+        dataset_name: str,
+        sample_id: int,
+        sample_entity: EntitySample,
+        request: Request,
+    ) -> EntitySample:
+        """Updates a sample entity by id and dataset name. The body of the request should
+        be a JSON serialized EntitySample object
+
+        :param dataset_name: The name of the dataset.
+        :type dataset_name: str
+        :param sample_id: The id of the sample.
+        :type sample_id: int
+        :param sample_entity: The sample entity data to update.
+        :type sample_entity: EntitySample
+        :param request: The request object.
+        :type request: Request
+        :raises HTTPException: If the sample is not found.
+        :raises HTTPException: If the dataset is not found.
+        :return: The retrieved sample entity after update operation.
+        :rtype: EntitySample
+        """
+
+        self._log_api(request=request)
+
+        if dataset_name in self._interfaces_map:
+            try:
+                return self._interfaces_map[dataset_name].put_sample(
+                    sample_id, sample_entity
+                )
+            except KeyError as e:
+                raise HTTPException(status_code=404, detail=str(e))
+        else:
+            raise HTTPException(status_code=404, detail=f"{dataset_name} not found")
+
+    async def search_samples(
+        self,
+        dataset_name: str,
+        request: Request,
+        search_entity: EntitySampleSearchRequest,
+    ) -> EntitySampleSearchResponse:
+
+        self._log_api(request=request)
+
+        if dataset_name in self._interfaces_map:
+
+            try:
+                samples_entities = self._interfaces_map[dataset_name].search_samples(
+                    search_entity.proto_sample
+                )
+
+                if search_entity.only_pagination:
+                    pagination = EntityPagination.create_from_sequence(samples_entities)
+                    return EntitySampleSearchResponse(pagination=pagination)
+                else:
+                    filtered = search_entity.pagination.filter(samples_entities)
+                    return EntitySampleSearchResponse(
+                        samples=filtered, pagination=search_entity.pagination
+                    )
+            except Exception:
+                import traceback
+
+                raise HTTPException(status_code=500, detail=f"{traceback.format_exc()}")
+        else:
+
+            raise HTTPException(
+                status_code=404, detail=f"Dataset {dataset_name} not found"
+            )
diff --git a/pipelime/sequences/operations.py b/pipelime/sequences/operations.py
index c9551d0..3688087 100644
--- a/pipelime/sequences/operations.py
+++ b/pipelime/sequences/operations.py
@@ -4,7 +4,7 @@
 import multiprocessing
 import random
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Optional, Sequence, Union
+from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Union
 
 import dictquery as dq
 import numpy as np
@@ -12,11 +12,11 @@
 import rich
 from choixe.spooks import Spook
 from loguru import logger
-from rich.progress import track
+from pipelime.tools.progress import pipelime_track
 from schema import Or, Schema
 
 from pipelime.sequences.samples import GroupedSample, Sample, SamplesSequence
-from pipelime.sequences.stages import SampleStage
+from pipelime.sequences.stages import SampleStage, StageRemap
 from pipelime.tools.idgenerators import IdGenerator, IdGeneratorInteger
 
 
@@ -54,7 +54,7 @@ def is_valid_data(self, o: Any) -> bool:
         :return: True if the object matches the schema
         :rtype: bool
         """
-        return self._schema.validate(o)
+        return self._schema.is_valid(o)
 
     def __repr__(self) -> str:
         return str(self._schema)
@@ -138,7 +138,12 @@ class MappableOperation(SequenceOperation):
     kind of operation.
     """
 
-    def __init__(self, num_workers: int = 0, progress_bar: bool = False) -> None:
+    def __init__(
+        self,
+        num_workers: int = 0,
+        progress_bar: bool = False,
+        progress_callback: Optional[Callable[[dict], None]] = None,
+    ) -> None:
         """Constructor for `MappableOperation` subclasses
 
         :param num_workers: the number of multiprocessing workers, set to -1 to spawn as
@@ -146,10 +151,14 @@ def __init__(self, num_workers: int = 0, progress_bar: bool = False) -> None:
         :type num_workers: int, optional
         :param progress_bar: Enable/disable rich progress bar printing on stdout, defaults to False
         :type progress_bar: bool, optional
+        :param progress_callback: callback function to call with progress information,
+        defaults to None
+        :type progress_callback: Optional[Callable[[dict], None]], optional
         """
         super().__init__()
         self._num_workers = num_workers
         self._progress_bar = progress_bar
+        self._progress_callback = progress_callback
         self.pbar_desc = f"{self.__class__.__name__}..."
 
     def input_port(self) -> OperationPort:
@@ -179,12 +188,19 @@ def __call__(self, x: SamplesSequence) -> SamplesSequence:
             pool = multiprocessing.Pool(processes=num_workers)
             new_sequence = pool.imap(self.apply_to_sample, x)
 
-            if self._progress_bar:
-                new_sequence = track(new_sequence, total=total)
+            new_sequence = pipelime_track(
+                new_sequence,
+                total=total,
+                track_callback=self._progress_callback,
+                disable=not self._progress_bar,
+            )
 
         else:
-            if self._progress_bar:
-                x = track(x)
+            x = pipelime_track(
+                x,
+                track_callback=self._progress_callback,
+                disable=not self._progress_bar,
+            )
 
             for sample in x:
                 new_sample = self.apply_to_sample(sample)
@@ -282,9 +298,10 @@ def to_dict(self):
 
 
 class OperationSubsample(SequenceOperation, Spook):
-    """Subsamples an input sequence picking one element every K."""
+    """Subsamples an input sequence picking one element every K
+    or a percentage of all the elements."""
 
-    def __init__(self, factor: Union[int, float]) -> None:
+    def __init__(self, factor: Union[int, float], start: Union[int, float] = 0) -> None:
         """Instantiates an `OperationSubsample` with a given factor.
 
         :param factor: Set to:
@@ -293,9 +310,21 @@ def __init__(self, factor: Union[int, float]) -> None:
         - a `float` to specify a percentage of input elements to preserve
 
         :type factor: Union[int, float]
+        :param start: defaults to 0, set to:
+
+        - an `int` (when `factor` is int) to specify a starting index for the subsampling factor
+        - a `float` (when `factor` is float) to specify a percentage to discard before the elements to preserve
+
+        :type start: Union[int, float], optional
         """
         super().__init__()
         self._factor = factor
+        self._start = start
+
+        assert self._start >= 0
+
+        if isinstance(self._factor, int):
+            assert isinstance(self._start, int)
 
     def input_port(self) -> OperationPort:
         return OperationPort(SamplesSequence)
@@ -308,21 +337,21 @@ def __call__(self, x: SamplesSequence) -> SamplesSequence:
 
         new_samples = x.samples.copy()
         if isinstance(self._factor, int):
-            new_samples = new_samples[
-                :: self._factor
-            ]  # Pick an element each `self._factor` elements
+            # Pick an element each `self._factor` elements
+            new_samples = new_samples[self._start :: self._factor]
         elif isinstance(self._factor, float):
+            new_start = int(len(new_samples) * min(max(self._start, 0), 1.0))
             new_size = int(len(new_samples) * min(max(self._factor, 0), 1.0))
-            new_samples = new_samples[:new_size]
+            new_samples = new_samples[new_start : new_start + new_size]
 
         return SamplesSequence(samples=new_samples)
 
     @classmethod
     def spook_schema(cls) -> dict:
-        return {"factor": Or(float, int)}
+        return {"factor": Or(float, int), "start": Or(float, int)}
 
     def to_dict(self):
-        return {"factor": self._factor}
+        return {"factor": self._factor, "start": self._start}
 
 
 class OperationShuffle(SequenceOperation, Spook):
@@ -753,7 +782,7 @@ def __init__(self, path_or_func: Union[str, Callable]) -> None:
                 assert self._check_sample is not None
             else:
                 logger.warning(
-                    f"The input script function is empty! Operation performs no checks!"
+                    "The input script function is empty! Operation performs no checks!"
                 )
 
             self._serializable = True
@@ -762,7 +791,7 @@ def __init__(self, path_or_func: Union[str, Callable]) -> None:
             self._check_sample = path_or_func
             self._serializable = False
         else:
-            raise NotImplementedError(f"Only str|Callable are allowed as input script ")
+            raise NotImplementedError("Only str|Callable are allowed as input script ")
 
     def input_port(self):
         return OperationPort(SamplesSequence)
@@ -858,3 +887,17 @@ def to_dict(self) -> dict:
     def from_dict(cls, d: dict) -> OperationStage:
         stage = Spook.create(d.pop("stage"))
         return cls(stage, **d)
+
+
+class OperationRemapKeys(OperationStage):
+    def __init__(
+        self, remap: Mapping[str, str], remove_missing: bool = True, **kwargs
+    ) -> None:
+        """Creates an `OperationRemapKeys` from keys str:str map .
+
+        :param remap: mapping from old keys to new keys
+        :type remap: Mapping[str, str]
+        :param remove_missing: if True, missing keys will be removed from the sample
+        :type remove_missing: bool
+        """
+        super().__init__(stage=StageRemap(remap, remove_missing), **kwargs)
diff --git a/pipelime/sequences/pipes.py b/pipelime/sequences/pipes.py
index f877699..11bf655 100644
--- a/pipelime/sequences/pipes.py
+++ b/pipelime/sequences/pipes.py
@@ -289,5 +289,6 @@ def draw_to_file(self, filename: str = None):
             A.draw(fname)
             return A, fname
 
-        except ImportError:
+        except ImportError as e:
+            print(e)
             return None, None
diff --git a/pipelime/sequences/proxies.py b/pipelime/sequences/proxies.py
new file mode 100644
index 0000000..64237d4
--- /dev/null
+++ b/pipelime/sequences/proxies.py
@@ -0,0 +1,266 @@
+from pipelime.sequences.samples import Sample, SamplesSequence
+from pipelime.sequences.stages import SampleStage, StageCompose
+from collections import OrderedDict
+from bisect import bisect_right
+
+from typing import (
+    Optional,
+    Union,
+    Any,
+    MutableMapping,
+    Sequence,
+    Collection,
+    Callable,
+)
+from abc import ABC, abstractmethod
+
+
+class CachedSamplesSequence(SamplesSequence):
+    class SampleCache(ABC):
+        @property
+        def get_fn(self) -> Callable[[int], Sample]:
+            return self._get_fn
+
+        @get_fn.setter
+        def get_fn(self, fn: Callable[[int], Sample]):
+            self._get_fn = fn
+
+        @property
+        def key_list(self) -> Optional[Collection[str]]:
+            return self._key_list
+
+        @key_list.setter
+        def key_list(self, kk: Optional[Collection[str]]):
+            self._key_list = kk
+
+        def eval_sample(self, idx: int) -> Sample:
+            item = self.get_fn(idx)
+            for k in self.key_list if self.key_list is not None else item.keys():
+                _ = item[k]
+            return item
+
+        @abstractmethod
+        def get_sample(self, idx: int) -> Sample:
+            pass
+
+        @abstractmethod
+        def clear_cache(self):
+            pass
+
+    class EndlessCachePolicy(SampleCache):
+        """Store items in cache with no limit."""
+
+        def __init__(self):
+            self.init_cache()
+
+        def get_sample(self, idx: int) -> Sample:
+            item = self._cache_map.get(idx, None)
+            if item is None:
+                item = self.eval_sample(idx)  # NB: use `self` here, NOT `super()`!!!!
+                self.add_sample(idx, item)
+            return item
+
+        def add_sample(self, idx: int, s: Sample):
+            self._cache_map[idx] = s
+
+        def init_cache(self):
+            self._cache_map: MutableMapping[int, Sample] = {}
+
+        def clear_cache(self):
+            self._cache_map.clear()
+
+    class BoundedCachePolicy(EndlessCachePolicy):
+        def __init__(self, max_cache_size: int):
+            """Store only the last max_cache_size items in cache.
+
+            Args:
+                max_cache_size (int): maximum items to store in the FIFO queue.
+            """
+            super().__init__()
+            self._max_cache_size = max_cache_size
+
+        def add_sample(self, idx: int, s: Sample):
+            if self._max_cache_size > 0:
+                self._cache_map[idx] = s
+                self._cache_map.move_to_end(idx)  # ensure s is the last Sample
+                if len(self._cache_map) > self._max_cache_size:
+                    self._cache_map.popitem(False)
+
+        def init_cache(self):
+            self._cache_map: OrderedDict[int, Sample] = OrderedDict()
+
+    class PersistentCachePolicy(BoundedCachePolicy):
+        def __init__(
+            self,
+            cache_dir: Optional[str],
+            compress: Union[bool, int] = False,
+            verbose: bool = False,
+            clear_cache: bool = False,
+            max_buffer_size: int = 0,
+        ):
+            """Dump items to disk.
+
+            Args:
+                cache_dir (Optional[str]): the path to the cache folder; use None to
+                    disable caching.
+                compress (Union[bool, int], optional): whether to compress the stored
+                    data on disk; if an integer [1-9], sets the amount of compression.
+                    Defaults to False.
+                verbose (bool, optional): whether to print debug messages when data is
+                    evaluated. Defaults to False.
+                clear_cache (bool, optional): whether to clear the cache before using
+                    it, otherwise existing values will be used. Defaults to False.
+                max_buffer_size (int, optional): maximum items to store in a FIFO queue
+                    on system memory. Defaults to 0.
+            """
+            super().__init__(max_buffer_size)
+            from joblib import Memory
+
+            self._memory = Memory(
+                location=cache_dir, compress=compress, verbose=1 if verbose else 0
+            )
+            self.eval_sample = self._memory.cache(  # type: ignore
+                self.eval_sample, ignore=["self"]
+            )
+            if clear_cache:
+                self.clear_cache()
+
+        def clear_cache(self):
+            super().clear_cache()
+            self._memory.clear()
+
+    def __init__(
+        self,
+        source: Sequence[Sample],
+        sample_cache: SampleCache,  # don't make a default, jsonargparse would complain
+        forced_keys: Optional[Collection[str]] = None,
+        stage: Optional[SampleStage] = None,
+    ):
+        """A proxy to cache previous outputs.
+
+        Args:
+            source (Sequence[Sample]): the source sequence.
+            sample_cache (SampleCache): the cache of samples.
+            forced_keys (Optional[Collection[str]], optional): always access (and load)
+                these keys from any retrieved Sample before caching; if None, all keys
+                are accessed. Defaults to None.
+            stage (Optional[SampleStage], optional): a Stage to apply to samples.
+                Defaults to None.
+        """
+        super().__init__(source, stage)
+        self._sample_cache = sample_cache
+        self._sample_cache.get_fn = super().__getitem__
+        self._sample_cache.key_list = forced_keys
+
+    def __getitem__(self, idx: int) -> Sample:
+        return self._sample_cache.get_sample(idx)
+
+    def merge(self, other: SamplesSequence) -> SamplesSequence:
+        self._sample_cache.clear_cache()
+        return super().merge(other)
+
+
+class ConcatSamplesSequence(SamplesSequence):
+    def __init__(
+        self, sources: Sequence[SamplesSequence], stage: Optional[SampleStage] = None
+    ):
+        """Concatenate a sequence of SamplesSequence.
+
+        Args:
+            sources (Sequence[SamplesSequence]): the SamplesSequence to concatenate.
+            stage (Optional[SampleStage], optional): a Stage to apply to samples.
+                Defaults to None.
+        """
+        concat_samples: Sequence[Sample] = []
+        self._src_stages: Sequence[SampleStage] = []
+        self._seq_length: Sequence[int] = [0]
+        for sseq in sources:
+            concat_samples += sseq.samples
+            self._src_stages.append(sseq.stage)
+            self._seq_length.append(self._seq_length[-1] + len(sseq))
+
+        super().__init__(concat_samples, stage)
+
+    def __getitem__(self, idx: int) -> Sample:
+        if idx >= len(self):
+            raise IndexError
+        sample = self._samples[idx]
+        seq_id = bisect_right(self._seq_length, idx) - 1
+
+        return self._stage(self._src_stages[seq_id](sample))
+
+
+class FilteredSamplesSequence(SamplesSequence):
+    def __init__(
+        self,
+        source: SamplesSequence,
+        filter_fn: Callable[[Sample], bool],
+        stage: Optional[SampleStage] = None,
+    ):
+        """A filtered view of an input SamplesSequence.
+
+        Args:
+            source (SamplesSequence): the SamplesSequence to filter.
+            filter_fn (Callable[[Sample], bool]): a callable returning True for any
+                valid sample.
+            stage (Optional[SampleStage], optional): a Stage to apply to samples.
+                Defaults to None.
+        """
+        if stage is None:
+            stage = source.stage
+        elif source.stage is not None:
+            stage = StageCompose([source.stage, stage])
+        super().__init__(list(filter(filter_fn, source.samples)), stage)
+
+
+class SortedSamplesSequence(SamplesSequence):
+    def __init__(
+        self,
+        source: SamplesSequence,
+        key_fn: Callable[[Sample], Any],
+        stage: Optional[SampleStage] = None,
+    ):
+        """A sorted view of an input SamplesSequence.
+
+        Args:
+            source (SamplesSequence): the SamplesSequence to sort.
+            key_fn (Callable[[Sample], Any]): the key function to compare Samples. Use
+                `functools.cmp_to_key` to convert a compare function, ie, accepting two
+                arguments, to a key function.
+            stage (Optional[SampleStage], optional): a Stage to apply to samples.
+                Defaults to None.
+        """
+        if stage is None:
+            stage = source.stage
+        elif source.stage is not None:
+            stage = StageCompose([source.stage, stage])
+        super().__init__(sorted(source.samples, key=key_fn), stage)
+
+
+class SlicedSamplesSequence(SamplesSequence):
+    def __init__(
+        self,
+        source: SamplesSequence,
+        start_idx: Optional[int] = None,
+        end_idx: Optional[int] = None,
+        step: Optional[int] = None,
+        stage: Optional[SampleStage] = None,
+    ):
+        """Extract a slice [start_idx:end_idx:step] from the input sequence.
+
+        Args:
+            source (SamplesSequence): the SamplesSequence to slice.
+            start_idx (Optional[int], optional): the first index.
+                Defaults to None (ie, first element).
+            end_idx (Optional[int], optional): the final index.
+                Defaults to None (ie, last element).
+            step_idx (Optional[int], optional): the slice step.
+                Defaults to None (ie, 1).
+            stage (Optional[SampleStage], optional): a Stage to apply to the samples.
+                Defaults to None.
+        """
+        if stage is None:
+            stage = source.stage
+        elif source.stage is not None:
+            stage = StageCompose([source.stage, stage])
+        super().__init__(source.samples[start_idx:end_idx:step], stage)
diff --git a/pipelime/sequences/readers/base.py b/pipelime/sequences/readers/base.py
index a02ccb3..79a0733 100644
--- a/pipelime/sequences/readers/base.py
+++ b/pipelime/sequences/readers/base.py
@@ -9,15 +9,15 @@
 class ReaderTemplate(object):
     def __init__(
         self,
-        extensions_map: Dict[str, any],
-        root_files_keys: Sequence[str],
+        extensions_map: Dict[str, any] = None,
+        root_files_keys: Sequence[str] = None,
         idx_length: int = 5,
     ):
-        self.extensions_map = extensions_map
-        self.root_files_keys = root_files_keys
+        self.extensions_map = extensions_map if extensions_map is not None else {}
+        self.root_files_keys = root_files_keys if root_files_keys else []
         self.idx_length = idx_length
 
-    def __eq__(self, o: object) -> bool:
+    def __eq__(self, o: "ReaderTemplate") -> bool:
         equality = True
         equality = equality and (not DeepDiff(o.extensions_map, self.extensions_map))
         equality = equality and (set(self.root_files_keys) == set(o.root_files_keys))
diff --git a/pipelime/sequences/readers/filesystem.py b/pipelime/sequences/readers/filesystem.py
index a0bfed3..9652e46 100644
--- a/pipelime/sequences/readers/filesystem.py
+++ b/pipelime/sequences/readers/filesystem.py
@@ -1,17 +1,19 @@
 import multiprocessing
 from pathlib import Path
-from typing import Union
+from typing import Dict, Union
 
 from loguru import logger
-from schema import Optional
-
+import networkx as nx
+from typing import Optional
 from pipelime.filesystem.toolkit import FSToolkit
 from pipelime.sequences.readers.base import BaseReader, ReaderTemplate
 from pipelime.sequences.samples import FileSystemSample
+from pipelime.sequences.stages import SampleStage
 
 
 class UnderfolderReader(BaseReader):
     DATA_SUBFOLDER = "data"
+    PRIVATE_KEY_QUALIFIER = "_"
 
     def __init__(
         self,
@@ -19,29 +21,85 @@ def __init__(
         copy_root_files: bool = True,
         lazy_samples: bool = True,
         num_workers: int = 0,
+        enable_plugins: bool = True,
     ) -> None:
+        """Initialize a new underfolder reader
 
-        self._folder = Path(folder)
+        :param folder: Path to the root folder
+        :type folder: str
+        :param copy_root_files: TRUE to propagate root files to samples, defaults to True
+        :type copy_root_files: bool, optional
+        :param lazy_samples: TRUE to load only files skeleton and not data within , defaults to True
+        :type lazy_samples: bool, optional
+        :param num_workers: num workers for multiprocessing data loading , defaults to 0
+        :type num_workers: int, optional
+        :param enable_plugins:  TRUE to enable plugins activation, defaults to True
+        :type enable_plugins: bool, optional
+        :raises FileNotFoundError: [description]
+        """
+
+        self._folder = Path(folder).resolve()
         self._copy_root_files = copy_root_files
         self._datafolder = self._folder / self.DATA_SUBFOLDER
         self._lazy_samples = lazy_samples
         self._num_workers = num_workers
-        assert self._datafolder.exists(), f'No data folder found: "{self._datafolder}"'
+
+        # Checks for valid folder
+        if not self._datafolder.exists():
+            raise FileNotFoundError(
+                f"It seems not a valid Underfolder, folder {self._datafolder} not found"
+            )
 
         # builds tree from subfolder with underscore notation
         self._tree = FSToolkit.tree_from_underscore_notation_files(self._datafolder)
         self._ids = list(sorted(self._tree.keys()))
-        self._root_files = [x for x in Path(self._folder).glob("*") if x.is_file()]
-        self._root_files = [x for x in self._root_files if not x.name.startswith(".")]
+
+        # extract all root files
+        root_files = [x for x in Path(self._folder).glob("*") if x.is_file()]
+
+        # purge hidden files from root files
+        root_files = [x for x in root_files if not x.name.startswith(".")]
+
+        # extract private root files among root files
+        private_root_files = list(
+            filter(
+                lambda x: x.name.startswith(self.PRIVATE_KEY_QUALIFIER),
+                root_files,
+            )
+        )
+
+        # purge private root files from root files
+        root_files = list(
+            filter(
+                lambda x: not x.name.startswith(self.PRIVATE_KEY_QUALIFIER),
+                root_files,
+            )
+        )
+
+        # build public root data
         self._root_data = {}
         self._root_files_keys = set()
-        for f in self._root_files:
+        for f in root_files:
             self._root_files_keys.add(f.stem)
-            self._root_data[f.stem] = str(f)  # FSToolkit.load_data(f)
+            self._root_data[f.stem] = str(f)
 
+        # build private root data
+        self._root_private_files_keys = set()
+        self._root_private_data = {}
+        for f in private_root_files:
+            self._root_private_files_keys.add(
+                f.stem.replace(self.PRIVATE_KEY_QUALIFIER, "", 1)
+            )
+            self._root_private_data[
+                f.stem.replace(self.PRIVATE_KEY_QUALIFIER, "", 1)
+            ] = str(
+                f
+            )  # FSToolkit.load_data(f)
+
+        # Load samples
         if self._num_workers == -1 or self._num_workers > 0:
             if self._lazy_samples:
-                logger.warning(f"Multiprocessing with Lazy Samples are useless!")
+                logger.warning("Multiprocessing with Lazy Samples are useless!")
             pool = multiprocessing.Pool(
                 None if self._num_workers == -1 else self._num_workers
             )
@@ -53,18 +111,55 @@ def __init__(
 
         super().__init__(samples=samples)
 
+        # Spawn plugins
+        self._plugins_map = UnderfolderPlugins.parse(self) if enable_plugins else {}
+
     def _read_sample(self, idx: int):
         data = dict(self._tree[self._ids[idx]])
         if self._copy_root_files:
             data.update(self._root_data)
 
-        return FileSystemSample(
-            data_map=data, lazy=self._lazy_samples, id=self._ids[idx]
-        )
+        purged_id = self.purge_id(self._ids[idx])
+        return FileSystemSample(data_map=data, lazy=self._lazy_samples, id=purged_id)
+
+    @property
+    def plugins_map(self) -> Dict[str, "UnderfolderPlugin"]:
+        return self._plugins_map
+
+    @property
+    def folder(self) -> Path:
+        return self._folder
+
+    @property
+    def lazy_samples(self) -> bool:
+        return self._lazy_samples
+
+    @property
+    def copy_root_files(self) -> bool:
+        return self._copy_root_files
+
+    @property
+    def num_workers(self) -> int:
+        return self._num_workers
+
+    @property
+    def root_data(self) -> dict:
+        return self._root_data
+
+    @property
+    def root_files_keys(self) -> dict:
+        return self._root_files_keys
+
+    @property
+    def root_private_data(self):
+        return self._root_private_data
 
     def is_root_key(self, key: str):
         return key in self._root_files_keys
 
+    def is_root_private_key(self, key: str):
+        return key in self._root_private_files_keys
+
     def get_reader_template(self) -> Union[ReaderTemplate, None]:
         """Retrieves the template of the underfolder reader, i.e. a mapping
         between sample_key/file_extension and a list of root files keys
@@ -80,7 +175,7 @@ def get_reader_template(self) -> Union[ReaderTemplate, None]:
                 raise TypeError(f"Anomalous sample type found: {type(sample)}")
 
             extensions_map = {}
-            idx_length = len(str(sample.id))
+            idx_length = len(self._ids[0])
             for key, filename in sample.filesmap.items():
                 extensions_map[key] = Path(filename).suffix.replace(".", "")
 
@@ -125,3 +220,163 @@ def from_dict(cls, d: dict):
 
     def to_dict(self) -> dict:
         return {"folder": str(self._folder), "copy_root_files": self._copy_root_files}
+
+
+class UnderfolderPlugin:
+    PLUGIN_NAME = ""
+
+    def apply(self, reader: UnderfolderReader, plugin_data: any):
+        """Generic method to apply a plugin to a reader
+
+        :param reader: UnderfolderReader to apply the plugin to
+        :type reader: UnderfolderReader
+        :param plugin_data: Plugin data, can be any data loadable by FSToolkit
+        :type plugin_data: any
+        """
+        pass
+
+
+class UnderfolderLinksPlugin(UnderfolderPlugin):
+    PLUGIN_NAME = "underfolder_links"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._links_graph = None
+
+    @property
+    def links_graph(self) -> nx.Graph:
+        return self._links_graph
+
+    def apply(self, reader: UnderfolderReader, plugin_data: any):
+        self._links_graph = self.build_links_graph(reader.folder)
+        underfolder_links: dict = plugin_data
+        for link in underfolder_links:
+            if Path(link).exists():
+                linked_reader = UnderfolderReader(
+                    folder=link,
+                    copy_root_files=reader.copy_root_files,
+                    lazy_samples=reader.lazy_samples,
+                    num_workers=reader.num_workers,
+                )
+                if len(linked_reader) != len(reader.samples):
+                    raise ValueError(
+                        f"Linked reader has a different number of samples"
+                        f"({len(linked_reader)}) than the current reader ({len(reader.samples)})"
+                    )
+
+                reader.root_data.update(linked_reader.root_data)
+                reader.root_files_keys.update(linked_reader.root_files_keys)
+                reader.samples = [
+                    x.merge(y) for x, y in zip(linked_reader, reader.samples)
+                ]
+
+    @classmethod
+    def build_links_graph(cls, source_folder: str, graph: Optional[nx.DiGraph] = None):
+        source_reader = UnderfolderReader(
+            folder=source_folder, lazy_samples=True, enable_plugins=False
+        )
+        if graph is None:
+            graph = nx.DiGraph()
+
+        if cls.PLUGIN_NAME in source_reader.root_private_data:
+            links = FSToolkit.load_data(
+                source_reader.root_private_data[cls.PLUGIN_NAME]
+            )
+            for link in links:
+                graph.add_edge(source_folder, link)
+                cls.build_links_graph(link, graph)
+
+        loops = []
+        try:
+            loops = nx.find_cycle(graph)
+        except Exception:
+            pass
+        if len(loops) > 0:
+            raise RuntimeError("Cycle detected in the Underfolder links graph")
+        return graph
+
+    @classmethod
+    def link_underfolders(cls, source_folder: str, target_folder: str):
+        """Links two Underfolder adding target_folder to links in source_folder
+
+        :param source_folder: Underfolder folder where to add links
+        :type source_folder: str
+        :param target_folder: Underfolder folder to add
+        :type target_folder: str
+        """
+
+        source_folder = Path(source_folder)
+        source_reader = UnderfolderReader(folder=source_folder, lazy_samples=True)
+        target_reader = UnderfolderReader(folder=target_folder, lazy_samples=True)
+        if len(source_reader) != len(target_reader):
+            raise RuntimeError(
+                "Cannot link underfolders with different number of samples"
+            )
+
+        # Builds private key filename
+        key = UnderfolderLinksPlugin.PLUGIN_NAME
+        private_key_file = (
+            source_folder / f"{UnderfolderReader.PRIVATE_KEY_QUALIFIER}{key}.yml"
+        )
+
+        # Create private key file if not present
+        if not source_reader.is_root_private_key(key):
+            FSToolkit.store_data(private_key_file, [])
+
+        # Loads private key data
+        prev_links = FSToolkit.load_data(private_key_file)
+
+        # Update private key data
+        prev_links.append(target_folder)
+        FSToolkit.store_data(private_key_file, prev_links)
+
+
+class UnderfolderStagePlugin(UnderfolderPlugin):
+    PLUGIN_NAME = "underfolder_stage"
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def apply(self, reader: UnderfolderReader, plugin_data: any):
+        reader.stage = SampleStage.create(plugin_data)
+
+    @classmethod
+    def set_stages(cls, source_folder: str, stage: SampleStage):
+        """Sets the Stage of the entire Underfolder
+
+        :param source_folder: Underfolder folder
+        :type source_folder: str
+        :param stage: Sample stage to set
+        :type stage: SampleStage
+        """
+
+        source_folder = Path(source_folder)
+        UnderfolderReader(folder=source_folder, lazy_samples=True)
+
+        # Builds private key filename
+        key = UnderfolderStagePlugin.PLUGIN_NAME
+        private_key_file = (
+            source_folder / f"{UnderfolderReader.PRIVATE_KEY_QUALIFIER}{key}.yml"
+        )
+
+        # Create private key file if not present
+        FSToolkit.store_data(private_key_file, stage.serialize())
+
+
+class UnderfolderPlugins:
+
+    PLUGINS_MAP = {
+        UnderfolderLinksPlugin.PLUGIN_NAME: UnderfolderLinksPlugin,
+        UnderfolderStagePlugin.PLUGIN_NAME: UnderfolderStagePlugin,
+    }
+
+    @classmethod
+    def parse(cls, reader: UnderfolderReader) -> Dict[str, UnderfolderPlugin]:
+        plugins_map = {}
+        for key in reader.root_private_data:
+            if key in cls.PLUGINS_MAP:
+                plugin = cls.PLUGINS_MAP[key]()
+                plugin.apply(reader, FSToolkit.load_data(reader.root_private_data[key]))
+                plugins_map[key] = plugin
+
+        return plugins_map
diff --git a/pipelime/sequences/readers/h5.py b/pipelime/sequences/readers/h5.py
index 8b54630..1ac5a97 100644
--- a/pipelime/sequences/readers/h5.py
+++ b/pipelime/sequences/readers/h5.py
@@ -1,11 +1,11 @@
 from pathlib import Path
-from typing import Dict, Hashable, Sequence, Union
+from typing import Hashable, Sequence, Union
 
 import h5py
 from schema import Optional
 
 from pipelime.h5.toolkit import H5Database, H5ToolKit
-from pipelime.sequences.readers.base import BaseReader
+from pipelime.sequences.readers.base import BaseReader, ReaderTemplate
 from pipelime.sequences.samples import MemoryItem, MetaItem, Sample
 
 
@@ -18,27 +18,67 @@ def source(self) -> any:
         return self._item
 
 
+class MultiGroup:
+    """Manages H5PY groups fusion, if only one group is present it should behave like
+    a normal H5py group
+    """
+
+    def __init__(self, groups: Sequence[h5py.Group]) -> None:
+        self._groups = groups
+
+    def keys(self) -> Sequence[Hashable]:
+        keys = set()
+        for group in self._groups:
+            keys.update(group.keys())
+        return list(keys)
+
+    def __contains__(self, key: Hashable) -> bool:
+        for group in self._groups:
+            if key in group:
+                return True
+        return False
+
+    def __getitem__(self, key: Hashable) -> h5py.Group:
+        for group in reversed(self._groups):
+            if key in group:
+                return group[key]
+        raise KeyError(key)
+
+    def get(self, key: Hashable, getlink: bool = True) -> Union[h5py.Group, any]:
+        for group in reversed(self._groups):
+            if key in group:
+                return group.get(key, getlink=getlink)
+        raise KeyError(key)
+
+    def merge(self, other: "MultiGroup") -> "MultiGroup":
+        return MultiGroup(self._groups + other._groups)
+
+
 class H5Sample(Sample):
     def __init__(
         self,
-        group: h5py.Group,
+        group: Union[h5py.Group, MultiGroup],
         lazy: bool = True,
         copy_global_items: bool = True,
         id: Hashable = None,
     ):
         """Creates a H5Sample based on a key/filename map
 
-        :param group: h5py Group
-        :type group: h5py.Group
+        :param group: h5py Group or MultiGroup
+        :type group: Union[h5py.Group, MultiGroup]
         :param lazy: FALSE to preload data (slow), defaults to False
         :type lazy: bool, optional
-        :param copy_global_items: TRUE to propagate global item to samples, defaults to True
+        :param copy_global_items: TRUE to propagate global item to samples, defaults to
+            True
         :type copy_global_items: bool, optional
         :param id: hashable value used as id
         :type id: Hashable, optional
         """
         super().__init__(id=id)
-        self._group = group
+        self._group = MultiGroup([group]) if isinstance(group, h5py.Group) else group
+        self._cached = {}
+        self._lazy = lazy
+        self._copy_global_items = copy_global_items
         self._copy_global_items = copy_global_items
         self._keys = set()
         for key in self._group.keys():
@@ -51,7 +91,7 @@ def __init__(
         self._cached = {}
         if not lazy:
             for k in self.keys():
-                d = self[k]
+                self.get(k)
 
     def is_cached(self, key) -> bool:
         return key in self._cached
@@ -85,6 +125,14 @@ def __iter__(self):
     def __len__(self):
         return len(self._keys)
 
+    def merge(self, other: "H5Sample") -> "H5Sample":
+        merged_groups = self._group.merge(other._group)
+        merged_cache = self._cached.copy()
+        merged_cache.update(other._cached)
+        newsample = H5Sample(merged_groups, id=self.id)
+        newsample._cached = merged_cache
+        return newsample
+
     def copy(self):
         newsample = H5Sample(self._group, id=self.id)
         newsample._cached = self._cached.copy()
@@ -109,18 +157,6 @@ def flush(self):
             del self._cached[k]
 
 
-class H5ReaderTemplate(object):
-    def __init__(
-        self,
-        extensions_map: Dict[str, any],
-        root_files_keys: Sequence[str],
-        idx_length: int = 5,
-    ):
-        self.extensions_map = extensions_map
-        self.root_files_keys = root_files_keys
-        self.idx_length = idx_length
-
-
 class H5Reader(BaseReader):
     DATA_SUBFOLDER = "data"
 
@@ -132,7 +168,6 @@ def __init__(
         self._copy_root_files = copy_root_files
         self._lazy_samples = lazy_samples
 
-        # builds tree from subfolder with underscore notation
         self._h5database = H5Database(filename=self._filename, readonly=True)
         self._h5database.open()
 
@@ -164,13 +199,13 @@ def __init__(
     def is_root_key(self, key: str):
         return key in self._root_files_keys
 
-    def get_reader_template(self) -> Union[H5ReaderTemplate, None]:
+    def get_reader_template(self) -> Union[ReaderTemplate, None]:
         """Retrieves the template of the h5 reader, i.e. a mapping
         between sample_key/file_extension and a list of root files keys
 
         :raises TypeError: If first sample is not a H5Sample
-        :return: None if dataset is empty, otherwise an H5ReaderTemplate
-        :rtype: Union[H5ReaderTemplate, None]
+        :return: None if dataset is empty, otherwise a ReaderTemplate
+        :rtype: Union[ReaderTemplate, None]
         """
 
         if len(self) > 0:
@@ -184,7 +219,7 @@ def get_reader_template(self) -> Union[H5ReaderTemplate, None]:
                 if sample.get_encoding(key) is not None:
                     extensions_map[key] = sample.get_encoding(key)
 
-            return H5ReaderTemplate(
+            return ReaderTemplate(
                 extensions_map=extensions_map,
                 root_files_keys=list(self._root_files_keys),
                 idx_length=idx_length,
diff --git a/pipelime/sequences/readers/summary.py b/pipelime/sequences/readers/summary.py
index 5d64a07..6834439 100644
--- a/pipelime/sequences/readers/summary.py
+++ b/pipelime/sequences/readers/summary.py
@@ -103,13 +103,16 @@ def _build_type_info(cls, obj: Any) -> TypeInfo:
 
     @classmethod
     def build_item_info(cls, reader: BaseReader, key: str, k: int) -> ItemInfo:
-        """Instantiates an ItemInfo object from the first k items whose key match the input key.
+        """Instantiates an ItemInfo object from the first k items whose key match the
+        input key.
 
         :param reader: Input pipelime reader
         :type reader: BaseReader
         :param key: The query key to inspect
         :type key: str
-        :param k: The first k samples to inspect, this is done to avoid reading the entire dataset and speedup the summary creation. Set `k<1` to inspect every sample.
+        :param k: The first k samples to inspect, this is done to avoid reading the
+        entire dataset and speedup the summary creation. Set `k<1` to inspect every
+        sample.
         :type k: int
         :return: An ItemInfo object
         :rtype: ItemInfo
@@ -166,7 +169,9 @@ def build_all(cls, reader: BaseReader, k: int) -> Sequence[ItemInfo]:
 
         :param reader: Input pipelime BaseReader
         :type reader: BaseReader
-        :param k: The first k samples to inspect, this is done to avoid reading the entire dataset and speedup the summary creation. Set to k < 1 to inspect every sample.
+        :param k: The first k samples to inspect, this is done to avoid reading the
+        entire dataset and speedup the summary creation. Set to k < 1 to inspect
+        every sample.
         :type k: int
         :return: The list of all ItemInfos
         :rtype: Sequence[ItemInfo]
diff --git a/pipelime/sequences/samples.py b/pipelime/sequences/samples.py
index 1a4a7d8..47d871b 100644
--- a/pipelime/sequences/samples.py
+++ b/pipelime/sequences/samples.py
@@ -1,10 +1,9 @@
 import uuid
 from abc import abstractmethod
-from collections.abc import MutableMapping
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Hashable, Sequence
-
+from typing import Any, Mapping, Hashable, Sequence, MutableMapping, Union
+import functools
 from pipelime.filesystem.toolkit import FSToolkit
 
 
@@ -14,7 +13,7 @@ def __init__(self) -> None:
         pass
 
     @abstractmethod
-    def source(self) -> any:
+    def source(self) -> Any:
         pass
 
 
@@ -22,16 +21,16 @@ class MemoryItem(MetaItem):
     def __init__(self) -> None:
         super().__init__()
 
-    def source(self) -> any:
+    def source(self) -> Any:
         return None
 
 
-class FilesystemItem(MetaItem):
+class FileSystemItem(MetaItem):
     def __init__(self, path: str) -> None:
         super().__init__()
         self._path = Path(path)
 
-    def source(self) -> any:
+    def source(self) -> Path:
         return self._path
 
 
@@ -52,11 +51,15 @@ def rename(self, old_key: str, new_key: str):
         pass
 
     @abstractmethod
-    def copy(self):
+    def copy(self) -> "Sample":
         pass
 
     @abstractmethod
-    def metaitem(self, key: any) -> MetaItem:
+    def metaitem(self, key: Any) -> MetaItem:
+        pass
+
+    @abstractmethod
+    def merge(self, other: "Sample") -> "Sample":
         pass
 
     @property
@@ -115,6 +118,14 @@ def __len__(self):
     def __repr__(self) -> str:
         return str(self._samples)
 
+    def merge(self, other: "GroupedSample") -> "GroupedSample":
+        samples = self._samples
+        others_samples = other._samples
+        if len(samples) != len(others_samples):
+            raise ValueError("Cannot merge samples with different lengths")
+        merged_samples = [x.merge(y) for x, y in zip(samples, others_samples)]
+        return GroupedSample(samples=merged_samples)
+
     def copy(self):
         return GroupedSample(samples=self._samples)
 
@@ -122,7 +133,7 @@ def rename(self, old_key: str, new_key: str):
         for x in self._samples:
             x.rename(old_key=old_key, new_key=new_key)
 
-    def metaitem(self, key: any):
+    def metaitem(self, key: Any):
         for x in self._samples:
             return x.metaitem(key)
         return None
@@ -158,6 +169,11 @@ def __len__(self):
     def __repr__(self) -> str:
         return str(self._data)
 
+    def merge(self, other: "PlainSample") -> "PlainSample":
+        new_data = self._data.copy()
+        new_data.update(other._data.copy())
+        return PlainSample(data=new_data, id=self.id)
+
     def copy(self):
         return PlainSample(data=self._data.copy(), id=self.id)
 
@@ -166,12 +182,14 @@ def rename(self, old_key: str, new_key: str):
             self._data[new_key] = self._data[old_key]
             del self._data[old_key]
 
-    def metaitem(self, key: any):
+    def metaitem(self, key: Any):
         return MemoryItem()
 
 
 class FileSystemSample(Sample):
-    def __init__(self, data_map: dict, lazy: bool = True, id: Hashable = None):
+    def __init__(
+        self, data_map: MutableMapping[str, str], lazy: bool = True, id: Hashable = None
+    ):
         """Creates a FileSystemSample based on a key/filename map
 
         :param data_map: key/filename map
@@ -182,11 +200,11 @@ def __init__(self, data_map: dict, lazy: bool = True, id: Hashable = None):
         :type id: Hashable, optional
         """
         super().__init__(id=id)
-        self._filesmap = data_map
+        self._filesmap: MutableMapping[str, str] = data_map
         self._cached = {}
         if not lazy:
             for k in self.keys():
-                d = self[k]
+                self.get(k)
 
     @property
     def filesmap(self):
@@ -207,19 +225,28 @@ def __setitem__(self, key, value):
         self._cached[key] = value
 
     def __delitem__(self, key):
-        if key in self._cached:
-            del self._cached[key]
-        if key in self._filesmap:
-            del self._filesmap[key]
+        self._cached.pop(key, None)
+        self._filesmap.pop(key, None)
 
     def __iter__(self):
         return iter(set.union(set(self._filesmap.keys()), set(self._cached.keys())))
 
     def __len__(self):
-        return len(self._filesmap)
+        return len(set.union(set(self._filesmap.keys()), set(self._cached.keys())))
+
+    def merge(self, other: "FileSystemSample") -> "FileSystemSample":
+        new_filesmap = self._filesmap.copy()
+        new_cache = self._cached.copy()
+
+        new_filesmap.update(other._filesmap)
+        new_cache.update(other._cached)
+
+        newsample = FileSystemSample(new_filesmap, id=self.id)
+        newsample._cached = new_cache
+        return newsample
 
     def copy(self):
-        newsample = FileSystemSample(self._filesmap, id=self.id)
+        newsample = FileSystemSample(self._filesmap.copy(), id=self.id)
         newsample._cached = self._cached.copy()
         return newsample
 
@@ -228,10 +255,12 @@ def rename(self, old_key: str, new_key: str):
             self._filesmap[new_key] = self._filesmap.pop(old_key)
             if old_key in self._cached:
                 self._cached[new_key] = self._cached.pop(old_key)
+        if new_key not in self._cached and old_key in self._cached:
+            self._cached[new_key] = self._cached.pop(old_key)
 
-    def metaitem(self, key: any):
+    def metaitem(self, key: Any):
         if key in self._filesmap:
-            return FilesystemItem(self._filesmap[key])
+            return FileSystemItem(self._filesmap[key])
         else:
             return MemoryItem()
 
@@ -245,7 +274,7 @@ def flush(self, keys: Sequence[str] = None):
         for k in keys:
             del self._cached[k]
 
-    def update(self, other: Dict[str, Any]) -> None:
+    def update(self, other: Mapping[str, Any]) -> None:
         if isinstance(other, FileSystemSample):
             self.filesmap.update(other.filesmap)
             for k in other.keys():
@@ -256,21 +285,58 @@ def update(self, other: Dict[str, Any]) -> None:
 
 
 class SamplesSequence(Sequence):
-    def __init__(self, samples: Sequence[Sample]):
+    # ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️
+    # Removed type hinting for stage argument, which resulted in circular import
+    # between pipelime.sequences.stages and pipelime.sequences.samples modules
+
+    # Refactoring is needed to decouple these classes or to reorganize these modules
+    # to avoid this circular dependency!
+    # ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️
+
+    def __init__(self, samples: Sequence[Sample], stage=None):
+        """Constructor for `SamplesSequence`
+
+        :param samples: A sequence of samples that this SampleSequence will contain
+        :type samples: Sequence[Sample]
+        :param stage: A stage to apply to each sample of the sequence when the
+        __getitem__ is called, if set to `None` no stage is applied, defaults to None
+        :type stage: Optional[SampleStage], optional
+        """
         self._samples = samples
 
+        # This import here is due to circular dependency 💀💀💀 !!
+        from pipelime.sequences.stages import StageIdentity
+
+        self._stage = StageIdentity() if stage is None else stage
+
     @property
     def samples(self):
         return self._samples
 
+    @property
+    def stage(self):
+        return self._stage
+
+    @stage.setter
+    def stage(self, stage):
+        # This import here is due to circular dependency 💀💀💀 !!
+        from pipelime.sequences.stages import SampleStage
+
+        assert isinstance(stage, SampleStage)
+        self._stage = stage
+
+    @samples.setter
+    def samples(self, samples: Sequence[Sample]):
+        self._samples = samples
+
     def __len__(self) -> int:
         return len(self._samples)
 
-    def __getitem__(self, idx: int):
+    def __getitem__(self, idx: int) -> Sample:
         if idx >= len(self):
             raise IndexError
 
-        return self._samples[idx]
+        return self._stage(self._samples[idx])
 
     def is_normalized(self) -> bool:
         """Checks for normalization i.e. each sample has to contain same keys.
@@ -297,3 +363,27 @@ def best_zfill(self) -> int:
         :rtype: int
         """
         return len(str(len(self)))
+
+    @classmethod
+    def purge_id(cls, id: Hashable) -> Union[int, str]:
+        try:
+            return int(id)  # type: ignore
+        except Exception:
+            return str(id)
+
+    def merge(self, other: "SamplesSequence") -> "SamplesSequence":
+        new_samples = [x.merge(y) for x, y in zip(self, other)]
+        return SamplesSequence(samples=new_samples)
+
+    @classmethod
+    def merge_sequences(
+        cls, sequences: Sequence["SamplesSequence"]
+    ) -> "SamplesSequence":
+        """Merges multiple sequences into one
+
+        :param sequences: sequences to merge
+        :type sequences: SamplesSequence
+        :return: merged sequence
+        :rtype: SamplesSequence
+        """
+        return functools.reduce(lambda x, y: x.merge(y), sequences)
diff --git a/pipelime/sequences/stages.py b/pipelime/sequences/stages.py
index a0c6c3e..88e5eea 100644
--- a/pipelime/sequences/stages.py
+++ b/pipelime/sequences/stages.py
@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-from typing import Sequence
-
+from dataclasses import dataclass, field
+from typing import Union, Collection, Sequence, Mapping, Tuple, Optional, Any, List
+import io
+from pathlib import Path
 import albumentations as A
 from choixe.spooks import Spook
-
-from pipelime.sequences.samples import Sample
+from pipelime.sequences.samples import Sample, FileSystemSample
+from pipelime.filesystem.toolkit import FSToolkit
 
 
 class SampleStage(ABC, Spook):
@@ -44,17 +46,18 @@ class StageIdentity(SampleStage):
     def __init__(self):
         super().__init__()
 
-    def __call__(self, x: any) -> any:
+    def __call__(self, x: Sample) -> Sample:
         return x
 
 
 class StageRemap(SampleStage):
-    def __init__(self, remap: dict, remove_missing: bool = True):
+    def __init__(self, remap: Mapping[str, str], remove_missing: bool = True):
         """Remaps keys in sample
 
         :param remap: old_key:new_key dictionary remap
-        :type remap: dict
-        :param remove_missing: if TRUE missing keys in remap will be removed in the output sample, defaults to True
+        :type remap: Mapping[str, str]
+        :param remove_missing: if TRUE missing keys in remap will be removed in the
+            output sample, defaults to True
         :type remove_missing: bool, optional
         """
         super().__init__()
@@ -84,16 +87,16 @@ def to_dict(self):
 
 
 class StageKeysFilter(SampleStage):
-    def __init__(self, keys: list, negate: bool = False):
+    def __init__(self, key_list: Collection[str], negate: bool = False):
         """Filter sample keys
 
-        :param keys: list of keys to preserve
-        :type keys: list
+        :param key_list: list of keys to preserve
+        :type key_list: Collection[str]
         :param negate: TRUE to delete input keys, FALSE delete all but keys
         :type negate: bool
         """
         super().__init__()
-        self._keys = keys
+        self._keys = key_list
         self._negate = negate
 
     def __call__(self, x: Sample) -> Sample:
@@ -107,38 +110,50 @@ def __call__(self, x: Sample) -> Sample:
 
     @classmethod
     def spook_schema(cls) -> dict:
-        return {"keys": list, "negate": bool}
+        return {"key_list": list, "negate": bool}
 
     def to_dict(self):
-        return {"keys": self._keys, "negate": self._negate}
+        return {"key_list": self._keys, "negate": self._negate}
 
 
 class StageAugmentations(SampleStage):
-    def __init__(self, transform_cfg: dict, targets: dict):
+    def __init__(self, transform_cfg: Union[dict, str], targets: dict):
         super().__init__()
-        self._transform_cfg = transform_cfg
+
+        self._transform: A.Compose = (
+            A.load(
+                transform_cfg,
+                data_format="json" if transform_cfg.endswith("json") else "yaml",
+            )
+            if isinstance(transform_cfg, str)
+            else A.from_dict(transform_cfg)
+        )
+
+        # get an up-to-date description
+        self._transform_cfg = A.to_dict(self._transform)
+
         self._targets = targets
-        self._transform: A.Compose = A.from_dict(transform_cfg)
         self._transform.add_targets(self._purge_targets(self._targets))
 
     def _purge_targets(self, targets: dict):
-        # TODO: could it be wrong if targets also contains 'image' or 'mask' (aka default target)?
+        # TODO: could it be wrong if targets also contains
+        # 'image' or 'mask' (aka default target)?
         return targets
 
     def __call__(self, x: Sample) -> Sample:
 
         try:
-            out = x.copy()
+            x = x.copy()
             to_transform = {}
-            for k, data in x.items():
+            for k in x.keys():
                 if k in self._targets:
-                    to_transform[k] = data
+                    to_transform[k] = x[k]
 
             _transformed = self._transform(**to_transform)
             for k in self._targets.keys():
-                out[k] = _transformed[k]
+                x[k] = _transformed[k]
 
-            return out
+            return x
         except Exception as e:
             raise Exception(f"Stage[{self.__class__.__name__}] -> {e}")
 
@@ -154,3 +169,213 @@ def from_dict(cls, d: dict):
 
     def to_dict(self):
         return {"transform_cfg": self._transform_cfg, "targets": self._targets}
+
+
+@dataclass
+class RemoteParams:
+    """Remote parameters.
+
+    :param scheme: the protocol used, eg, 'file', 's3' etc.
+    :type scheme: str
+    :param netloc: the ip address and port or 'localhost'.
+    :type netloc: str
+    :param base_path: the base path for all uploads, eg, the name of the bucket.
+    :type base_path: str
+    :param init_args: keyword arguments forwarded to the remote initializer.
+        Defaults to {}.
+    :type init_args: Mapping[str, str], optional
+    """
+
+    scheme: str
+    netloc: str
+    base_path: str
+    init_args: Mapping[str, Any] = field(default_factory=dict)
+    url: str = field(init=False)
+
+    def __post_init__(self):
+        from urllib.parse import ParseResult
+
+        ip_and_port = self.netloc.split(":", 1)
+        ip_addr = ip_and_port[0]
+        port = ip_and_port[1] if len(ip_and_port) > 1 else ""
+        if not ip_addr or ip_addr == "localhost":
+            ip_addr = "127.0.0.1"
+        ip_and_port = f"{ip_addr}:{port}" if port else ip_addr
+
+        self.url = ParseResult(
+            scheme=self.scheme,
+            netloc=ip_and_port,
+            path=self.base_path,
+            params="",
+            query="",
+            fragment="",
+        ).geturl()
+
+
+class StageUploadToRemote(SampleStage):
+    def __init__(
+        self,
+        remotes: Union[RemoteParams, Sequence[RemoteParams]],
+        key_ext_map: Mapping[str, Optional[str]],
+    ):
+        """Upload sample data to one or more remotes. The uploaded items are then
+        managed through a RemoteMetaItem.
+
+        :param remotes: the list of remotes.
+        :type remotes: Union[RemoteParams, Sequence[RemoteParams]]
+        :param key_ext_map: the item keys to upload and the
+            associated file extension to use. If no extension is given, the item is
+            pickled.
+        :type key_list: Mapping[str, Optional[str]]
+        """
+        from pipelime.filesystem.remotes import BaseRemote, create_remote
+
+        def _check_ext(e: Optional[str]) -> str:
+            if isinstance(e, str) and len(e) > 0:
+                return ("." + e) if e[0] != "." else e
+            return ".pickle"
+
+        self._key_ext_map = {k: _check_ext(e) for k, e in key_ext_map.items()}
+        self._remotes: Sequence[Tuple[BaseRemote, str, str]] = []
+        for rm in remotes if isinstance(remotes, Sequence) else [remotes]:
+            rm_instance = create_remote(rm.scheme, rm.netloc, **rm.init_args)
+            if rm_instance is not None:
+                self._remotes.append(
+                    (
+                        rm_instance,
+                        rm.base_path,
+                        rm.url,
+                    )
+                )
+
+    def _fssample_shortcut(self, x: Sample, k: str, ext: str) -> Tuple[List[str], bool]:
+        url_list = []
+        uploaded = False
+        if isinstance(x, FileSystemSample) and not x.is_cached(k):
+            if FSToolkit.is_remote_file(x.filesmap[k]):
+                # read current remote list
+                url_list = FSToolkit.load_remote_list(x.filesmap[k])
+            elif ext == "".join(Path(x.filesmap[k]).suffixes):
+                # just upload the file
+                for rm, base_path, _ in self._remotes:
+                    target = rm.upload_file(x.filesmap[k], base_path)
+                    if target is not None:
+                        url_list.append(target)
+                uploaded = True
+        return url_list, uploaded
+
+    def _upload_to_remotes(self, url_list, sample, key, ext):
+        from urllib.parse import urlparse
+
+        # parsed url to easily find duplicates (see below)
+        parsed_url_list = [urlparse(u) for u in url_list]
+        parsed_url_list = [
+            RemoteParams(
+                scheme=u.scheme,
+                netloc=u.netloc,
+                base_path=Path(u.path[1:]).parent.as_posix(),
+            ).url
+            for u in parsed_url_list
+        ]
+
+        data_stream = None
+        data_size = 0
+        for rm, base_path, rm_url in self._remotes:
+            # first check if the remote is already listed
+            if rm_url not in parsed_url_list:
+                if data_stream is None:
+                    item_data = None
+                    if not url_list:
+                        # the item is not a 'remote' file
+                        # however, it can still be a remote list if not bound to a file
+                        item_data = sample[key]
+                        if FSToolkit.is_remote_list(item_data):
+                            url_list = item_data
+                            item_data = None
+
+                    # this list come from a remote file or from a 'memory' item
+                    if url_list:
+                        # manually get the item from the remote
+                        item_data = FSToolkit.load_remote_data(url_list)
+                        if FSToolkit.is_remote_list(item_data):
+                            raise RuntimeError(
+                                f"Stage[{self.__class__.__name__}] "
+                                "recursive remotes not allowed!"
+                            )
+
+                    # store the item as binary blob
+                    data_stream = io.BytesIO()
+                    FSToolkit.store_data_to_stream(data_stream, ext, item_data)
+
+                    data_stream.seek(0, io.SEEK_END)
+                    data_size = data_stream.tell()
+                    data_stream.seek(0, io.SEEK_SET)
+
+                target = rm.upload_stream(data_stream, data_size, base_path, ext)
+                if target is not None:
+                    url_list.append(target)
+                    parsed_url_list.append(rm_url)
+
+        return url_list
+
+    def __call__(self, x: Sample) -> Sample:
+        x = x.copy()
+        for k, ext in self._key_ext_map.items():
+            if k in x:
+                url_list, uploaded = self._fssample_shortcut(x, k, ext)
+                if not uploaded:
+                    url_list = self._upload_to_remotes(url_list, x, k, ext)
+
+                # replace item value with the list of remote urls
+                x[k] = url_list
+        return x
+
+
+class StageRemoveRemote(SampleStage):
+    def __init__(
+        self,
+        remotes: Union[RemoteParams, Sequence[RemoteParams]],
+        key_list: Sequence[str],
+    ):
+        """Remove a remote in remote lists. To avoid an unintended data loss, the actual
+        file is not deleted on the remote.
+
+        :param remotes: the remote to remove.
+        :type remotes: Union[RemoteParams, Sequence[RemoteParams]]
+        :param key_list: the target item keys.
+        :type key_list: Sequence[str]
+        """
+        if not isinstance(remotes, Sequence):
+            remotes = [remotes]
+        self._remotes = [rm.url for rm in remotes]
+        self._key_list = key_list
+
+    def _as_remote_url(self, url) -> str:
+        from urllib.parse import urlparse
+
+        url = urlparse(url)
+        return RemoteParams(
+            scheme=url.scheme,
+            netloc=url.netloc,
+            base_path=Path(url.path[1:]).parent.as_posix(),
+        ).url
+
+    def __call__(self, x: Sample) -> Sample:
+        x = x.copy()
+        for k in self._key_list:
+            url_list = []
+            if (
+                isinstance(x, FileSystemSample)
+                and not x.is_cached(k)
+                and FSToolkit.is_remote_file(x.filesmap[k])
+            ):
+                url_list = FSToolkit.load_remote_list(x.filesmap[k])
+            else:
+                data = x[k]
+                if FSToolkit.is_remote_list(data):
+                    url_list = data
+            if url_list:
+                x[k] = [
+                    u for u in url_list if self._as_remote_url(u) not in self._remotes
+                ]
+        return x
diff --git a/pipelime/sequences/streams/__init__.py b/pipelime/sequences/streams/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pipelime/sequences/streams/base.py b/pipelime/sequences/streams/base.py
new file mode 100644
index 0000000..0a6ae3f
--- /dev/null
+++ b/pipelime/sequences/streams/base.py
@@ -0,0 +1,237 @@
+from abc import abstractmethod
+from typing import Dict, Sequence, Tuple
+import imageio
+from pipelime.sequences.readers.base import BaseReader
+from pipelime.sequences.writers.base import BaseWriter
+from pipelime.sequences.samples import Sample
+import numpy as np
+import cv2
+import io
+
+
+class ItemConverter:
+    IMAGE_FORMATS = ["jpg", "jpeg", "png"]
+    MATRIX_FORMATS = ["matrix"]
+    DICT_FORMATS = ["dict"]
+    TEXT_FORMATS = ["txt"]
+
+    @classmethod
+    def data_to_item(cls, data: any, format: str) -> any:
+        """Convert data to item.
+
+        :param data: data to convert
+        :type data: any
+        :param format: format of data
+        :type format: str
+        :raises NotImplementedError: if format is not supported
+        :raises ValueError: if data is not supported
+        :return: converted data
+        :rtype: any
+        """
+        if format in cls.IMAGE_FORMATS:
+            return imageio.imread(data.getbuffer())
+        elif format in cls.MATRIX_FORMATS:
+            return np.array(data["data"])
+        elif format in cls.DICT_FORMATS:
+            return data
+        else:
+            raise ValueError(f"Unknown format {format}")
+
+    @classmethod
+    def item_to_image_data(cls, item: np.ndarray, format: str) -> io.BytesIO:
+        """Converts a numpy array to a BytesIO object.
+
+        :param item: The numpy array to convert.
+        :type item: np.ndarray
+        :param format: The format of the image.
+        :type format: str
+        :return: The BytesIO object.
+        :rtype: io.BytesIO
+        """
+        item = cv2.cvtColor(item, cv2.COLOR_BGR2RGB)
+        _, im_png = cv2.imencode(f".{format}", item)
+        return io.BytesIO(im_png.tobytes())
+
+    @classmethod
+    def item_to_matrix_data(cls, item: np.ndarray, format: str) -> dict:
+        """Converts a numpy array to a dictionary {data:[...]}
+
+        :param item: The numpy array to convert.
+        :type item: np.ndarray
+        :param format: The format of the numeric data.
+        :type format: str
+        :return: The dictionary {data:[...]}
+        :rtype: dict
+        """
+        return {"data": item.tolist()}
+
+    @classmethod
+    def item_to_dict_data(cls, item: dict, format: str) -> dict:
+        """Converts a dictionary to a dictionary ?? COPILOT fault :)
+
+        :param item: The dictionary to convert.
+        :type item: dict
+        :param format: The format of the specific dictionary.
+        :type format: str
+        :return: the output dictionary
+        :rtype: dict
+        """
+        return item
+
+    @classmethod
+    def item_to_data(cls, item: any, format: str) -> any:
+        """Converts an item to a specific format.
+
+        :param item: The item to convert.
+        :type item: any
+        :param format: The format of the specific item.
+        :type format: str
+        :raises ValueError: If the format is not supported.
+        :return: The converted item.
+        :rtype: any
+        """
+        if format in cls.IMAGE_FORMATS:
+            return cls.item_to_image_data(item, format)
+        elif format in cls.MATRIX_FORMATS:
+            return cls.item_to_matrix_data(item, format)
+        elif format in cls.DICT_FORMATS:
+            return cls.item_to_dict_data(item, format)
+        else:
+            raise ValueError(f"Format {format} not supported yet")
+
+    @classmethod
+    def format_to_mimetype(cls, format: str) -> str:
+        """Converts a format string to the corresponding mimetype.
+
+        :param format: The format to convert.
+        :type format: str
+        :raises ValueError: If the format is not supported.
+        :return: The mimetype.
+        :rtype: str
+        """
+        if format in cls.IMAGE_FORMATS:
+            return f"image/{format}"
+        elif format in cls.MATRIX_FORMATS:
+            return "application/json"
+        elif format in cls.DICT_FORMATS:
+            return "application/json"
+        else:
+            return f"application/{format}"
+
+    @classmethod
+    def item_filename_to_data(cls, item_filename: str) -> io.BytesIO:
+        """Converts an item filename to a bytes stream
+
+        :param item_filename: The item filename to convert.
+        :type item_filename: str
+        :return: file bytes IO
+        :rtype: io.BytesIO
+        """
+        return io.BytesIO(open(item_filename, "rb").read())
+
+
+class DatasetStream:
+    def __init__(self) -> None:
+        self._reader: BaseReader = None
+        self._writer: BaseWriter = None
+
+    @property
+    def reader(self) -> BaseReader:
+        return self._reader
+
+    @property
+    def writer(self) -> BaseWriter:
+        return self._writer
+
+    @abstractmethod
+    def __len__(self):
+        pass
+
+    def flush(self):
+        pass
+
+    def add_root_files_keys(self, root_files_keys: Sequence[str]) -> None:
+        """Adds the root files keys of the dataset.
+
+        :param root_files_keys: The root files keys of the dataset.
+        :type root_files_keys: Sequence[str]
+        """
+        pass
+
+    def add_extensions_map(self, extensions_map: Dict[str, str]) -> None:
+        """Adds the extensions map of the dataset.
+
+        :param extensions_map: The extensions map of the dataset.
+        :type extensions_map: Dict[str, str]
+        """
+        pass
+
+    @abstractmethod
+    def manifest(self) -> dict:
+        """Returns the manifest of the dataset with infos about size and
+        sample's keys.
+
+        :raises ValueError: If the dataset is empty.
+        :return: The manifest of the dataset.
+        :rtype: dict
+        """
+        pass
+
+    @abstractmethod
+    def get_sample(self, sample_id: int) -> Sample:
+        """Returns the sample with the given id.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: int
+        :raises ValueError: If the sample_id is out of range.
+        :return: The sample with the given id.
+        :rtype: Sample
+        """
+        pass
+
+    @abstractmethod
+    def get_item(self, sample_id: int, item: str) -> any:
+        """Returns the sample's item with the given name.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: int
+        :param item: The name of the item.
+        :type item: str
+        :raises ValueError: If the sample_id is out of range or if the item is not in the sample.
+        :return: The item with the given name.
+        :rtype: any
+        """
+
+    @abstractmethod
+    def get_data(self, sample_id: int, item: str, format: str) -> Tuple[any, str]:
+        """Returns the sample's item with the given name in the given format.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: int
+        :param item: The name of the item.
+        :type item: str
+        :param format: The format of the item.
+        :type format: str
+        :return: The item with the given name in the given format.
+        :rtype: Tuple[any, str]
+        """
+        pass
+
+    @abstractmethod
+    def set_data(
+        self, sample_id: int, item: str, data: any, format: str
+    ) -> Tuple[any, str]:
+        """Sets the sample's item with the given name in the given format.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: int
+        :param item: The name of the item.
+        :type item: str
+        :param data: The data to set.
+        :type data: any
+        :param format: The format of the data.
+        :type format: str
+        :return: The item with the given name in the given format.
+        :rtype: Tuple[any, str]
+        """
+        pass
diff --git a/pipelime/sequences/streams/underfolder.py b/pipelime/sequences/streams/underfolder.py
new file mode 100644
index 0000000..12de516
--- /dev/null
+++ b/pipelime/sequences/streams/underfolder.py
@@ -0,0 +1,236 @@
+from typing import Dict, Hashable, KeysView, Optional, Sequence, Tuple
+from pipelime.filesystem.toolkit import FSToolkit
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.streams.base import DatasetStream, ItemConverter
+from pipelime.sequences.writers.filesystem import UnderfolderWriter
+from pipelime.sequences.samples import FileSystemSample, SamplesSequence
+
+
+class UnderfolderStream(DatasetStream):
+    def __init__(
+        self, folder: str, allowed_keys: Optional[Sequence[str]] = None
+    ) -> None:
+        """Creates an UnderfolderStream object that reads and writes samples from and to
+        the given Underfolder format dataset
+
+        :param folder: the Undefolder folder
+        :type folder: str
+        :param allowed_keys: list of allowed keys the user can modify , defaults to None
+        :type allowed_keys: Optional[Sequence[str]], optional
+        """
+        super().__init__()
+        self._folder = folder
+        self._reader = UnderfolderReader(folder=folder)
+        self._reader.flush()
+        self._allowed_keys = allowed_keys
+        self._writer = None
+        self._samples_map = {}
+        for sample in self._reader:
+            self._samples_map[sample.id] = sample
+
+        if len(self._reader) > 0:
+            self._reload_writer()
+
+    def is_key_allowed(self, key: str) -> bool:
+        """Checks if the given key is allowed to be modified.
+
+        :param key: The key to check.
+        :type key: str
+        :return: True if the key is allowed to be modified, False otherwise.
+        :rtype: bool
+        """
+        if self._allowed_keys is not None and key not in self._allowed_keys:
+            return False
+        return True
+
+    def _reload_writer(
+        self,
+        additional_root_files_keys: Optional[Sequence[str]] = None,
+        additional_extensions_map: Optional[Dict[str, str]] = None,
+    ) -> None:
+
+        root_files_keys = self._reader.get_reader_template().root_files_keys
+        extensions_map = self._reader.get_reader_template().extensions_map
+
+        if additional_root_files_keys is not None:
+            root_files_keys = root_files_keys + additional_root_files_keys
+
+        if additional_extensions_map is not None:
+            extensions_map.update(additional_extensions_map)
+
+        self._writer = UnderfolderWriter(
+            folder=self._folder,
+            root_files_keys=root_files_keys,
+            extensions_map=extensions_map,
+            zfill=self._reader.get_reader_template().idx_length,
+            remove_duplicates=True,
+        )
+
+    def add_root_files_keys(self, root_files_keys: Sequence[str]) -> None:
+        """Adds the root files keys of the dataset.
+
+        :param root_files_keys: The root files keys of the dataset.
+        :type root_files_keys: Sequence[str]
+        """
+        self._reload_writer(additional_root_files_keys=root_files_keys)
+
+    def add_extensions_map(self, extensions_map: Dict[str, str]) -> None:
+        """Adds the extensions map of the dataset.
+
+        :param extensions_map: The extensions map of the dataset.
+        :type extensions_map: Dict[str, str]
+        """
+        self._reload_writer(additional_extensions_map=extensions_map)
+
+    def flush(self):
+        return self._reader.flush()
+
+    def __len__(self):
+        return len(self._reader)
+
+    def manifest(self) -> dict:
+        """Returns the manifest of the dataset with infos about size and
+        sample's keys.
+
+        :return: The manifest of the dataset.
+        :rtype: dict
+        """
+        if len(self._reader) > 0:
+            sample = self._reader[0]
+            keys = list(sample.keys())
+            return {
+                "size": len(self),
+                "keys": keys,
+                "sample_ids": list(self.get_sample_ids()),
+            }
+        else:
+            return {
+                "size": 0,
+                "keys": [],
+                "sample_ids": [],
+            }
+
+    def get_sample_ids(self) -> KeysView:
+        """Get all the sample ids of the dataset.
+
+        :return: The sample ids of the dataset.
+        :rtype: KeysView
+        """
+        return self._samples_map.keys()
+
+    def get_sample(self, sample_id: Hashable) -> FileSystemSample:
+        """Returns the sample with the given id.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: Hashable
+        :raises KeyError: If the sample_id is out of range.
+        :return: The sample with the given id.
+        :rtype: Sample
+        """
+        if sample_id in self._samples_map:
+            return self._samples_map[sample_id]
+        else:
+            raise KeyError(f"Sample id '{sample_id}' not found")
+
+    def get_item(self, sample_id: Hashable, item: str) -> any:
+        """Returns the sample's item with the given name.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: Hashable
+        :param item: The name of the item.
+        :type item: str
+        :raises KeyError: If the item is not in the sample.
+        :return: The item with the given name.
+        :rtype: any
+        """
+        sample = self.get_sample(sample_id)
+        if item in sample:
+            return sample[item]
+        else:
+            raise KeyError(f"Item '{item}' not found")
+
+    def get_item_filename(self, sample_id: Hashable, item: str) -> str:
+        """Returns the filename of the item with the given name.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: Hashable
+        :param item: The name of the item.
+        :type item: str
+        :raises KeyError: If the item is not in the sample.
+        :return: The filename of the item with the given name.
+        :rtype: str
+        """
+
+        sample = self.get_sample(sample_id)
+        if item in sample.filesmap:
+            return sample.filesmap[item]
+        else:
+            raise KeyError(f"Item '{item}' not found in filesmap")
+
+    def get_data(self, sample_id: Hashable, item: str, format: str) -> Tuple[any, str]:
+        """Returns the sample's item with the given name in the given format.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: Hashable
+        :param item: The name of the item.
+        :type item: str
+        :param format: The format of the item.
+        :type format: str
+        :return: The item with the given name in the given format.
+        :rtype: Tuple[any, str]
+        """
+
+        item = self.get_item(sample_id, item)
+        mimetype = ItemConverter.format_to_mimetype(format)
+        return ItemConverter.item_to_data(item, format), mimetype
+
+    def get_bytes(self, sample_id: Hashable, item: str) -> Tuple[bytes, str]:
+        """Returns the sample's item with the given name in bytes.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: Hashable
+        :param item: The name of the item.
+        :type item: str
+        :return: The item with the given name in bytes.
+        :rtype: Tuple[bytes, str]
+        """
+        item_filename = self.get_item_filename(sample_id, item)
+        item_extension = FSToolkit.get_file_extension(item_filename)
+        mimetype = ItemConverter.format_to_mimetype(item_extension)
+        return ItemConverter.item_filename_to_data(item_filename), mimetype
+
+    def set_data(
+        self, sample_id: Hashable, item: str, data: any, format: str
+    ) -> Tuple[any, str]:
+        """Sets the sample's item with the given name in the given format.
+
+        :param sample_id: The id of the sample.
+        :type sample_id: Hashable
+        :param item: The name of the item.
+        :type item: str
+        :param data: The data to set [io.BytesIO, dict]
+        :type data: any
+        :param format: The format of the data.
+        :type format: str
+        :raises KeyError: If the sample_id is not found
+        :raises PermissionError: If the item key is not allowed to be modified.
+        :return: The item with the given name in the given format.
+        :rtype: Tuple[any, str]
+        """
+        if sample_id not in self._samples_map:
+            raise KeyError(f"Sample id '{sample_id}' not found")
+
+        if not self.is_key_allowed(item):
+            raise PermissionError(f"Item '{item}' not allowed")
+
+        if self._writer is not None:
+            sample = self.get_sample(sample_id)
+            old_keys = list(sample.keys())
+            sample[item] = ItemConverter.data_to_item(data, format)
+
+            # Copy sample in order to delete key only on current instance
+            sample = sample.copy()
+            for key in old_keys:
+                if key != item:
+                    del sample[key]
+            self._writer(SamplesSequence([sample]))
diff --git a/pipelime/sequences/validation.py b/pipelime/sequences/validation.py
index 285acbe..0dd9664 100644
--- a/pipelime/sequences/validation.py
+++ b/pipelime/sequences/validation.py
@@ -1,13 +1,48 @@
+from abc import ABC, abstractclassmethod, abstractmethod
 import importlib.machinery
+from pathlib import Path
 import types
 import uuid
+from schema import Schema, SchemaError
+from pipelime.sequences.samples import Sample
+from pipelime.sequences.stages import SampleStage
+from pipelime.sequences.operations import OperationStage
 
-from schema import Schema
 
-from pipelime.sequences.samples import Sample
+class SampleSchema(ABC):
+    class ValidationError(Exception):
+        pass
+
+    @abstractmethod
+    def validate(self, sample: Sample) -> bool:
+        """Validate a sample.
+
+        Args:
+            sample (Sample): The sample to validate.
 
+        Raises:
+            NotImplementedError: If the method is not implemented.
+
+        Returns:
+            bool: True if the sample is valid, False otherwise.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    @abstractclassmethod
+    def load_from_file(cls, filename: str) -> "SampleSchema":
+        """Load a schema from a file.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+        raise NotImplementedError
+
+
+class SampleSchemaWithSchema(SampleSchema):
+    DEFAULT_SCHEMA_VARIABLE_NAME = "schema"
+    DEFAULT_DEEP_VARIABLE_NAME = "deep"
 
-class SampleSchema:
     def __init__(self, schema: Schema, deep: bool = True):
         self._schema = schema
         self._deep = deep
@@ -20,33 +55,112 @@ def deep(self) -> bool:
     def schema(self) -> Schema:
         return self._schema
 
-    def validate(self, sample: Sample):
-        if not self.deep:
-            self.schema.validate(sample.skeleton)
-        else:
-            self.schema.validate(dict(sample))
+    def validate(self, sample: Sample) -> bool:
+        """Validate a sample with 'schema'.
 
+        Args:
+            sample (Sample): The sample to validate.
 
-class SchemaLoader:
-    DEFAULT_SCHEMA_VARIABLE_NAME = "schema"
-    DEFAULT_DEEP_VARIABLE_NAME = "deep"
+        Raises:
+            SampleSchema.ValidationError: If the sample is not valid.
+
+        Returns:
+            bool: True if the sample is valid, False otherwise.
+        """
+        try:
+            if not self.deep:
+                self.schema.validate(sample.skeleton)
+            else:
+                self.schema.validate(dict(sample))
+        except SchemaError as e:
+            raise SampleSchema.ValidationError(e)
+        return True
 
     @classmethod
-    def load(cls, filename: str) -> SampleSchema:
+    def _load_module(cls, path: str):
+        loader = importlib.machinery.SourceFileLoader(str(uuid.uuid1()), path)
+        mod = types.ModuleType(loader.name)
+        loader.exec_module(mod)
+        return mod
+
+    @classmethod
+    def load_from_file(cls, filename: str) -> "SampleSchemaWithSchema":
+        """Load a schema from a file parsing it as a python file. The python file
+        should contain a DEFAULT_SCHEMA_VARIABLE_NAME and a DEFAULT_DEEP_VARIABLE_NAME
+
+        Raises:
+            ModuleNotFoundError: [description]
+
+        Returns:
+            [type]: [description]
+        """
         module = cls._load_module(str(filename))
-        sname = SchemaLoader.DEFAULT_SCHEMA_VARIABLE_NAME
+        sname = cls.DEFAULT_SCHEMA_VARIABLE_NAME
         if not hasattr(module, sname):
             raise ModuleNotFoundError(
                 f"No variable with name {sname} found in file: {filename}"
             )
 
         schema = Schema(getattr(module, sname))
-        deep = getattr(module, SchemaLoader.DEFAULT_DEEP_VARIABLE_NAME, True)
-        return SampleSchema(schema=schema, deep=deep)
+        deep = getattr(module, cls.DEFAULT_DEEP_VARIABLE_NAME, True)
+        return SampleSchemaWithSchema(schema=schema, deep=deep)
+
+
+class SchemaLoader:
+    SCHEMA_EXTENSIONS_MAP = {"schema": SampleSchemaWithSchema}
 
     @classmethod
-    def _load_module(cls, path: str):
-        loader = importlib.machinery.SourceFileLoader(str(uuid.uuid1()), path)
-        mod = types.ModuleType(loader.name)
-        loader.exec_module(mod)
-        return mod
+    def is_a_valid_schema_file(cls, filename: str) -> bool:
+        """Check if a file is a valid schema file.
+
+        Args:
+            filename (str): The file to check.
+
+        Returns:
+            bool: True if the file is a valid schema file, False otherwise.
+        """
+        return Path(filename).suffix.replace(".", "") in cls.SCHEMA_EXTENSIONS_MAP
+
+    @classmethod
+    def load(cls, filename: str) -> SampleSchema:
+        """Loads a generic schema from a file. This is a factory method based on the
+        file extension.
+
+        Args:
+            filename (str): The file to load the schema from.
+
+        Raises:
+            NotImplementedError: If the file extension is not supported.
+
+        Returns:
+            SampleSchema: The loaded SampleSchema.
+        """
+        ext = Path(filename).suffix.replace(".", "")
+        if not cls.is_a_valid_schema_file(filename):
+            raise NotImplementedError(
+                f"File extension {ext} is not supported. Supported extensions"
+                f"are: {cls.SCHEMA_EXTENSIONS_MAP.keys()}"
+            )
+
+        return cls.SCHEMA_EXTENSIONS_MAP[ext].load_from_file(filename)
+
+
+class StageValidate(SampleStage):
+    """This is a generic stage that validates a sample. Accepts a generic schema"""
+
+    def __init__(self, sample_schema: SampleSchema):
+        self._sample_schema = sample_schema
+
+    def __call__(self, x: Sample) -> Sample:
+        try:
+            self._sample_schema.validate(x)
+        except Exception as e:
+            raise SampleSchema.ValidationError(f"Sample ID: {x.id} -> {str(e)}")
+        return x
+
+
+class OperationValidate(OperationStage):
+    """This is an OperationStage that validates a sample sequence. Accepts a generic schema"""
+
+    def __init__(self, sample_schema: SampleSchema, **kwargs):
+        super().__init__(stage=StageValidate(sample_schema), **kwargs)
diff --git a/pipelime/sequences/writers/filesystem.py b/pipelime/sequences/writers/filesystem.py
index bbb2596..fe40702 100644
--- a/pipelime/sequences/writers/filesystem.py
+++ b/pipelime/sequences/writers/filesystem.py
@@ -1,82 +1,115 @@
+import functools
 import multiprocessing
-import os
 import re
 import shutil
 from pathlib import Path
-
-from choixe.spooks import Spook
-from rich.progress import track
-from schema import Optional, Or
-
+from typing import Callable, Dict, Sequence, Optional, Any
+from schema import Or
+from enum import Enum
+from loguru import logger
 from pipelime.filesystem.toolkit import FSToolkit
-from pipelime.sequences.readers.base import BaseReader
+from pipelime.sequences.readers.base import BaseReader, ReaderTemplate
 from pipelime.sequences.samples import (
-    FilesystemItem,
+    FileSystemItem,
     FileSystemSample,
     Sample,
     SamplesSequence,
 )
 from pipelime.sequences.writers.base import BaseWriter
+from pipelime.tools.progress import pipelime_track
 
 
 class UnderfolderWriter(BaseWriter):
     DATA_SUBFOLDER = "data"
     DEFAULT_EXTENSION = "pkl"
+    WORKER_ID = 0
 
     def __init__(
         self,
         folder: str,
-        root_files_keys: list = None,
-        extensions_map: dict = None,
+        root_files_keys: Optional[Sequence[str]] = None,
+        extensions_map: Optional[Dict[str, str]] = None,
         zfill: int = 5,
         copy_files: bool = True,
         use_symlinks: bool = False,
-        force_copy_keys: list = None,
+        force_copy_keys: Optional[Sequence[str]] = None,
+        remove_duplicates: bool = False,
         num_workers: int = 0,  # typing is here because 'schema.Optional' conflicts
+        progress_callback: Optional[Callable[[dict], None]] = None,
     ) -> None:
         """UnderfolderWriter for an input SamplesSequence
 
         :param folder: destiantion folder
         :type folder: str
-        :param root_files_keys: list of keys to write as underfolder root files (only the first element
-        encountered will be stored as root file, the others will be discarded), defaults to None
-        :type root_files_keys: list, optional
-        :param extensions_map: dictionary of regex/extension to retrieve extension for each key matching
-        the corresponding regex. Unmatched keys will be stored as PICKLE object file, defaults to None
-        :type extensions_map: dict, optional
-        :param zfill: Length of zero padding in case of integer sample indices, defaults to 5
+        :param root_files_keys: list of keys to write as underfolder root files (only
+        the first element encountered will be stored as root file, the others will be
+        discarded), defaults to None
+        :type root_files_keys: Optional[Sequence[str]]
+        :param extensions_map: dictionary of regex/extension to retrieve extension for
+        each key matching the corresponding regex. Unmatched keys will be stored as
+        PICKLE object file, defaults to None
+        :type extensions_map: Optional[Dict[str, str]]
+        :param zfill: Length of zero padding in case of integer sample indices,
+        defaults to 5
         :type zfill: int, optional
-        :param copy_files: TRUE to copy FileSystemSample directly if not cached before, defaults to True
+        :param copy_files: TRUE to copy FileSystemSample directly if not cached before,
+        defaults to True
         :type copy_files: bool, optional
-        :param use_symlinks: if TRUE (and copy_files == TRUE) the copy will be replaced with a symlink, defaults to False
+        :param use_symlinks: if TRUE (and copy_files == TRUE) the copy will be replaced
+        with a symlink, defaults to False
         :type use_symlinks: bool, optional
-        :param force_copy_keys: A list of string keys that will be copied/symlinked regardless of their
-        cached/modified state (all changes will be lost), if they contain file system items.
-        This is useful in case you want to quickly write an item that you know for certain it was not modified, even if it was cached.
-        :type force_copy_keys: list, optional
-        :param num_workers: if 0 disable multiprocessing, if -1 use Multiprocessing pool with all available processors, if > 0 use Multiprocessing using as many processes
+        :param force_copy_keys: A list of string keys that will be copied/symlinked
+        regardless of their cached/modified state (all changes will be lost), if they
+        contain file system items. This is useful in case you want to quickly write an
+        item that you know for certain it was not modified, even if it was cached.
+        :type force_copy_keys: Optional[Sequence[str]]
+        :param remove_duplicates: if TRUE, will check for duplicates (same item with
+        different extension) in the destination folder and delete them before storing,
+        defaults to False
+        :type remove_duplicates: bool, optional
+        :param num_workers: if 0 disable multiprocessing, if -1 use Multiprocessing
+        pool with all available processors, if > 0 use Multiprocessing using as many
+        processes
         :type num_workers: int, optional
+        :param progress_callback: callback function to call with progress information,
+        defaults to None
+        :type progress_callback: Optional[Callable[[dict], None]], optional
         """
         self._folder = Path(folder)
 
-        self._empty_template = root_files_keys is None and extensions_map is None
         self._root_files_keys = root_files_keys if root_files_keys is not None else []
         self._extensions_map = extensions_map if extensions_map is not None else {}
         self._zfill = zfill
         self._copy_files = copy_files
         self._use_symlinks = use_symlinks
+        self._force_copy_keys = force_copy_keys if force_copy_keys is not None else []
+        self._remove_duplicates = remove_duplicates
         self._num_workers = num_workers
-
-        if force_copy_keys is None:
-            force_copy_keys = []
-
-        self._force_copy_keys = force_copy_keys
+        self._progress_callback = progress_callback
+        self._track = (
+            pipelime_track
+            if progress_callback is None
+            else functools.partial(pipelime_track, disable=True)
+        )
         self._datafolder = self._folder / self.DATA_SUBFOLDER
-        if not self._datafolder.exists():
-            self._datafolder.mkdir(parents=True, exist_ok=True)
+        self._datafolder.mkdir(parents=True, exist_ok=True)
 
         # multiprocessing attrs
-        self._saved_root_keys = None
+        self._saved_root_keys = {}
+
+        if self._use_symlinks:
+            import platform
+
+            if platform.system() == "Windows":
+                logger.warning(
+                    "Symlink is not supported on Windows,"
+                    " switching to file deep copy"
+                )
+                self._use_symlinks = False
+
+    @property
+    def _empty_template(self) -> bool:
+        return not (self._root_files_keys or self._extensions_map)
 
     def _build_sample_basename(self, sample: Sample):
         if isinstance(sample.id, int):
@@ -86,10 +119,8 @@ def _build_sample_basename(self, sample: Sample):
 
     def _build_item_extension(self, key: str):
         extension = UnderfolderWriter.DEFAULT_EXTENSION
-        for r, ext in self._extensions_map.items():
-            if re.match(r, key) is not None:
-                extension = ext
-                break
+        if key in self._extensions_map:
+            extension = self._extensions_map[key]
         return extension
 
     def _is_root_key(self, key: str):
@@ -98,7 +129,7 @@ def _is_root_key(self, key: str):
                 return True
         return False
 
-    def _process_sample(self, sample: Sample):
+    def _process_sample(self, sample: Sample, only_root_keys: bool = False):
         basename = self._build_sample_basename(sample)
 
         for key in sample.keys():
@@ -109,13 +140,15 @@ def _process_sample(self, sample: Sample):
                     itemname = f"{key}.{self._build_item_extension(key)}"
                     output_file = Path(self._folder) / itemname
                     self._write_sample_item(output_file, sample, key)
-            else:
+            elif not only_root_keys:
                 itemname = f"{basename}_{key}.{self._build_item_extension(key)}"
                 output_file = Path(self._datafolder) / itemname
 
                 self._write_sample_item(output_file, sample, key)
 
     def __call__(self, x: SamplesSequence) -> None:
+        if len(x) == 0:
+            return
 
         if isinstance(x, BaseReader) and self._empty_template:
             template = x.get_reader_template()
@@ -126,33 +159,58 @@ def __call__(self, x: SamplesSequence) -> None:
 
         self._zfill = max(self._zfill, x.best_zfill())
 
-        if self._num_workers > 0 or self._num_workers == -1:
-            manager = multiprocessing.Manager()
-            self._saved_root_keys = manager.dict()
-            pool = multiprocessing.Pool(
+        self._saved_root_keys = {}
+        if self._num_workers > 1 or self._num_workers == -1:
+            # first save root keys, if any
+            self._process_sample(x[0], True)
+
+            with multiprocessing.Pool(
                 processes=None if self._num_workers == -1 else self._num_workers
-            )
-            list(track(pool.imap_unordered(self._process_sample, x), total=len(x)))
+            ) as pool:
+                list(
+                    self._track(
+                        pool.imap_unordered(self._process_sample, x),
+                        total=len(x),
+                        description="Writing Underfolder",
+                        track_callback=self._progress_callback,
+                    )
+                )
         else:
-            self._saved_root_keys = {}
-            for sample in track(x):
+            for sample in self._track(
+                x,
+                description="Writing Underfolder",
+                track_callback=self._progress_callback,
+            ):
                 self._process_sample(sample)
 
-    def _copy_filesystem_item(self, output_file: Path, item: FilesystemItem) -> None:
+    def _copy_filesystem_item(self, output_file: Path, item: FileSystemItem) -> None:
         path = item.source()
-        if not self._use_symlinks:
-            if path != output_file:
+        if path != output_file:
+            if not self._use_symlinks:
                 shutil.copy(path, output_file)
-        else:
-            os.symlink(path, output_file)
+            else:
+                output_file.symlink_to(path)
+
+    def _dump_cached_item(self, output_file: Path, item: Any):
+        FSToolkit.store_data(str(output_file), item)
+
+    def _remove_duplicate_files(self, output_file: Path):
+        duplicates = list(output_file.parent.glob(f"{output_file.stem}.*"))
+        for f in duplicates:
+            if f.suffix != output_file.suffix:
+                f.unlink()
 
     def _write_sample_item(self, output_file: Path, sample: Sample, key: str):
         copy_item = False
+
+        if self._remove_duplicates:
+            self._remove_duplicate_files(output_file)
+
         if self._copy_files and isinstance(sample, FileSystemSample):
             item = sample.metaitem(key)
             if (
                 not sample.is_cached(key) or key in self._force_copy_keys
-            ) and isinstance(item, FilesystemItem):
+            ) and isinstance(item, FileSystemItem):
                 path = item.source()
                 if path.suffix == output_file.suffix:
                     copy_item = True
@@ -160,8 +218,7 @@ def _write_sample_item(self, output_file: Path, sample: Sample, key: str):
         if copy_item:
             self._copy_filesystem_item(output_file, item)
         else:
-            item = sample[key]
-            FSToolkit.store_data(output_file, item)
+            self._dump_cached_item(output_file, sample[key])
 
     @classmethod
     def bean_schema(cls) -> dict:
@@ -188,3 +245,119 @@ def to_dict(self) -> dict:
             "extensions_map": self._extensions_map,
             "zfill": self._zfill,
         }
+
+
+class UnderfolderWriterV2(UnderfolderWriter):
+    class FileHandling(Enum):
+        ALWAYS_WRITE_FROM_CACHE = 0
+        ALWAYS_COPY_FROM_DISK = 1
+        COPY_IF_NOT_CACHED = 2
+
+    class CopyMode(Enum):
+        DEEP_COPY = 0
+        SYM_LINK = 1
+        HARD_LINK = 2
+
+    def __init__(
+        self,
+        folder: str,
+        file_handling: FileHandling = FileHandling.COPY_IF_NOT_CACHED,
+        copy_mode: CopyMode = CopyMode.DEEP_COPY,
+        force_copy_keys: Optional[Sequence[str]] = None,
+        reader_template: Optional[ReaderTemplate] = None,
+        remove_duplicates: bool = False,
+        num_workers: int = 0,
+        progress_callback: Optional[Callable[[dict], None]] = None,
+    ):
+        root_file_keys, extensions_map, zfill = (
+            (
+                reader_template.root_files_keys,
+                reader_template.extensions_map,
+                reader_template.idx_length,
+            )
+            if reader_template is not None
+            else (None, None, 5)
+        )
+
+        super().__init__(
+            folder=folder,
+            root_files_keys=root_file_keys,
+            extensions_map=extensions_map,
+            zfill=zfill,
+            force_copy_keys=force_copy_keys,
+            remove_duplicates=remove_duplicates,
+            num_workers=num_workers,
+            progress_callback=progress_callback,
+        )
+
+        self._file_handling = file_handling
+        self._copy_mode = copy_mode
+
+        if self._copy_mode is UnderfolderWriterV2.CopyMode.SYM_LINK:
+            import platform
+
+            if platform.system() == "Windows":
+                logger.warning(
+                    "Symlink is not supported on Windows,"
+                    " switching to file deep copy"
+                )
+                self._copy_mode = UnderfolderWriterV2.CopyMode.DEEP_COPY
+
+    def _copy_filesystem_item(self, output_file: Path, item: FileSystemItem) -> None:
+        path = item.source()
+        if path != output_file:
+            if self._copy_mode is UnderfolderWriterV2.CopyMode.DEEP_COPY:
+                shutil.copy(path, output_file)
+            elif self._copy_mode is UnderfolderWriterV2.CopyMode.SYM_LINK:
+                output_file.symlink_to(path)
+            elif self._copy_mode is UnderfolderWriterV2.CopyMode.HARD_LINK:
+
+                # TODO: if you call the writer __call__ multiple times, due to batching,
+                # and the root_files_keys are the same, the hard link will throw an error
+                # because the file already exists. This is a workaround for now.
+                if not output_file.exists():
+                    try:
+                        # (new in version 3.10)
+                        output_file.hardlink_to(path)  # type: ignore
+                    except AttributeError:
+                        import os
+
+                    os.link(path, output_file)
+
+    def _write_sample_item(self, output_file: Path, sample: Sample, key: str):
+        if self._remove_duplicates:
+            self._remove_duplicate_files(output_file)
+
+        if (
+            self._file_handling
+            is not UnderfolderWriterV2.FileHandling.ALWAYS_WRITE_FROM_CACHE
+        ):
+            # copy source file if possible
+            def _is_item_cached():
+                try:
+                    # sample is, eg, a FileSystemSample
+                    return sample.is_cached(key)  # type: ignore
+                except AttributeError:  # pragma: no cover
+                    return False  # pragma: no cover
+
+            meta_item = sample.metaitem(key)
+            if (
+                isinstance(meta_item, FileSystemItem)
+                and meta_item.source().suffix == output_file.suffix
+                and (
+                    self._file_handling
+                    is UnderfolderWriterV2.FileHandling.ALWAYS_COPY_FROM_DISK
+                    or key in self._force_copy_keys
+                    or not _is_item_cached()
+                )
+            ):
+                # * it is a file AND
+                # * suffix does not change (eg, no image re-encoding) AND
+                # ** always copy OR
+                # ** this key must be always copied OR
+                # ** item is not cached
+                self._copy_filesystem_item(output_file, meta_item)
+                return
+
+        # if any of the above checks failed, just dump the item value
+        self._dump_cached_item(output_file, sample[key])
diff --git a/pipelime/sequences/writers/h5.py b/pipelime/sequences/writers/h5.py
index 35310e1..3d4b21f 100644
--- a/pipelime/sequences/writers/h5.py
+++ b/pipelime/sequences/writers/h5.py
@@ -1,10 +1,7 @@
 from pathlib import Path
-
 import h5py
-from choixe.spooks import Spook
 from rich.progress import track
 from schema import Optional, Or
-
 from pipelime.h5.toolkit import H5Database, H5ToolKit
 from pipelime.sequences.readers.base import BaseReader
 from pipelime.sequences.samples import Sample, SamplesSequence
@@ -89,7 +86,7 @@ def __call__(self, x: SamplesSequence) -> None:
                         global_links[key] = h5py.SoftLink(group[key].name)
             break
 
-        for sample in track(x):
+        for sample in track(x, description="Writing H5"):
             basename = self._build_sample_basename(sample)
 
             for key in sample.keys():
diff --git a/pipelime/tools/bytes.py b/pipelime/tools/bytes.py
index 76ddf46..775ba0b 100644
--- a/pipelime/tools/bytes.py
+++ b/pipelime/tools/bytes.py
@@ -8,14 +8,14 @@
 
 class DataCoding(object):
 
-    IMAGE_CODECS = ["jpg", "jpeg", "png", "tiff", "bmp"]
+    IMAGE_CODECS = ("jpg", "jpeg", "png", "tiff", "bmp")
     IMAGE_CODECS_IS_LOSSY = {"jpg": True, "jpeg": True}
     IMAGE_CODECS_HAS_ALPHA = {"png": True}
 
-    NUMPY_CODECS = ["npy"]
-    TEXT_CODECS = ["txt"]
-    METADATA_CODECS = ["json", "yml", "yaml"]
-    PICKLE_CODECS = ["pkl"]
+    NUMPY_CODECS = ("npy",)
+    TEXT_CODECS = ("txt",)
+    METADATA_CODECS = ("json", "yml", "yaml", "toml", "tml")
+    PICKLE_CODECS = ("pkl", "pickle")
 
     @classmethod
     def is_image_extension(cls, extension: str):
diff --git a/pipelime/tools/click.py b/pipelime/tools/click.py
new file mode 100644
index 0000000..b7706e7
--- /dev/null
+++ b/pipelime/tools/click.py
@@ -0,0 +1,31 @@
+from re import X
+import click
+
+
+class ClickTools:
+    @classmethod
+    def parse_additional_args(cls, ctx: click.Context) -> dict:
+        """Parse additional arguments from the click context. Usually they are parameters
+        not defined in the click command but provided to CLI.
+
+        Args:
+            ctx (click.Context): click context
+
+        Raises:
+            click.UsageError: if arguments list is not even
+
+        Returns:
+            dict: additional arguments
+        """
+
+        # remove '--' and '-' from args names
+        x = [x.strip("-") for x in ctx.args]
+
+        # check even number of args
+        if len(x) % 2 != 0:
+            raise click.UsageError("Invalid number of arguments")
+
+        # create dict from pairs key:value
+        x = [x[i : i + 2] for i in range(0, len(x), 2)]
+
+        return dict(x)
diff --git a/pipelime/tools/dictionaries.py b/pipelime/tools/dictionaries.py
new file mode 100644
index 0000000..3d9426f
--- /dev/null
+++ b/pipelime/tools/dictionaries.py
@@ -0,0 +1,200 @@
+import dictquery
+from typing import Any, List, Sequence, Tuple, Dict, Optional
+
+
+class DictionaryWalker:
+    @classmethod
+    def flatten_as_lists(
+        cls,
+        d: dict,
+        discard_keys: Optional[Sequence[str]] = None,
+    ) -> Sequence[Tuple[List[str], Any]]:
+        return cls.walk(d, discard_keys=discard_keys)
+
+    @classmethod
+    def flatten_as_tuples(
+        cls,
+        d: dict,
+        discard_keys: Optional[Sequence[str]] = None,
+    ) -> Sequence[Tuple[Tuple[str], Any]]:
+        return [(tuple(x), y) for x, y in cls.walk(d, discard_keys=discard_keys)]
+
+    @classmethod
+    def flatten_as_dicts(
+        cls,
+        d: dict,
+        discard_keys: Optional[Sequence[str]] = None,
+    ) -> Sequence[Tuple[str, Any]]:
+
+        return [
+            (".".join(x), y)
+            for x, y in cls.flatten_as_lists(d, discard_keys=discard_keys)
+        ]
+
+    @classmethod
+    def flatten(
+        cls,
+        d: dict,
+        discard_keys: Optional[Sequence[str]] = None,
+    ) -> dict:
+
+        return {k: v for k, v in cls.flatten_as_dicts(d, discard_keys=discard_keys)}
+
+    @classmethod
+    def walk(
+        cls,
+        d: Dict,
+        path: Sequence = None,
+        chunks: Sequence = None,
+        discard_keys: Optional[Sequence[str]] = None,
+    ) -> Sequence[Tuple[str, Any]]:
+        """Deep visit of dictionary building a plain sequence of pairs(key, value) where key has a pydash notation
+        : param d: input dictionary
+        : type d: Dict
+        : param path: private output value for path(not use), defaults to None
+        : type path: Sequence, optional
+        : param chunks: private output to be fileld with retrieved pairs(not use), defaults to None
+        : type chunks: Sequence, optional
+        : param discard_private_qualifiers: TRUE to discard keys starting with private qualifier, defaults to True
+        : type discard_private_qualifiers: bool, optional
+        : return: sequence of retrieved pairs
+        : rtype: Sequence[Tuple[str, Any]]
+        """
+
+        discard_keys = discard_keys if discard_keys is not None else []
+        root = False
+        if path is None:
+            path, chunks, root = [], [], True
+        if isinstance(d, dict):
+            for k, v in d.items():
+                path.append(k)
+                if isinstance(v, dict) or isinstance(v, list):
+                    cls.walk(
+                        v,
+                        path=path,
+                        chunks=chunks,
+                        discard_keys=discard_keys,
+                    )
+                else:
+                    keys = list(map(str, path))
+                    if not (
+                        len(discard_keys) > 0
+                        and any([x for x in discard_keys if x in keys])
+                    ):
+                        chunks.append((keys, v))
+                path.pop()
+        elif isinstance(d, list):
+            for idx, v in enumerate(d):
+                path.append(idx)
+                cls.walk(
+                    v,
+                    path=path,
+                    chunks=chunks,
+                    discard_keys=discard_keys,
+                )
+                path.pop()
+        else:
+            keys = list(map(str, path))
+            chunks.append((keys, d))
+        if root:
+            return chunks
+
+
+class DictionaryUtils:
+    @classmethod
+    def flatten(cls, d: dict, parent_key="", sep=".") -> Dict[str, any]:
+        """Computes a flattened dictionary.
+        For example:
+
+        {
+            "a": {
+                "b": {
+                    "c": 1
+                }
+                "d": 2
+            }
+        }
+
+        becomes:
+
+        {
+            "a.b.c": 1,
+            "a.d": 2
+        }
+
+        :param d: [description]
+        :type d: dict
+        :param parent_key: [description], defaults to ""
+        :type parent_key: str, optional
+        :param sep: [description], defaults to "."
+        :type sep: str, optional
+        :return: [description]
+        :rtype: Dict[str, any]
+        """
+        return {k: v for k, v in DictionaryWalker.flatten_as_dicts(d)}
+
+
+class DictSearch:
+    KEY_PLACEHOLDER = "$V"
+
+    @classmethod
+    def match_queries(cls, proto_dict: dict, target_dict: dict) -> bool:
+        """Match a dict against a dict proto. The dict proto is a dict where values
+        are 'dictquery'-like strings (i.e. info here:
+        https://github.com/cyberlis/dictquery).
+        For example with the proto dict:
+
+        .. code-block:: python
+
+            {
+                'a': {
+                    'b': {
+                        'c': '>= 1',
+                    },
+                },
+            }
+
+        will match positive if the target dict is {'a': {'b': {'c': 10}}} and negative
+        if the target dict is {'a': {'b': {'c': 0}}}.
+
+        The proto dict can contains also multiple occurrences of the same key. In this
+        case the value should contains placeholders for the occurrence number.
+        For example:
+
+        .. code-block:: python
+
+            {
+                'a': {
+                    'b': '$V >= 2 AND $V <= 10'
+            }
+
+        The proto dict can contains also None values, in this case nothing happens, the
+        query is bypassed.
+
+        :param proto_dict: the proto dict
+        :type proto_dict: dict
+        :param target_dict: the target dict to match
+        :type target_dict: dict
+        :return: TRUE if query matches, FALSE otherwise
+        :rtype: bool
+        """
+
+        flatten_proto_dict = DictionaryUtils.flatten(proto_dict)
+
+        valid = True
+        for key, value in flatten_proto_dict.items():
+            query = DictSearch.build_query(key, value)
+            if query is not None:
+                valid = dictquery.match(target_dict, query)
+            if not valid:
+                break
+        return valid
+
+    @classmethod
+    def build_query(cls, key: str, value: Optional[str] = None) -> Optional[str]:
+        if value is None:
+            return None
+        if "$V" not in value:
+            return f"`{key}` {value}"
+        else:
+            return value.replace("$V", f"`{key}`")
diff --git a/pipelime/tools/modules.py b/pipelime/tools/modules.py
new file mode 100644
index 0000000..b9d40c3
--- /dev/null
+++ b/pipelime/tools/modules.py
@@ -0,0 +1,21 @@
+import importlib
+import types
+import uuid
+
+
+class ModulesUtils:
+    @classmethod
+    def load_module_from_file(cls, path: str) -> types.ModuleType:
+        loader = importlib.machinery.SourceFileLoader(str(uuid.uuid1()), path)
+        mod = types.ModuleType(loader.name)
+        loader.exec_module(mod)
+        return mod
+
+    @classmethod
+    def load_variable_from_file(cls, path: str, name: str) -> any:
+        module = ModulesUtils.load_module_from_file(path)
+        if not hasattr(module, name):
+            raise ModuleNotFoundError(
+                f"No variable with name {name} found in file: {path}"
+            )
+        return getattr(module, name)
diff --git a/pipelime/tools/progress.py b/pipelime/tools/progress.py
new file mode 100644
index 0000000..c4ed304
--- /dev/null
+++ b/pipelime/tools/progress.py
@@ -0,0 +1,187 @@
+from typing import Callable, Iterable, List, Sequence, Optional, Any, Sized, Union
+from rich.console import Console
+from rich.progress import (
+    Progress,
+    ProgressColumn,
+    TextColumn,
+    BarColumn,
+    TimeRemainingColumn,
+    ProgressType,
+    _TrackThread,
+    TaskID,
+)
+
+
+class PipelimeProgress(Progress):
+    def __init__(
+        self,
+        *columns: Union[str, ProgressColumn],
+        console: Optional[Console] = None,
+        auto_refresh: bool = True,
+        refresh_per_second: float = 10,
+        speed_estimate_period: float = 30,
+        transient: bool = False,
+        redirect_stdout: bool = True,
+        redirect_stderr: bool = True,
+        get_time: Optional[Any] = None,
+        disable: bool = False,
+        expand: bool = False,
+        style: str = "bar.back",
+        complete_style: str = "bar.complete",
+        finished_style: str = "bar.finished",
+        pulse_style: str = "bar.pulse",
+        track_callback: Optional[Callable[[Any], None]] = None,
+    ) -> None:
+        """Simple wrapper for rich progress bar. It adds a user callback to track progress."""
+
+        self._track_callback = track_callback
+
+        super().__init__(
+            *columns,
+            auto_refresh=auto_refresh,
+            console=console,
+            transient=transient,
+            get_time=get_time,
+            refresh_per_second=refresh_per_second or 10,
+            disable=disable,
+        )
+
+    def _manage_track_callback(
+        self, task_id: Optional[TaskID] = None, total: float = 1.0, advance: float = 1.0
+    ):
+        if self._track_callback is not None:
+            self._track_callback(
+                {"task_id": task_id, "advance": advance, "total": total}
+            )
+
+    def track(
+        self,
+        sequence: Union[Iterable[ProgressType], Sequence[ProgressType]],
+        total: Optional[float] = None,
+        task_id: Optional[TaskID] = None,
+        description: str = "Working...",
+        update_period: float = 0.1,
+    ) -> Iterable[ProgressType]:
+        """Track progress by iterating over a sequence.
+
+        Args:
+            sequence (Sequence[ProgressType]): A sequence of values you want to iterate over and track progress.
+            total: (float, optional): Total number of steps. Default is len(sequence).
+            task_id: (TaskID): Task to track. Default is new task.
+            description: (str, optional): Description of task, if new task is created.
+            update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1.
+
+        Returns:
+            Iterable[ProgressType]: An iterable of values taken from the provided sequence.
+        """
+
+        if total is None:
+            if isinstance(sequence, Sized):
+                task_total = float(len(sequence))
+            else:
+                raise ValueError(
+                    f"unable to get size of {sequence!r}, please specify 'total'"
+                )
+        else:
+            task_total = total
+
+        if task_id is None:
+            task_id = self.add_task(description, total=task_total)
+        else:
+            self.update(task_id, total=task_total)
+
+        if self.live.auto_refresh:
+            with _TrackThread(self, task_id, update_period) as track_thread:
+                for value in sequence:
+                    yield value
+                    self._manage_track_callback(
+                        task_id=task_id,
+                        advance=1,
+                        total=task_total,
+                    )
+                    track_thread.completed += 1
+        else:
+            advance = self.advance
+            refresh = self.refresh
+            for value in sequence:
+                yield value
+                self._manage_track_callback(
+                    task_id=task_id,
+                    advance=1,
+                    total=task_total,
+                )
+                advance(task_id, 1)
+                refresh()
+
+
+def pipelime_track(
+    sequence: Union[Sequence[ProgressType], Iterable[ProgressType]],
+    description: str = "Working...",
+    total: Optional[float] = None,
+    auto_refresh: bool = True,
+    console: Optional[Console] = None,
+    transient: bool = False,
+    get_time: Optional[Callable[[], float]] = None,
+    refresh_per_second: float = 10,
+    style: str = "bar.back",
+    complete_style: str = "bar.complete",
+    finished_style: str = "bar.finished",
+    pulse_style: str = "bar.pulse",
+    update_period: float = 0.1,
+    disable: bool = False,
+    track_callback: Optional[Callable[[Any], None]] = None,
+) -> Iterable[ProgressType]:
+    """Track progress by iterating over a sequence.
+
+    Args:
+        sequence (Iterable[ProgressType]): A sequence (must support "len") you wish to iterate over.
+        description (str, optional): Description of task show next to progress bar. Defaults to "Working".
+        total: (float, optional): Total number of steps. Default is len(sequence).
+        auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True.
+        transient: (bool, optional): Clear the progress on exit. Defaults to False.
+        console (Console, optional): Console to write to. Default creates internal Console instance.
+        refresh_per_second (float): Number of times per second to refresh the progress information. Defaults to 10.
+        style (StyleType, optional): Style for the bar background. Defaults to "bar.back".
+        complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete".
+        finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done".
+        pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse".
+        update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1.
+        disable (bool, optional): Disable display of progress.
+        track_callback: (Callable[[Any], None], optional): Callback to call with progress information.
+    Returns:
+        Iterable[ProgressType]: An iterable of the values in the sequence.
+
+    """
+
+    columns: List["ProgressColumn"] = (
+        [TextColumn("🍋|[progress.description]{task.description}")]
+        if description
+        else [TextColumn("🍋|")]
+    )
+    columns.extend(
+        (
+            BarColumn(
+                style=style,
+                complete_style=complete_style,
+                finished_style=finished_style,
+                pulse_style=pulse_style,
+            ),
+            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+            TimeRemainingColumn(),
+        )
+    )
+    progress = PipelimeProgress(
+        *columns,
+        auto_refresh=auto_refresh,
+        console=console,
+        transient=transient,
+        get_time=get_time,
+        refresh_per_second=refresh_per_second or 10,
+        disable=disable,
+        track_callback=track_callback,
+    )
+
+    with progress:
+        yield from progress.track(
+            sequence, total=total, description=description, update_period=update_period
+        )
diff --git a/pipelime/tools/toydataset.py b/pipelime/tools/toydataset.py
index 9b3d795..45e95af 100644
--- a/pipelime/tools/toydataset.py
+++ b/pipelime/tools/toydataset.py
@@ -1,12 +1,12 @@
 import json
 import uuid
 from pathlib import Path
-from typing import Sequence
-
 import numpy as np
 from PIL import Image, ImageDraw
-
+from typing import Callable, Tuple
 from pipelime.filesystem.toolkit import FSToolkit
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.tools.progress import pipelime_track
 
 
 class ToyDatasetGenerator(object):
@@ -105,7 +105,7 @@ def generate_2d_objects_images(self, size, objects):
             "instances": np.array(instances),
         }
 
-    def generate_image_sample(self, size, max_label=5, objects_number_range=[1, 5]):
+    def generate_image_sample(self, size, max_label=5, objects_number_range=(1, 5)):
 
         objects = []
         bboxes = []
@@ -121,6 +121,7 @@ def generate_image_sample(self, size, max_label=5, objects_number_range=[1, 5]):
             bboxes.append(box)
             keypoints.extend(kps)
 
+        rnd_bin = int(474)  # written as big-endian binary, equals to an image header
         data = self.generate_2d_objects_images(size, objects)
         data.update(
             {
@@ -128,43 +129,64 @@ def generate_image_sample(self, size, max_label=5, objects_number_range=[1, 5]):
                 "keypoints": keypoints,
                 "label": np.random.randint(max_label + 1),
                 "id": str(uuid.uuid1()),
+                "bin": rnd_bin.to_bytes((rnd_bin.bit_length() + 7) // 8, "big"),
             }
         )
         return data
 
     @classmethod
     def generate_toy_dataset(
-        cls, output_folder: str, size: int, image_size: Sequence[int], zfill: int = 5
+        cls,
+        output_folder: str,
+        size: int,
+        image_size: int = 256,
+        zfill: int = 5,
+        suffix: str = "",
+        as_underfolder: bool = False,
+        max_label: int = 5,
+        objects_number_range: Tuple[int, int] = (1, 5),
+        progress_callback: Callable[[dict], None] = None,
     ):
 
         output_folder = Path(output_folder)
+        if as_underfolder:
+            output_folder = output_folder / UnderfolderReader.DATA_SUBFOLDER
         output_folder.mkdir(parents=True, exist_ok=True)
 
         generator = ToyDatasetGenerator()
 
-        for idx in range(size):
+        for idx in pipelime_track(range(size), track_callback=progress_callback):
 
             # Generate sample
-            sample = generator.generate_image_sample([image_size, image_size])
+            sample = generator.generate_image_sample(
+                [image_size, image_size], max_label, objects_number_range
+            )
 
             # Extracts metadata
             metadata = {
-                "bboxes": sample["bboxes"],
-                "keypoints": sample["keypoints"],
-                "label": sample["label"],
-                "id": sample["id"],
+                f"bboxes{suffix}": sample["bboxes"],
+                f"keypoints{suffix}": sample["keypoints"],
+                f"label{suffix}": sample["label"],
+                f"id{suffix}": sample["id"],
+                f"bin{suffix}": int.from_bytes(sample["bin"], "big"),
+                f"index{suffix}": idx,
+                "suffix": suffix,
             }
 
             metadata = json.loads(json.dumps(metadata))
 
             # Naming
             name = str(idx).zfill(zfill)
-            image_name = f"{name}_image.png"
-            mask_name = f"{name}_mask.png"
-            instances_name = f"{name}_inst.png"
-            metadata_name = f"{name}_metadata.yml"
-            keypoints_name = f"{name}_keypoints.txt"
-            bboxes_name = f"{name}_bboxes.npy"
+            image_name = f"{name}_image{suffix}.png"
+            mask_name = f"{name}_mask{suffix}.png"
+            instances_name = f"{name}_inst{suffix}.png"
+            metadata_name = f"{name}_metadata{suffix}.yml"
+            metadataj_name = f"{name}_metadataj{suffix}.json"
+            # metadatat_name = f"{name}_metadatat{suffix}.toml"
+            keypoints_name = f"{name}_keypoints{suffix}.txt"
+            keypointsp_name = f"{name}_keypointsp{suffix}.pickle"
+            bboxes_name = f"{name}_bboxes{suffix}.npy"
+            bin_name = f"{name}_bin{suffix}.bin"
 
             FSToolkit.store_data(str(output_folder / image_name), sample["rgb"])
             FSToolkit.store_data(str(output_folder / mask_name), sample["mask"])
@@ -174,5 +196,11 @@ def generate_toy_dataset(
             FSToolkit.store_data(
                 str(output_folder / keypoints_name), sample["keypoints"]
             )
+            FSToolkit.store_data(
+                str(output_folder / keypointsp_name), sample["keypoints"]
+            )
             FSToolkit.store_data(str(output_folder / bboxes_name), sample["bboxes"])
+            FSToolkit.store_data(str(output_folder / bin_name), sample["bin"])
             FSToolkit.store_data(str(output_folder / metadata_name), metadata)
+            FSToolkit.store_data(str(output_folder / metadataj_name), metadata)
+            # FSToolkit.store_data(str(output_folder / metadatat_name), metadata)
diff --git a/requirements.txt b/requirements.txt
index cbb1180..5c6cb07 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,13 +1,20 @@
-click>=7.1.2
+click>=8.0.3
 rich>=9.9.0
 dictquery
 schema
 numpy
 albumentations>=0.5.2
-choixe
+choixe>=0.1.11
 imageio<=2.10.5
 pydash
 Pillow>=8.0.0
 appdirs
 loguru
-h5py
\ No newline at end of file
+h5py
+pydantic
+fastapi
+requests
+joblib
+deepdiff
+networkx
+redis
\ No newline at end of file
diff --git a/requirements_dev.txt b/requirements_dev.txt
index 9f06e4f..3761e2d 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -1,12 +1,12 @@
-pip==21.3.1
-bump2version==0.5.11
-wheel==0.33.6
-watchdog==0.9.0
-flake8==3.7.8
-tox==3.14.0
-coverage==4.5.4
-Sphinx==4.3.0
-twine==1.14.0
-pytest==4.6.5
-pytest-runner==5.1
-sphinx_rtd_theme==1.0.0
\ No newline at end of file
+bump2version
+wheel
+watchdog
+flake8
+tox
+coverage
+Sphinx
+twine
+pytest
+pytest-runner
+sphinx_rtd_theme
+black
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index 5a22364..5bb69fb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.2
+current_version = 0.1.3
 commit = True
 tag = True
 
diff --git a/setup.py b/setup.py
index 00e079a..10aeb1f 100644
--- a/setup.py
+++ b/setup.py
@@ -4,10 +4,10 @@
 
 from setuptools import setup, find_packages
 
-with open("README.rst") as readme_file:
+with open("README.rst", encoding='UTF-8') as readme_file:
     readme = readme_file.read()
 
-with open("requirements.txt") as requirements_file:
+with open("requirements.txt", encoding='UTF-8') as requirements_file:
     requirements = requirements_file.readlines()
 
 setup_requirements = [
@@ -18,6 +18,10 @@
     "pytest>=3",
 ]
 
+extras_requirements = {
+    'minio': ['minio']
+}
+
 setup(
     author="daniele de gregorio",
     author_email="daniele.degregorio@eyecan.ai",
@@ -40,6 +44,7 @@
         ],
     },
     install_requires=requirements,
+    extras_require=extras_requirements,
     license="GNU General Public License v3",
     long_description=readme,
     include_package_data=True,
@@ -50,6 +55,6 @@
     test_suite="tests",
     tests_require=test_requirements,
     url="https://github.com/eyecan-ai/pipelime",
-    version="0.1.2",
+    version="0.1.3",
     zip_safe=False,
 )


Test patch:
diff --git a/examples/garbage/op_tests.py b/examples/garbage/op_tests.py
deleted file mode 100644
index 6c36830..0000000
--- a/examples/garbage/op_tests.py
+++ /dev/null
@@ -1,132 +0,0 @@
-from numpy.lib.utils import source
-from pipelime.lib import AddOp, Dict2ListOp, FileSystemSample, FilterByQueryOp, PlainSample, Sample, SamplesSequence, ShuffleOp, SplitByQueryOp, SplitsOp, SubsampleOp
-import sys
-from memory_profiler import profile
-from pathlib import Path
-
-# @profile
-
-
-def my_func():
-
-    dataset_folder = Path('/home/daniele/Desktop/experiments/2021-01-28.PlaygroundDatasets/lego_00')
-    fmap = {
-        'image': dataset_folder / 'data/00000_image.jpg',
-        'xray': dataset_folder / 'data/00000_xray.png',
-        'metadata': dataset_folder / 'data/00000_metadata.yml',
-        'pose': dataset_folder / 'data/00000_pose.txt'
-    }
-
-    # s = FileSystemSample(data_map=fmap, preload=True)
-    # s = PlainSample(data={'a': 2})
-    samples = [PlainSample(data={'idx': idx, 'metadata': {'idx': idx, 'name': str(idx)}}) for idx in range(25)]
-    # samples = [FileSystemSample(data_map=fmap) for idx in range(250)]
-    d = SamplesSequence(samples=samples)
-
-    op = AddOp.build_from_dict({
-        'type': 'AddOp',
-        'options': {}
-    })
-
-    op_sub = SubsampleOp.build_from_dict({
-        'type': 'SubsampleOp',
-        'options': {
-            'factor': 0.5
-        }
-    })
-
-    op_shuf = ShuffleOp.build_from_dict({
-        'type': 'ShuffleOp',
-        'options': {
-            'seed': -1
-        }
-    })
-
-    op_splits = SplitsOp.build_from_dict({
-        'type': 'SplitsOp',
-        'options': {
-            'split_map': {
-                'train': 0.5,
-                'val': 0.25,
-                'test': 0.25
-            }
-        }
-    })
-
-    op_d2l = Dict2ListOp.build_from_dict({
-        'type': 'Dict2ListOp',
-        'options': {}
-    })
-
-    op_query = FilterByQueryOp.build_from_dict({
-        'type': 'FilterByQueryOp',
-        'options': {
-            'query': "`metadata.idx` < 5"
-        }
-    })
-
-    op_query2 = SplitByQueryOp.build_from_dict({
-        'type': 'SplitByQueryOp',
-        'options': {
-            'query': "`metadata.idx` < 5"
-        }
-    })
-
-    d = op([d, d, d])
-    d = op_shuf(d)
-    print(len(d))
-
-    a, b = op_query2(d)
-    print(len(a), len(b))
-
-    d = op_query(d)
-    print(len(d))
-
-    d = op_sub(d)
-    print(len(d))
-
-    out = op_splits(d)
-    for k, v in out.items():
-        print(k, len(v))
-
-    out_l = op_d2l(out)
-    for d in out_l:
-        print(len(d))
-
-    # op2 = AddOp()
-    # op3 = ShuffleOp()
-    # op4 = SplitsOp(split_map={'train': 0.6, 'val': 0.18, 'test': 0.22})
-    # op5 = Dict2ListOp()
-
-    # op_q = FilterByQueryOp(query="`metadata.name` CONTAINS '2'")
-
-    # print(len(d))
-    # d = op_q(d)
-    # print(len(d))
-
-    # for s in d:
-    #     print(s)
-
-    # op.print()
-    # op4.print()
-    # op5.print()
-
-    # print(op.match(op4))
-    # d_sum = op([d, d, d, d])
-
-    # print(len(d_sum))
-    # d_split = op4(d_sum)
-    # for k, v in d_split.items():
-    #     print(k, len(v))
-
-    # for s in op5(d_split):
-    #     print(len(s))
-    # a = op.input_port()
-    # b = op.input_port()
-    # print(a.json_schema(0) == b.json_schema(0))
-    # print(len(d))
-    # print(len(d_sum))
-
-
-if __name__ == '__main__':
-    my_func()
diff --git a/examples/garbage/pipe_test_2.py b/examples/garbage/pipe_test_2.py
deleted file mode 100644
index bc6aaa7..0000000
--- a/examples/garbage/pipe_test_2.py
+++ /dev/null
@@ -1,196 +0,0 @@
-
-
-import rich
-from pipelime.lib import AddOp, PlainSample, SamplesSequence, SequenceOpFactory
-from choixe.configurations import XConfig
-
-pipe = {
-    'input': {
-        'X': None
-    },
-    'modules': [
-        {
-            'input': 'X',
-            'output': ['GOOD', 'BAD'],
-            'op': {
-                'type': 'SplitByQueryOp',
-                'options': {
-                    'query': '`metadata.label` == 0'
-                }
-            }
-        },
-        # {
-        #     'input': 'BAD',
-        #     'output': {
-        #         'a': 'BAD_0',
-        #         'b': 'BAD_1'
-        #     },
-        #     'op': {
-        #         'type': 'SplitsOp',
-        #         'options': {
-        #             'split_map': {
-        #                 'a': 0.5,
-        #                 'b': 0.5
-        #             }
-        #         }
-        #     }
-        # },
-        # {
-        #     'input': 'GOOD',
-        #     'output': {
-        #         'a': 'GOOD_0',
-        #         'b': 'GOOD_1'
-        #     },
-        #     'op': {
-        #         'type': 'SplitsOp',
-        #         'options': {
-        #             'split_map': {
-        #                 'a': 0.5,
-        #                 'b': 0.5
-        #             }
-        #         }
-        #     }
-        # },
-        # {
-        #     'input': ['BAD_0', 'GOOD_0'],
-        #     'output': 'train',
-        #     'op': {
-        #         'type': 'AddOp',
-        #         'options': {}
-        #     }
-        # },
-        # {
-        #     'input': ['BAD_1', 'GOOD_1'],
-        #     'output': 'testval',
-        #     'op': {
-        #         'type': 'AddOp',
-        #         'options': {}
-        #     }
-        # },
-        # {
-        #     'input': 'testval',
-        #     'output': 'testval',
-        #     'op': {
-        #         'type': 'ShuffleOp',
-        #         'options': {
-        #             'seed': -1
-        #         }
-        #     }
-        # },
-        # {
-        #     'input': 'testval',
-        #     'output': {
-        #         'a': 'test',
-        #         'b': 'val'
-        #     },
-        #     'op': {
-        #         'type': 'SplitsOp',
-        #         'options': {
-        #             'split_map': {
-        #                 'a': 0.5,
-        #                 'b': 0.5
-        #             }
-        #         }
-        #     }
-        # },
-        # {
-        #     'input': 's',
-        #     'output': 's',
-        #     'op': {
-        #         'type': 'SubsampleOp',
-        #         'options': {
-        #             'factor': 0.5
-        #         }
-        #     }
-        # },
-        # {
-        #     'input': 's',
-        #     'output': {
-        #         'a': 'train',
-        #         'b': 'val',
-        #         'c': 'test',
-        #     },
-        #     'op': {
-        #         'type': 'SplitsOp',
-        #         'options': {
-        #             'split_map': {
-        #                 'a': 0.8,
-        #                 'b': 0.1,
-        #                 'c': 0.1
-        #             }
-        #         }
-        #     }
-        # }
-    ]
-}
-
-cfg = XConfig.from_dict(pipe)
-cfg.save_to('/tmp/cfgs/gino.yml')
-
-pipe = XConfig(filename='/tmp/cfgs/gino.yml')
-rich.print(SequenceOpFactory.FACTORY_MAP)
-
-N = 10
-samples_a = [PlainSample(data={'idx': idx, 'metadata': {'idx': idx, 'name': str(idx), 'label': 0}}) for idx in range(N)]
-samples_b = [PlainSample(data={'idx': idx, 'metadata': {'idx': idx, 'name': str(idx), 'label': 1}}) for idx in range(N)]
-samples = samples_a + samples_b
-
-dataset = SamplesSequence(samples=samples)
-
-
-pipe['input']['X'] = dataset
-
-data_map = {}
-
-
-def check_input(i):
-    if isinstance(i, str):
-        assert i in data_map, f'Input "{i}" not present'
-    elif isinstance(i, list):
-        [check_input(x) for x in i]
-    elif isinstance(i, dict):
-        [check_input(x) for x, _ in i.items()]
-
-
-def parse_input(i):
-    if isinstance(i, str):
-        return data_map[i]
-    elif isinstance(i, list):
-        return [parse_input(x) for x in i]
-    elif isinstance(i, dict):
-        return {k: parse_input(x) for k, x in i.items()}
-
-
-def push_output(o, data):
-    if isinstance(o, str):
-        data_map[o] = data
-    elif isinstance(o, list):
-        for idx, name in enumerate(o):
-            data_map[name] = data[idx]
-    elif isinstance(o, dict):
-        for k, name in o.items():
-            data_map[name] = data[k]
-
-
-for name, d in pipe['input'].items():
-    data_map[name] = d
-
-for m in pipe['modules']:
-    i = m['input']
-    o = m['output']
-    op = SequenceOpFactory.create(m['op'])
-
-    try:
-        check_input(i)
-        i = parse_input(i)
-
-        out = op(i)
-        push_output(o, out)
-    except Exception as e:
-        print(f"Error on module: {m}. {e}")
-        break
-
-for k, v in data_map.items():
-    print(k, len(v))
-    for s in v:
-        print(s)
diff --git a/examples/garbage/test_cfg.yml b/examples/garbage/test_cfg.yml
deleted file mode 100644
index 1e6dd6e..0000000
--- a/examples/garbage/test_cfg.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-__version__: 0.5.1
-transform:
-  __class_fullname__: albumentations.core.composition.Compose
-  additional_targets: {}
-  bbox_params:
-    { 
-      check_each_transform: false,
-      format: yolo,
-      label_fields: null,
-      min_area: 0.0,
-      min_visibility: 0.0,
-    }
-  keypoint_params: null
-  p: 1.0
-  transforms:
-  - __class_fullname__: albumentations.augmentations.transforms.RandomCrop
-    always_apply: false
-    height: 450
-    p: 1.0
-    width: 450
-  - __class_fullname__: albumentations.augmentations.transforms.HorizontalFlip
-    always_apply: false
-    p: 0.5
-  - 
-    {
-      __class_fullname__: albumentations.augmentations.transforms.RandomBrightnessContrast,
-      always_apply: false,
-      brightness_by_max: true,
-      brightness_limit: [-0.2, 0.2],
-      contrast_limit: [-0.2, 0.2],
-      p: 0.2,
-    }
-  - __class_fullname__: pipelime.augmentations.transforms.PadIfNeededV2
-    min_height: 70
-    min_width: 70
diff --git a/tests/conftest.py b/tests/conftest.py
index c72ce98..6b6da0f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -37,7 +37,17 @@ def toy_dataset_small(tmpdir):
         "zfill": zfill,
         "keypoints_format": "xyas",
         "bboxes_format": "pascal_voc",
-        "expected_keys": ["image", "mask", "inst", "keypoints", "metadata", "bboxes"],
+        "expected_keys": [
+            "image",
+            "mask",
+            "inst",
+            "keypoints",
+            "keypointsp",
+            "metadata",
+            "metadataj",
+            "bboxes",
+            "bin",
+        ],
         "root_keys": ["global_meta"],
     }
 
@@ -46,7 +56,7 @@ def toy_dataset_small(tmpdir):
 def toy_h5dataset_small(tmpdir):
     folder = Path(tmpdir.mkdir("toy_h5dataset"))
     datafolder = folder / "data"
-    filename = folder / "h5dataset.h5"
+    # filename = folder / "h5dataset.h5"
     size = 32
     image_size = 256
     zfill = 5
@@ -112,7 +122,17 @@ def filesystem_datasets(data_folder):
                     "should_pass": False,
                 },
             },
-        }
+        },
+        "empty_underfolder": {
+            "folder": Path(data_folder) / "datasets" / "underfolder_empty",
+            "type": "Undefolder",
+            "schemas": {},
+        },
+        "minimnist_underfolder_queries": {
+            "folder": Path(data_folder) / "datasets" / "underfolder_minimnist_queries",
+            "type": "Undefolder",
+            "schemas": {},
+        },
     }
 
 
@@ -121,33 +141,46 @@ def sample_underfolder_minimnist(filesystem_datasets):
     return filesystem_datasets["minimnist_underfolder"]
 
 
+@pytest.fixture(scope="session")
+def sample_underfolder_empty(filesystem_datasets):
+    return filesystem_datasets["empty_underfolder"]
+
+
+@pytest.fixture(scope="session")
+def sample_underfolder_minimnist_queries(filesystem_datasets):
+    return filesystem_datasets["minimnist_underfolder_queries"]
+
+
 @pytest.fixture(scope="session")
 def h5_datasets(data_folder):
     return {
         "minimnist_h5": {
             "filename": Path(data_folder) / "datasets" / "underfolder_minimnist.h5",
             "type": "H5",
-            # 'schemas': {
-            #     'simple': {
-            #         'filename': Path(data_folder) / 'datasets' / 'underfolder_minimnist_schemas' / 'simple.schema',
-            #         'valid': True,
-            #         'should_pass': True
-            #     },
-            #     'deep': {
-            #         'filename': Path(data_folder) / 'datasets' / 'underfolder_minimnist_schemas' / 'deep.schema',
-            #         'valid': True,
-            #         'should_pass': True
-            #     },
-            #     'invalid': {
-            #         'filename': Path(data_folder) / 'datasets' / 'underfolder_minimnist_schemas' / 'invalid.schema',
-            #         'valid': True,
-            #         'should_pass': False
-            #     },
-            #     'bad_file': {
-            #         'filename': Path(data_folder) / 'datasets' / 'underfolder_minimnist_schemas' / 'bad.schema',
-            #         'valid': False,
-            #         'should_pass': False
-            #     }
-            # }
         }
     }
+
+
+@pytest.fixture(scope="session")
+def minio(tmp_path_factory):
+    minio_path = os.environ.get("MINIO_APP")
+    if not minio_path or not Path(minio_path).is_file():
+        yield ""
+        return
+
+    try:
+        import minio  # noqa
+    except ModuleNotFoundError:
+        yield ""
+        return
+
+    from subprocess import Popen
+    from time import sleep
+
+    minio_root = tmp_path_factory.mktemp(".minio")
+    minio_proc = Popen([minio_path, "server", str(minio_root)])
+    sleep(5)
+    yield "minioadmin"
+
+    # teardown
+    minio_proc.terminate()
diff --git a/tests/pipelime/augmentations/conftest.py b/tests/pipelime/augmentations/conftest.py
deleted file mode 100644
index bcd149b..0000000
--- a/tests/pipelime/augmentations/conftest.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-
-@pytest.fixture(scope="session")
-def augmentations_test_configurations(data_folder):
-    configurations_folder = Path(data_folder) / "augmentations"
-    return [
-        {"filename": configurations_folder / "full_custom_augmentations.yml"},
-    ]
diff --git a/tests/pipelime/augmentations/test_factory.py b/tests/pipelime/augmentations/test_factory.py
deleted file mode 100644
index 5502cb8..0000000
--- a/tests/pipelime/augmentations/test_factory.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import albumentations as A
-import numpy as np
-from choixe.configurations import XConfig
-
-from pipelime.augmentations.factory import AugmentationsFactory
-
-
-class TestAugmentationsFactory(object):
-    def _transform_test_image(self, transform):
-
-        img = np.random.uniform(0.0, 1.0, (256, 256, 3))
-        img = transform(image=img)
-
-    def test_creation_from_file(self, augmentations_test_configurations):
-
-        for cfg in augmentations_test_configurations:
-
-            transform = AugmentationsFactory.build_from_file(cfg["filename"])
-            assert transform is not None
-
-            self._transform_test_image(transform)
-
-    def test_creation_from_cfg(self, augmentations_test_configurations):
-
-        for cfg in augmentations_test_configurations:
-
-            xc = XConfig(filename=cfg["filename"]).to_dict()
-            transform = AugmentationsFactory.build_from_dict(xc)
-            assert transform is not None
-
-            self._transform_test_image(transform)
diff --git a/tests/pipelime/cli/test_underfolder.py b/tests/pipelime/cli/test_underfolder.py
index 3ef57f3..8e7ed75 100644
--- a/tests/pipelime/cli/test_underfolder.py
+++ b/tests/pipelime/cli/test_underfolder.py
@@ -1,14 +1,12 @@
 from click.testing import CliRunner
-
-from pipelime.cli.underfolder.operations import operation_split
+import math
+import uuid
+from pathlib import Path
 from pipelime.sequences.readers.filesystem import UnderfolderReader
 
 
 class TestCLIUnderfolderOperationSum:
     def test_sum(self, tmpdir, sample_underfolder_minimnist):
-
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_sum
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -33,10 +31,6 @@ def test_sum(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationSubsample:
     def test_subsample(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_subsample
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -65,12 +59,43 @@ def test_subsample(self, tmpdir, sample_underfolder_minimnist):
 
             assert len(output_reader) == expected_length
 
+    def test_subsample_start(self, tmpdir, sample_underfolder_minimnist):
+        from pipelime.cli.underfolder.operations import operation_subsample
+
+        input_folder = sample_underfolder_minimnist["folder"]
+        input_dataset = UnderfolderReader(folder=input_folder)
+        L = len(input_dataset)
 
-class TestCLIUnderfolderOperationShuffle:
-    def test_shuffle(self, tmpdir, sample_underfolder_minimnist):
+        N = 5
+        start_N = 7
+        exp_N = math.ceil((L - start_N) / N)
+        perc = 1 / N
+        start_perc = 0.1
+        exp_perc = min(int(L * perc), L - int(L * perc))
+
+        for factor, start, exp_length in zip(
+            [perc, N], [start_perc, start_N], [exp_perc, exp_N]
+        ):
+            output_folder = Path(tmpdir.mkdir(str(uuid.uuid1())))
+            options = []
+            options.extend(["-i", str(input_folder)])
+            options.extend(["-o", f"{str(output_folder)}"])
+            options.extend(["-f", f"{factor}"])
+            options.extend(["-s", f"{start}"])
+
+            runner = CliRunner()
+            result = runner.invoke(operation_subsample, options)
+            print(result)
 
-        from pathlib import Path
+            assert result.exit_code == 0
+            output_reader = UnderfolderReader(folder=output_folder, lazy_samples=True)
+            print("OUTPUT", factor, output_folder)
 
+            assert len(output_reader) == exp_length
+
+
+class TestCLIUnderfolderOperationShuffle:
+    def test_shuffle(self, tmpdir, sample_underfolder_minimnist):
         from pipelime.cli.underfolder.operations import operation_shuffle
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -97,10 +122,6 @@ def test_shuffle(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationSplit:
     def test_split(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_split
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -115,7 +136,7 @@ def test_split(self, tmpdir, sample_underfolder_minimnist):
         options = []
         options.extend(["-i", str(input_folder)])
         for k, v in splits_map.items():
-            options.extend(["-s", f"{str(k)}", f"{str(v)}"])
+            options.extend(["-o", f"{str(k)}", "-s", f"{str(v)}"])
 
         runner = CliRunner()
         result = runner.invoke(operation_split, options)
@@ -132,14 +153,10 @@ def test_split(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationFilterByQuery:
     def test_filterbyquery(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_filterbyquery
 
         input_folder = sample_underfolder_minimnist["folder"]
-        input_dataset = UnderfolderReader(folder=input_folder)
+        UnderfolderReader(folder=input_folder)
 
         query = "`metadata.double` <= 10.0 AND `metadata.half` <= 2.5"
         # Samples whose double is <= 10 adn half <= 2.5 are  6! Check it!
@@ -164,10 +181,6 @@ def test_filterbyquery(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationSplitByQuery:
     def test_splitbyquery(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_splitbyquery
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -198,14 +211,10 @@ def test_splitbyquery(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationFilterByScript:
     def test_filterbyscript(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_filterbyscript
 
         input_folder = sample_underfolder_minimnist["folder"]
-        input_dataset = UnderfolderReader(folder=input_folder)
+        UnderfolderReader(folder=input_folder)
 
         # script
         func = ""
@@ -238,10 +247,6 @@ def test_filterbyscript(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationFilterKeys:
     def test_filterkeys(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_filterkeys
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -281,10 +286,6 @@ def test_filterkeys(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationOrderKeys:
     def test_orderkeys(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_orderby
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -318,10 +319,6 @@ def test_orderkeys(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationSplitByValue:
     def test_split_by_value(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_split_by_value
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -354,10 +351,6 @@ def test_split_by_value(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationGroupBy:
     def test_groupby(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import operation_groupby
 
         input_folder = sample_underfolder_minimnist["folder"]
@@ -388,10 +381,6 @@ def test_groupby(self, tmpdir, sample_underfolder_minimnist):
 
 class TestCLIUnderfolderOperationMix:
     def test_mix(self, tmpdir, sample_underfolder_minimnist):
-
-        import uuid
-        from pathlib import Path
-
         from pipelime.cli.underfolder.operations import (
             operation_filterkeys,
             operation_mix,
@@ -406,11 +395,11 @@ def test_mix(self, tmpdir, sample_underfolder_minimnist):
         runner = CliRunner()
         runner.invoke(
             operation_filterkeys,
-            f"-i {input_folder} -o {input_folder_a} -k pose -k label",
+            f'-i "{input_folder}" -o "{input_folder_a}" -k pose -k label',
         )
         runner.invoke(
             operation_filterkeys,
-            f"-i {input_folder} -o {input_folder_b} -k image -k points",
+            f'-i "{input_folder}" -o "{input_folder_b}" -k image -k points',
         )
 
         options = []
@@ -425,3 +414,354 @@ def test_mix(self, tmpdir, sample_underfolder_minimnist):
         assert len(output_reader) == len(input_dataset)
         keys = set(output_reader[0].keys())
         assert keys == {"pose", "label", "image", "points"}
+
+
+class TestCLIUnderfolderOperationRemapKeys:
+    def test_remapkeys(self, tmpdir, sample_underfolder_minimnist):
+        from pipelime.cli.underfolder.operations import operation_remap_keys
+
+        input_folder = sample_underfolder_minimnist["folder"]
+        input_reader = UnderfolderReader(folder=input_folder)
+
+        for remove_missing in [True, False]:
+            keys = {"metadata": "newmetadata", "image": "newimage"}
+
+            # Samples whose double is <= 10 adn half <= 2.5 are  6! Check it!
+            # expected_size = 6
+
+            output_folder = str(Path(tmpdir.mkdir(str(uuid.uuid1()))))
+
+            options = []
+            options.extend(["-i", str(input_folder)])
+            options.extend(["-o", f"{str(output_folder)}"])
+            for k0, k1 in keys.items():
+                options.extend(["-k", k0, k1])
+            if remove_missing:
+                options.extend(["-R"])
+
+            runner = CliRunner()
+            result = runner.invoke(operation_remap_keys, options)
+            print(result)
+
+            assert result.exit_code == 0
+
+            output_reader = UnderfolderReader(folder=output_folder, lazy_samples=True)
+            assert len(output_reader) == len(input_reader)
+            for sample in output_reader:
+                for k0, k1 in keys.items():
+                    assert k0 not in sample.keys()
+                    assert k1 in sample.keys()
+                if remove_missing:
+                    assert len(sample.keys()) == len(keys)
+                else:
+                    assert len(sample.keys()) > len(keys)
+
+
+class TestCLIUnderfolderOperationUpload:
+    def _recursive_folder_diff(self, folder_a, folder_b):
+        import filecmp
+
+        def _diff_files(dcmp: filecmp.dircmp):
+            assert len(dcmp.left_only) == 0
+            assert len(dcmp.right_only) == 0
+            for name in dcmp.common_files:
+                assert filecmp.cmp(
+                    str(Path(dcmp.left) / name),
+                    str(Path(dcmp.right) / name),
+                    shallow=False,
+                )
+            for sub_dcmp in dcmp.subdirs.values():
+                _diff_files(sub_dcmp)
+
+        _diff_files(filecmp.dircmp(folder_a, folder_b))
+
+    def _cli_upload(self, input_folder, output_folder, url_list):
+        from pipelime.cli.underfolder.operations import upload_to_remote
+
+        keys_to_upload = ["numbers", "image", "image_mask", "metadata", "metadatay"]
+
+        # run CLI
+        options = (
+            [
+                "-i",
+                input_folder,
+                "-o",
+                output_folder,
+                "--hardlink",
+            ]
+            + [r for u in url_list for r in ["-r", u]]
+            + [a for k in keys_to_upload for a in ["-k", k]]
+        )
+
+        runner = CliRunner()
+        result = runner.invoke(upload_to_remote, options)
+        assert result.exit_code == 0
+
+        return keys_to_upload
+
+    def _compare_dataset(self, folder_a, folder_b, uploaded_keys):
+        # get data back
+        import numpy as np
+
+        input_reader = UnderfolderReader(folder_a)
+        output_reader = UnderfolderReader(folder_b)
+        for x, y in zip(input_reader, output_reader):
+            for k in uploaded_keys:
+                assert y.metaitem(k).source().suffix == ".remote"
+
+            assert x.keys() == y.keys()
+            for k, v in x.items():
+                if isinstance(v, np.ndarray):
+                    assert np.array_equal(v, y[k])
+                else:
+                    assert v == y[k]
+
+    def test_cli_upload_to_file_remote(self, tmp_path, sample_underfolder_minimnist):
+        from urllib.parse import ParseResult
+
+        # input/output
+        input_folder = str(sample_underfolder_minimnist["folder"])
+
+        output_folder = tmp_path / "output"
+        output_folder.mkdir(parents=True)
+        output_folder = str(output_folder)
+
+        # data lakes
+        remote_root_a = tmp_path / "remote_a"
+        remote_root_a.mkdir(parents=True)
+        remote_root_a = remote_root_a.as_posix()
+
+        remote_root_b = tmp_path / "remote_b"
+        remote_root_b.mkdir(parents=True)
+        remote_root_b = remote_root_b.as_posix()
+
+        uploaded_keys = self._cli_upload(
+            input_folder,
+            output_folder,
+            [
+                ParseResult(
+                    scheme="file",
+                    netloc="localhost",
+                    path=remote_root_a,
+                    params="",
+                    query="",
+                    fragment="",
+                ).geturl(),
+                ParseResult(
+                    scheme="file",
+                    netloc="",
+                    path=remote_root_b,
+                    params="",
+                    query="",
+                    fragment="",
+                ).geturl(),
+            ],
+        )
+
+        # check remote data
+        self._recursive_folder_diff(remote_root_a, remote_root_b)
+
+        # check local data
+        self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+        # make remote_a unavailable and repeat
+        Path(remote_root_a).rename(remote_root_a + "_")
+        self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+        # make remote_b unavailable and repeat
+        Path(remote_root_a + "_").rename(remote_root_a)
+        Path(remote_root_b).rename(remote_root_b + "_")
+        self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+        # make both unavailable and repeat
+        import pytest
+
+        Path(remote_root_a).rename(remote_root_a + "_")
+        with pytest.raises(Exception):
+            self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+    def test_cli_upload_to_s3_remote(
+        self, tmp_path, sample_underfolder_minimnist, minio
+    ):
+        if not minio:
+            from pytest import skip
+
+            skip("MinIO unavailable")
+
+        from urllib.parse import ParseResult
+
+        # input/output
+        input_folder = str(sample_underfolder_minimnist["folder"])
+
+        output_folder = tmp_path / "output"
+        output_folder.mkdir(parents=True)
+        output_folder = str(output_folder)
+
+        # data lakes
+        s3_init_args = {
+            "access_key": f"{minio}",
+            "secret_key": f"{minio}",
+            "secure_connection": False,
+        }
+
+        remote_a_args = {
+            "scheme": "s3",
+            "netloc": "localhost:9000",
+            "path": "remote-a",
+            "params": "",
+            "query": ":".join([k + "=" + str(v) for k, v in s3_init_args.items()]),
+            "fragment": "",
+        }
+        remote_b_args = remote_a_args.copy()
+        remote_b_args["path"] = "remote-b"
+
+        uploaded_keys = self._cli_upload(
+            input_folder,
+            output_folder,
+            [
+                ParseResult(**remote_a_args).geturl(),
+                ParseResult(**remote_b_args).geturl(),
+            ],
+        )
+
+        # get the remote client
+        import pipelime.filesystem.remotes as plr
+        from minio import Minio
+        from minio.commonconfig import CopySource
+
+        s3_remote = plr.create_remote(
+            remote_a_args["scheme"], remote_a_args["netloc"], **s3_init_args
+        )
+        assert isinstance(s3_remote, plr.S3Remote)
+
+        minio_client = s3_remote.client
+        assert isinstance(minio_client, Minio)
+
+        assert minio_client.bucket_exists("remote-a")
+        assert minio_client.bucket_exists("remote-b")
+
+        minio_client.make_bucket("tmpbkt-a")
+        minio_client.make_bucket("tmpbkt-b")
+
+        # download data from buckets and compare
+        def _download_bucket(bkt_name, out_folder):
+            for obj in minio_client.list_objects(bkt_name):
+                minio_client.fget_object(
+                    bkt_name, obj.object_name, str(out_folder / obj.object_name)
+                )
+
+        dwld_remote_a = tmp_path / "dwld_remote_a"
+        dwld_remote_a.mkdir(parents=True)
+        _download_bucket("remote-a", dwld_remote_a)
+
+        dwld_remote_b = tmp_path / "dwld_remote_b"
+        dwld_remote_b.mkdir(parents=True)
+        _download_bucket("remote-b", dwld_remote_b)
+
+        self._recursive_folder_diff(str(dwld_remote_a), str(dwld_remote_b))
+
+        # check local data
+        self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+        # make remote_a unavailable and repeat
+        def _move_objects(source_bkt, target_bkt):
+            objlist = []
+            for obj in minio_client.list_objects(source_bkt):
+                minio_client.copy_object(
+                    target_bkt, obj.object_name, CopySource(source_bkt, obj.object_name)
+                )
+                objlist.append(obj.object_name)
+            for obj in objlist:
+                minio_client.remove_object(source_bkt, obj)
+
+        _move_objects("remote-a", "tmpbkt-a")
+        self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+        # make remote_b unavailable and repeat
+        _move_objects("tmpbkt-a", "remote-a")
+        _move_objects("remote-b", "tmpbkt-b")
+        self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+        # make both unavailable and repeat
+        import pytest
+
+        _move_objects("remote-a", "tmpbkt-a")
+        with pytest.raises(Exception):
+            self._compare_dataset(input_folder, output_folder, uploaded_keys)
+
+    def test_cli_remove_remote(self, tmp_path, sample_underfolder_minimnist):
+        from pipelime.cli.underfolder.operations import remove_remote
+        from urllib.parse import ParseResult
+
+        # input/output
+        input_folder = str(sample_underfolder_minimnist["folder"])
+
+        output_folder_a = tmp_path / "output_a"
+        output_folder_a.mkdir(parents=True)
+        output_folder_a = str(output_folder_a)
+
+        output_folder_b = tmp_path / "output_b"
+        output_folder_b.mkdir(parents=True)
+        output_folder_b = str(output_folder_b)
+
+        # data lakes
+        remote_root_a = tmp_path / "remote_a"
+        remote_root_a.mkdir(parents=True)
+        remote_root_a = remote_root_a.as_posix()
+
+        remote_root_b = tmp_path / "remote_b"
+        remote_root_b.mkdir(parents=True)
+        remote_root_b = remote_root_b.as_posix()
+
+        remote_urls = [
+            ParseResult(
+                scheme="file",
+                netloc="localhost",
+                path=remote_root_a,
+                params="",
+                query="",
+                fragment="",
+            ).geturl(),
+            ParseResult(
+                scheme="file",
+                netloc="",
+                path=remote_root_b,
+                params="",
+                query="",
+                fragment="",
+            ).geturl(),
+        ]
+
+        uploaded_keys = self._cli_upload(
+            input_folder,
+            output_folder_a,
+            remote_urls,
+        )
+
+        # remove remote_root_a
+        options = (
+            [
+                "-i",
+                output_folder_a,
+                "-o",
+                output_folder_b,
+                "--hardlink",
+                "-r",
+                remote_urls[0],
+            ]
+            + [a for k in uploaded_keys for a in ["-k", k]]
+        )
+
+        runner = CliRunner()
+        result = runner.invoke(remove_remote, options)
+        assert result.exit_code == 0
+
+        # check local data
+        self._compare_dataset(input_folder, output_folder_b, uploaded_keys)
+
+        # make remote unavailable
+        Path(remote_root_b).rename(remote_root_b + "_")
+        import pytest
+
+        with pytest.raises(Exception):
+            self._compare_dataset(input_folder, output_folder_b, uploaded_keys)
diff --git a/tests/pipelime/converters/__init__.py b/tests/pipelime/converters/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/pipelime/converters/test_subfolders2underfolder.py b/tests/pipelime/converters/test_subfolders2underfolder.py
new file mode 100644
index 0000000..05c6061
--- /dev/null
+++ b/tests/pipelime/converters/test_subfolders2underfolder.py
@@ -0,0 +1,55 @@
+from pathlib import Path
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.samples import FileSystemSample
+import shutil
+from pipelime.converters.subfolders2underfolder import Subfolders2Underfolder
+
+
+class TestSubfolders2Underfolder:
+    def test_conversion(self, tmpdir, sample_underfolder_minimnist):
+
+        folder = sample_underfolder_minimnist["folder"]
+        dataset = UnderfolderReader(folder=folder)
+
+        output_folder = Path(tmpdir.mkdir("subfolders_dataset"))
+        print("Conversion to: ", output_folder)
+
+        for sample in dataset:
+            sample: FileSystemSample
+            sample_id = sample["metadata"]["sample_id"]
+            split_A = int(sample["metadata"]["double"]) % 3
+            split_B = int(sample["metadata"]["double"]) % 5
+            split_C = "custom  string-to replace"
+            sample_path = (
+                output_folder
+                / str(sample_id)
+                / str(split_A)
+                / str(split_B)
+                / str(split_C)
+            )
+
+            print(sample_path)
+            if not sample_path.exists():
+                sample_path.mkdir(parents=True)
+
+            image_path = sample_path / Path(sample.filesmap["image"]).name
+            image_path_hidden = sample_path / f".{ Path(sample.filesmap['image']).name}"
+
+            shutil.copy(sample.filesmap["image"], image_path)
+            shutil.copy(sample.filesmap["image"], image_path_hidden)
+
+        output_folder_underfolder = Path(tmpdir.mkdir("underfolder_dataset")) / "empty"
+        converter = Subfolders2Underfolder(folder=output_folder)
+        converter.convert(output_folder_underfolder)
+
+        reloaded_undefolder = UnderfolderReader(folder=output_folder_underfolder)
+        assert len(reloaded_undefolder) == len(dataset)
+
+        print("RELOADED UNDERFODLER:")
+        for sample in reloaded_undefolder:
+            print(sample["metadata"]["category"])
+            original_sample_id = int(
+                sample["metadata"]["filename"].split("_", maxsplit=1)[0]
+            )
+            reloadeed_sample_id = int(sample["metadata"]["category"].split("_")[0])
+            assert reloadeed_sample_id == original_sample_id
diff --git a/tests/pipelime/filesystem/test_fstoolkit.py b/tests/pipelime/filesystem/test_fstoolkit.py
new file mode 100644
index 0000000..a247962
--- /dev/null
+++ b/tests/pipelime/filesystem/test_fstoolkit.py
@@ -0,0 +1,23 @@
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+import numpy as np
+
+
+class TestFSToolkit:
+    def test_supported_data_types(self, toy_dataset_small):
+        input_folder = toy_dataset_small["folder"]
+        reader = UnderfolderReader(input_folder, copy_root_files=False)
+
+        for x in reader:
+            assert x["image"].shape == (256, 256, 3)
+            assert x["mask"].shape == (256, 256)
+            assert x["inst"].shape == (256, 256)
+
+            assert x["metadata"] == x["metadataj"]
+            # assert x["metadata"] == x["metadatat"]
+
+            assert np.all(x["keypoints"] == x["keypointsp"])
+            assert np.all(x["keypoints"] == x["metadata"]["keypoints"])
+
+            assert np.all(x["bboxes"] == x["metadata"]["bboxes"])
+
+            assert int.from_bytes(x["bin"], "big") == x["metadata"]["bin"]
diff --git a/tests/pipelime/filesystem/test_remotes.py b/tests/pipelime/filesystem/test_remotes.py
new file mode 100644
index 0000000..e9f1b21
--- /dev/null
+++ b/tests/pipelime/filesystem/test_remotes.py
@@ -0,0 +1,66 @@
+import pipelime.filesystem.remotes as plr
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.samples import FileSystemSample
+
+from filecmp import cmp
+from typing import Mapping
+
+
+class TestRemotes:
+    def _upload_download(
+        self,
+        temp_folder,
+        srcdata_folder: str,
+        remote: plr.BaseRemote,
+        remote_base_path: str,
+    ):
+        # upload
+        source_to_remote: Mapping[str, str] = {}
+        for sample in UnderfolderReader(srcdata_folder, copy_root_files=False):
+            assert isinstance(sample, FileSystemSample)
+            for k, v in sample.filesmap.items():
+                remote_url = remote.upload_file(v, remote_base_path)
+                assert remote_url is not None
+                source_to_remote[v] = remote_url
+
+        # download and compare
+        local_root = temp_folder / "local"
+        local_root.mkdir(parents=True)
+
+        if remote_base_path.startswith('/'):
+            remote_base_path = remote_base_path[1:]
+
+        for original, rm_url in source_to_remote.items():
+            rm, rm_base_path, rm_name = plr.get_remote_and_paths(rm_url)
+            assert isinstance(rm, type(remote))
+            assert isinstance(rm_base_path, str)
+            assert isinstance(rm_name, str)
+            assert rm_base_path == remote_base_path
+
+            local_file = local_root / rm_name
+            assert rm.download_file(local_file, rm_base_path, rm_name)
+            assert cmp(str(local_file), original, shallow=False)
+
+    def test_file_remote(self, toy_dataset_small, tmp_path):
+        file_remote = plr.create_remote("file", "localhost")
+        assert file_remote is not None
+
+        srcdata_folder = toy_dataset_small["folder"]
+
+        remote_root = tmp_path / "remote"
+        remote_root.mkdir(parents=True)
+        remote_root = remote_root.as_posix()
+
+        self._upload_download(tmp_path, srcdata_folder, file_remote, remote_root)
+
+    def test_s3_remote(self, toy_dataset_small, tmp_path):
+        file_remote = plr.create_remote("file", "localhost")
+        assert file_remote is not None
+
+        srcdata_folder = toy_dataset_small["folder"]
+
+        remote_root = tmp_path / "remote"
+        remote_root.mkdir(parents=True)
+        remote_root = remote_root.as_posix()
+
+        self._upload_download(tmp_path, srcdata_folder, file_remote, remote_root)
diff --git a/tests/pipelime/piper/__init__.py b/tests/pipelime/piper/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/pipelime/piper/conftest.py b/tests/pipelime/piper/conftest.py
new file mode 100644
index 0000000..c3b07ba
--- /dev/null
+++ b/tests/pipelime/piper/conftest.py
@@ -0,0 +1,154 @@
+import pydantic
+import pytest
+from pathlib import Path
+
+from pipelime.sequences.validation import SampleSchema
+
+
+@pytest.fixture(scope="session")
+def piper_commands(data_folder):
+
+    base_path = Path(data_folder) / "piper" / "piper_commands"
+
+    def build_python_command(command: str):
+        return f"python {str(base_path / command)}"
+
+    return {
+        "fake_detector": {
+            "command": build_python_command("fake_detector.py"),
+            "valid": True,
+            "exception": None,
+        },
+        "wrong_outputs": {
+            "command": build_python_command("wrong_outputs.py"),
+            "valid": False,
+            "exception": KeyError,
+        },
+        "wrong_inputs": {
+            "command": build_python_command("wrong_inputs.py"),
+            "valid": False,
+            "exception": KeyError,
+        },
+        "not_a_piper": {
+            "command": build_python_command("not_a_piper.py"),
+            "valid": False,
+            "exception": TypeError,
+        },
+        "no_command_name": {
+            "command": build_python_command("no_command_name.py"),
+            "valid": True,
+            "exception": None,
+        },
+    }
+
+
+@pytest.fixture(scope="session")
+def piper_dags(data_folder, piper_commands, tmp_path_factory: pytest.TempPathFactory):
+
+    base_path = Path(data_folder) / "piper" / "dags"
+
+    return {
+        "complex": {
+            "folder": base_path / "complex",
+            "valid": True,
+            "exception": None,
+            "graph": True,
+            "executable": {
+                # If success is TRUE a "final_validation.py" file is needed in the
+                # dag folder
+                "success": True,
+                "executable_placeholders": {
+                    "CUSTOM_COMMAND": piper_commands["fake_detector"]["command"],
+                    "OUTPUT_FOLDER": tmp_path_factory.mktemp("output_folder"),
+                    "FINAL_FOLDER": tmp_path_factory.mktemp("final_folder"),
+                    "SIZE": 10,
+                    "SCHEMA_GENERATED": base_path
+                    / "complex"
+                    / "schema_generated.schema",
+                    "SCHEMA_REMAPPED": base_path / "complex" / "schema_remapped.schema",
+                    "SCHEMA_DETECTED": base_path / "complex" / "schema_detected.schema",
+                },
+                "exception": None,
+            },
+        },
+        "validation_error": {
+            "folder": base_path / "validation_error",
+            "valid": True,
+            "exception": None,
+            "graph": True,
+            "executable": {
+                # If success is TRUE a "final_validation.py" file is needed in the
+                # dag folder
+                "success": False,
+                "executable_placeholders": {
+                    "OUTPUT_FOLDER": tmp_path_factory.mktemp("output_folder"),
+                    "SIZE": 10,
+                    "SCHEMA_GENERATED": base_path
+                    / "validation_error"
+                    / "schema_generated.schema",
+                },
+                "exception": SampleSchema.ValidationError,
+            },
+        },
+        "validation_error_noschema_file": {
+            "folder": base_path / "validation_error_noschema_file",
+            "valid": True,
+            "exception": None,
+            "graph": True,
+            "executable": {
+                # If success is TRUE a "final_validation.py" file is needed in the
+                # dag folder
+                "success": False,
+                "executable_placeholders": {
+                    "OUTPUT_FOLDER": tmp_path_factory.mktemp("output_folder"),
+                    "SIZE": 10,
+                    "SCHEMA_GENERATED": base_path
+                    / "validation_error_noschema_file"
+                    / "schema_generated.schema",
+                },
+                "exception": SampleSchema.ValidationError,
+            },
+        },
+        "no_params": {
+            "folder": base_path / "no_params",
+            "valid": True,
+            "exception": None,
+            "graph": False,
+        },
+        "invalid_params": {
+            "folder": base_path / "invalid_params",
+            "valid": False,
+            "exception": KeyError,
+            "graph": False,
+        },
+        "invalid_nodes": {
+            "folder": base_path / "invalid_nodes",
+            "valid": False,
+            "exception": pydantic.ValidationError,
+            "graph": False,
+        },
+        "invalid_node_foreach": {
+            "folder": base_path / "invalid_node_foreach",
+            "valid": False,
+            "exception": KeyError,
+            "graph": False,
+        },
+        "invalid_node_foreach_do_content": {
+            "folder": base_path / "invalid_node_foreach_do_content",
+            "valid": False,
+            "exception": TypeError,
+            "graph": False,
+        },
+        "invalid_arg_foreach": {
+            "folder": base_path / "invalid_arg_foreach",
+            "valid": False,
+            "exception": KeyError,
+            "graph": False,
+        },
+        "invalid_arg_foreach_content": {
+            "folder": base_path / "invalid_arg_foreach_content",
+            "valid": False,
+            "exception": TypeError,
+            "graph": False,
+        },
+    }
diff --git a/tests/pipelime/piper/test_dag_draw.py b/tests/pipelime/piper/test_dag_draw.py
new file mode 100644
index 0000000..6ca0d0e
--- /dev/null
+++ b/tests/pipelime/piper/test_dag_draw.py
@@ -0,0 +1,53 @@
+import pytest
+from pipelime.pipes.parsers.factory import DAGConfigParserFactory
+from pipelime.pipes.graph import DAGNodesGraph
+
+
+from pytest import TempPathFactory
+
+
+class TestDrawDAGNodesGraph:
+    def test_draw_graph_creation(
+        self, piper_dags: dict, tmp_path_factory: TempPathFactory
+    ):
+
+        something_checked_control = False
+
+        try:
+            from pipelime.pipes.drawing.factory import NodesGraphDrawerFactory
+        except ImportError as e:
+            pytest.skip(f"Backend not installed: {e}")
+
+        backends = NodesGraphDrawerFactory.available_backends()
+
+        for dag_name, item in piper_dags.items():
+
+            folder = item["folder"]
+            valid = item.get("valid", False)
+            graph = item.get("graph", False)
+
+            dag_to_parse = folder / "dag_to_parse.yml"
+            params = folder / "params.yml"
+
+            if valid and graph:
+                something_checked_control = True
+                dag = DAGConfigParserFactory.parse_file(
+                    cfg_file=dag_to_parse,
+                    params_file=params if params.exists() else None,
+                )
+
+                graph = DAGNodesGraph.build_nodes_graph(dag_model=dag)
+
+                for backend in backends:
+                    drawer_folder = tmp_path_factory.mktemp(backend)
+                    drawer = NodesGraphDrawerFactory.create(backend)
+                    formats = drawer.exportable_formats()
+                    for format in formats:
+                        filename = drawer_folder / f"{dag_name}.{format}"
+                        print(backend, drawer_folder, filename)
+                        drawer.draw(graph)
+                        drawer.representation(graph)
+                        drawer.export(graph, filename, format)
+                        drawer.export(graph, filename, None)
+
+        assert something_checked_control
diff --git a/tests/pipelime/piper/test_dag_executors.py b/tests/pipelime/piper/test_dag_executors.py
new file mode 100644
index 0000000..79c27df
--- /dev/null
+++ b/tests/pipelime/piper/test_dag_executors.py
@@ -0,0 +1,139 @@
+from typing import Callable
+from pytest import TempPathFactory
+import pytest
+from pipelime.pipes.parsers.factory import DAGConfigParserFactory
+from pipelime.pipes.graph import DAGNodesGraph
+from pipelime.pipes.executors.naive import NaiveGraphExecutor
+from pipelime.tools.modules import ModulesUtils
+from choixe.configurations import XConfig
+
+
+class TestNaiveGraphExecutor:
+    def test_executor(self, piper_dags: dict, tmp_path_factory: TempPathFactory):
+
+        something_checked_control = False
+
+        for dag_name, item in piper_dags.items():
+            folder = item["folder"]
+            valid = item.get("valid", False)
+            graph = item.get("graph", False)
+            executable = item.get("executable", {})
+
+            executable_success = executable.get("success", False)
+            executable_exception = executable.get("exception", None)
+            executable_placeholders = executable.get("executable_placeholders", {})
+
+            dag_to_parse = folder / "dag_to_parse.yml"
+
+            if executable and valid and graph:
+                something_checked_control = True  # PARANOID!
+
+                run_folder = tmp_path_factory.mktemp("run")
+
+                # Load params with Placeholders
+                params_with_placeholders = folder / "params_with_placeholders.yml"
+                assert params_with_placeholders.exists()
+                cfg = XConfig(params_with_placeholders)
+
+                # Replace params placeholders with values provided by user in conftest
+                cfg.replace_variables_map(executable_placeholders)
+                assert len(cfg.available_placeholders()) == 0
+
+                # write parsed params to file
+                writed_params_file = run_folder / "params.yml"
+                cfg.save_to(writed_params_file)
+
+                # Load Dag with parsed params
+                dag = DAGConfigParserFactory.parse_file(
+                    cfg_file=dag_to_parse,
+                    params_file=writed_params_file,
+                )
+
+                # Build graph from NodesModel
+                graph = DAGNodesGraph.build_nodes_graph(dag_model=dag)
+
+                if executable_success:
+
+                    # Execute graph
+                    executor = NaiveGraphExecutor()
+                    executor.exec(graph, token="")
+
+                    # CHecks for FINAL VALIDATION file
+                    final_validation_file = folder / "final_validation.py"
+                    if final_validation_file.exists():
+
+                        # Extract final validation function
+                        func: Callable = ModulesUtils.load_variable_from_file(
+                            str(final_validation_file),
+                            "final_validation",
+                        )
+                        func(**executable_placeholders)
+
+                else:
+                    with pytest.raises(executable_exception):
+                        # Execute graph
+                        executor = NaiveGraphExecutor()
+                        executor.exec(graph, token="")
+
+        assert something_checked_control
+
+    def test_executor_with_additional_arguments(
+        self, piper_dags: dict, tmp_path_factory: TempPathFactory
+    ):
+
+        something_checked_control = False
+
+        for dag_name, item in piper_dags.items():
+            folder = item["folder"]
+            valid = item.get("valid", False)
+            graph = item.get("graph", False)
+            executable = item.get("executable", {})
+
+            executable_success = executable.get("success", False)
+            executable_exception = executable.get("exception", None)
+            executable_placeholders = executable.get("executable_placeholders", {})
+
+            dag_to_parse = folder / "dag_to_parse.yml"
+
+            if executable and valid and graph:
+                something_checked_control = True  # PARANOID!
+
+                run_folder = tmp_path_factory.mktemp("run")
+
+                # Load params with Placeholders
+                params_with_placeholders = folder / "params_with_placeholders.yml"
+
+                # Load Dag with parsed params
+                dag = DAGConfigParserFactory.parse_file(
+                    cfg_file=dag_to_parse,
+                    params_file=params_with_placeholders,
+                    additional_args=executable_placeholders,
+                )
+
+                # Build graph from NodesModel
+                graph = DAGNodesGraph.build_nodes_graph(dag_model=dag)
+
+                if executable_success:
+
+                    # Execute graph
+                    executor = NaiveGraphExecutor()
+                    executor.exec(graph, token="")
+
+                    # CHecks for FINAL VALIDATION file
+                    final_validation_file = folder / "final_validation.py"
+                    if final_validation_file.exists():
+
+                        # Extract final validation function
+                        func: Callable = ModulesUtils.load_variable_from_file(
+                            str(final_validation_file),
+                            "final_validation",
+                        )
+                        func(**executable_placeholders)
+
+                else:
+                    with pytest.raises(executable_exception):
+                        # Execute graph
+                        executor = NaiveGraphExecutor()
+                        executor.exec(graph, token="")
+
+        assert something_checked_control
diff --git a/tests/pipelime/piper/test_dag_graphs.py b/tests/pipelime/piper/test_dag_graphs.py
new file mode 100644
index 0000000..d2f124c
--- /dev/null
+++ b/tests/pipelime/piper/test_dag_graphs.py
@@ -0,0 +1,55 @@
+import rich
+from pipelime.pipes.parsers.factory import DAGConfigParserFactory
+from pipelime.pipes.graph import DAGNodesGraph
+import networkx as nx
+
+
+class TestDAGNodesGraph:
+    def test_graph_creation(self, piper_dags: dict):
+
+        something_checked_control = False
+
+        for dag_name, item in piper_dags.items():
+
+            folder = item["folder"]
+            valid = item.get("valid", False)
+            graph = item.get("graph", False)
+
+            dag_to_parse = folder / "dag_to_parse.yml"
+            params = folder / "params.yml"
+
+            if valid and graph:
+                something_checked_control = True
+                dag = DAGConfigParserFactory.parse_file(
+                    cfg_file=dag_to_parse,
+                    params_file=params if params.exists() else None,
+                )
+
+                graph = DAGNodesGraph.build_nodes_graph(dag_model=dag)
+                assert isinstance(graph, DAGNodesGraph)
+                assert isinstance(graph.raw_graph, nx.DiGraph)
+                assert len(graph.root_nodes) > 0
+
+                assert len(graph.operations_graph.raw_graph.nodes()) < len(
+                    graph.raw_graph.nodes()
+                )
+                assert len(graph.data_graph.raw_graph.nodes()) < len(
+                    graph.raw_graph.nodes()
+                )
+
+                execution_stack = graph.build_execution_stack()
+                assert len(execution_stack) > 0
+
+                nodes_to_execute = []
+                for layer in execution_stack:
+                    rich.print(layer)
+                    rich.print([x for x in layer])
+                    nodes_to_execute.extend([x for x in layer])
+
+                assert len(nodes_to_execute) == len(
+                    graph.operations_graph.raw_graph.nodes()
+                )
+                for nodetoexec in nodes_to_execute:
+                    assert nodetoexec in graph.operations_graph.raw_graph.nodes()
+
+        assert something_checked_control
diff --git a/tests/pipelime/piper/test_dag_parser.py b/tests/pipelime/piper/test_dag_parser.py
new file mode 100644
index 0000000..31e9d77
--- /dev/null
+++ b/tests/pipelime/piper/test_dag_parser.py
@@ -0,0 +1,51 @@
+import pytest
+from pipelime.pipes.parsers.factory import (
+    DAGConfigParserFactory,
+    DAGConfigParserFactoryConfigurationModel,
+)
+from pipelime.pipes.model import DAGModel
+from choixe.configurations import XConfig
+from deepdiff import DeepDiff
+
+
+class TestDAGSimpleParser:
+    def test_parsing(self, piper_dags: dict):
+
+        for dag_name, item in piper_dags.items():
+
+            folder = item["folder"]
+            valid = item["valid"]
+            exception = item["exception"]
+
+            dag_to_parse = folder / "dag_to_parse.yml"
+            dag_parsed = folder / "dag_parsed.yml"
+            params = folder / "params.yml"
+
+            # builds/validate the generic parser configuration
+            cfg = DAGConfigParserFactoryConfigurationModel(
+                **(XConfig(dag_to_parse).to_dict())
+            )
+
+            if valid:
+                dag = DAGConfigParserFactory.parse_file(
+                    cfg_file=dag_to_parse,
+                    params_file=params if params.exists() else None,
+                )
+
+                assert isinstance(dag, DAGModel)
+
+                if dag_parsed.exists():
+                    parsed_dag = DAGModel(**(XConfig(filename=dag_parsed).to_dict()))
+
+                    assert not DeepDiff(
+                        dag.dict(),
+                        parsed_dag.dict(),
+                        ignore_order=True,
+                    )
+            else:
+
+                with pytest.raises(exception):
+                    DAGConfigParserFactory.parse_file(
+                        cfg_file=dag_to_parse,
+                        params_file=params if params.exists() else None,
+                    )
diff --git a/tests/pipelime/piper/test_piper_commands.py b/tests/pipelime/piper/test_piper_commands.py
new file mode 100644
index 0000000..6685ea9
--- /dev/null
+++ b/tests/pipelime/piper/test_piper_commands.py
@@ -0,0 +1,23 @@
+import pytest
+from pipelime.pipes.piper import Piper
+import rich
+
+
+class TestCustomPiperCommands:
+    def test_custom_commands(self, piper_commands: dict):
+
+        assert len(piper_commands.items()) > 0  # PARANOID!
+
+        for command_name, item in piper_commands.items():
+            command = item["command"]
+            valid = item["valid"]
+            rich.print("Testing\n", item, valid)
+
+            if valid:
+                description = Piper.piper_command_description(command)
+                assert isinstance(description, dict)
+
+            else:
+                exc = item["exception"]
+                with pytest.raises(exc):
+                    description = Piper.piper_command_description(command)
diff --git a/tests/pipelime/piper/test_piper_operations.py b/tests/pipelime/piper/test_piper_operations.py
new file mode 100644
index 0000000..5dbc6e3
--- /dev/null
+++ b/tests/pipelime/piper/test_piper_operations.py
@@ -0,0 +1,103 @@
+from pipelime.pipes.piper import Piper
+import pytest
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from click.testing import CliRunner
+
+
+class TestPiperOperations:
+    def test_operations_infos(self):
+
+        operations = [
+            {"command": "pipelime underfolder remap_keys", "valid": True},
+            {"command": "pipelime underfolder filter_by_query", "valid": True},
+            {"command": "pipelime underfolder filter_by_script", "valid": True},
+            {"command": "pipelime underfolder filter_keys", "valid": True},
+            {"command": "pipelime underfolder group_by", "valid": True},
+            {"command": "pipelime underfolder mix", "valid": True},
+            {"command": "pipelime underfolder order_by", "valid": True},
+            {"command": "pipelime underfolder shuffle", "valid": True},
+            {"command": "pipelime underfolder split", "valid": True},
+            {"command": "pipelime underfolder split_by_query", "valid": True},
+            {
+                "command": "pipelime underfolder split_by_value",
+                "valid": False,
+            },  # ## INVALID PIPER
+            {"command": "pipelime underfolder subsample", "valid": True},
+            {"command": "pipelime underfolder sum", "valid": True},
+            {
+                "command": "pipelime underfolder summary",
+                "valid": False,
+            },  # ## INVALID PIPER
+        ]
+        for op in operations:
+
+            if op["valid"]:
+                description = Piper.piper_command_description(op["command"])
+                assert isinstance(description, dict)
+            else:
+                with pytest.raises(TypeError):
+                    Piper.piper_command_description(op["command"])
+
+
+class TestCLIUnderfolderOperationSplitPiper:
+    def test_split(self, tmpdir, sample_underfolder_minimnist):
+
+        import uuid
+        from pathlib import Path
+
+        from pipelime.cli.underfolder.operations import operation_split
+
+        input_folder = sample_underfolder_minimnist["folder"]
+        input_dataset = UnderfolderReader(folder=input_folder)
+
+        percs = [0.1, 0.2, 0.3, 0.4]
+        output_folders = [
+            str(Path(tmpdir.mkdir(str(uuid.uuid1())))) for _ in range(len(percs))
+        ]
+        splits_map = {k: v for k, v in zip(output_folders, percs)}
+
+        options = []
+        options.extend(["-i", str(input_folder)])
+        for k, v in splits_map.items():
+            options.extend(["-o", f"{str(k)}", "-s", f"{str(v)}"])
+        options.extend([Piper.piper_token_argument(), "token"])
+
+        runner = CliRunner()
+        result = runner.invoke(operation_split, options)
+        print(result)
+
+        assert result.exit_code == 0
+
+        cumulative = 0
+        for output_folder in output_folders:
+            output_reader = UnderfolderReader(folder=output_folder, lazy_samples=True)
+            cumulative += len(output_reader)
+        assert len(input_dataset) == cumulative
+
+
+class TestCLIUnderfolderOperationSumPiper:
+    def test_sum(self, tmpdir, sample_underfolder_minimnist):
+
+        from pathlib import Path
+
+        from pipelime.cli.underfolder.operations import operation_sum
+
+        input_folder = sample_underfolder_minimnist["folder"]
+        input_dataset = UnderfolderReader(folder=input_folder)
+        output_folder = Path(tmpdir.mkdir("test_sum"))
+        N = 5
+
+        options = []
+        for _ in range(N):
+            options.extend(["-i", str(input_folder)])
+        options.extend(["-o", f"{str(output_folder)}"])
+        options.extend([Piper.piper_token_argument(), "token"])
+
+        runner = CliRunner()
+        result = runner.invoke(operation_sum, options)
+        print(result)
+        assert result.exit_code == 0
+
+        output_reader = UnderfolderReader(folder=output_folder, lazy_samples=True)
+        print("OUTPUT", output_folder)
+        assert len(output_reader) == len(input_dataset) * N
diff --git a/tests/pipelime/sequences/api/__init__.py b/tests/pipelime/sequences/api/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/pipelime/sequences/api/test_underfolder_api.py b/tests/pipelime/sequences/api/test_underfolder_api.py
new file mode 100644
index 0000000..857a4d3
--- /dev/null
+++ b/tests/pipelime/sequences/api/test_underfolder_api.py
@@ -0,0 +1,388 @@
+import io
+import shutil
+from fastapi.exceptions import HTTPException
+
+import imageio
+from pipelime.sequences.api.entities import (
+    EntityDataset,
+    EntityPagination,
+    EntitySample,
+    EntitySampleSearchRequest,
+    EntitySampleSearchResponse,
+)
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.api.underfolder import UnderfolderAPI, UnderfolderInterface
+from fastapi.testclient import TestClient
+import pytest
+import rich
+
+from pipelime.tools.dictionaries import DictSearch
+
+
+class TestUnderfolderAPIBasic:
+    def test_api_basic(self, sample_underfolder_minimnist, tmp_path):
+
+        # creates underfolders from minimnist sample data
+        folders_names = ["A", "B"]
+        folders_map = {}
+        readers_map = {}
+        keys_map = {}
+        for folder_name in folders_names:
+            folder_path = tmp_path / folder_name
+            shutil.copytree(sample_underfolder_minimnist["folder"], folder_path)
+            folders_map[folder_name] = str(folder_path)
+            readers_map[folder_name] = UnderfolderReader(folder_path)
+            assert len(readers_map[folder_name]) > 0
+            keys_map[folder_name] = set(readers_map[folder_name][0].keys())
+
+        # creates the api
+        api = UnderfolderAPI(underfolders_map=folders_map, auth_callback=None)
+
+        # creates the API client
+        client = TestClient(api)
+        assert client is not None
+
+        response = client.get("/datasets")
+        assert response.status_code == 200
+
+        # Iterate to retrieved datasets
+        for dataset_name in response.json():
+
+            # Assert the dataset is an Entity
+            rich.print("Dataset", dataset_name)
+            assert dataset_name in folders_names
+            dataset_entity = EntityDataset(**response.json()[dataset_name])
+            assert isinstance(dataset_entity, EntityDataset)
+
+            # Get the wrong dataset
+            with pytest.raises(HTTPException):
+                client.get("/dataset/IMPOSSIBLE_NAME!@/")
+
+            # Get the dataset Entity from the single get_dataset endpoint
+            get_dataset_response = client.get(f"/dataset/{dataset_name}")
+            assert response.status_code == 200
+            dataset_entity2 = EntityDataset(**get_dataset_response.json())
+            assert isinstance(dataset_entity2, EntityDataset)
+
+            # Checks the dataset Entity names are the same
+            assert dataset_entity2.name == dataset_name
+            assert dataset_entity2.name == dataset_name
+
+            # checks for size stored in metadata is the same as the corresponding reader
+            assert dataset_entity.manifest.size == len(readers_map[dataset_name])
+            assert set(dataset_entity.manifest.keys) == keys_map[dataset_name]
+            assert len(dataset_entity.manifest.sample_ids) == len(
+                readers_map[dataset_name]
+            )
+
+            for sample_id in dataset_entity.manifest.sample_ids:
+
+                # wrong dataset name
+                with pytest.raises(HTTPException):
+                    client.get(f"/dataset/IMPOSSIBLE_DATAS3T_nMAE!/{sample_id}")
+
+                # wrong sample id
+                with pytest.raises(HTTPException):
+                    client.get(f"/dataset/{dataset_name}/-1")
+
+                # Get sample response
+                get_sample_response = client.get(f"/dataset/{dataset_name}/{sample_id}")
+                assert get_sample_response.status_code == 200
+
+                # Get sample entity from response
+                sample_entity = EntitySample(**get_sample_response.json())
+                assert isinstance(sample_entity, EntitySample)
+
+                # Check sample entity id is the same as the sample id
+                assert sample_entity.id == sample_id
+
+                # Check sample entity metadata has key as raw dataset
+                for key in sample_entity.metadata:
+                    assert key in keys_map[dataset_name]
+                    assert sample_entity.metadata[key] is not None
+
+                # Check sample entity data has key as raw dataset
+                for key in sample_entity.data:
+                    assert key in keys_map[dataset_name]
+                    assert sample_entity.data[key] is not None
+
+                    # For each raw data fetch corresponding file binaray data
+                    rich.print("Request:", f"/dataset/{dataset_name}/{sample_id}/{key}")
+                    get_sample_data_request = client.get(
+                        f"/dataset/{dataset_name}/{sample_id}/{key}"
+                    )
+
+                    # TEMPORARY TEST FOR FORMAT CHECKS
+                    with pytest.raises(NotImplementedError):
+                        client.get(
+                            f"/dataset/{dataset_name}/{sample_id}/{key}?format=WOW"
+                        )
+
+                    # wrong dataset name
+                    with pytest.raises(HTTPException):
+                        client.get(
+                            f"/dataset/IMPOSSIBLE_DATSAET_NMAE!/{sample_id}/{key}"
+                        )
+
+                    # wrong sample id
+                    with pytest.raises(HTTPException):
+                        client.get(f"/dataset/{dataset_name}/-1/{key}")
+
+                    # wrong item name
+                    with pytest.raises(HTTPException):
+                        client.get(
+                            f"/dataset/{dataset_name}/{sample_id}/IMPOSSIBLE_NAME"
+                        )
+
+                    # Checks the response is a binary data and is oK
+                    assert get_sample_data_request.status_code == 200
+                    assert len(get_sample_data_request.content) > 0
+
+                    # If the data is an image, check the image is readable
+                    if sample_entity.data[key].type == "image":
+                        img = imageio.imread(
+                            io.BytesIO(get_sample_data_request.content)
+                        )
+                        assert img is not None
+
+                # Pick random metadata and modify it
+                assert "metadata" in sample_entity.metadata
+                sample_entity.metadata["metadata"] = {"put": "inception"}
+                sample_entity.metadata["metadatay"] = [1, 2, 3, 4, 5]
+                sample_entity.data = {}
+
+                # wrong dataset name
+                with pytest.raises(HTTPException):
+                    client.put(
+                        f"/dataset/IMPOSSIBLE_DATAS3T_nMAE!/{sample_id}",
+                        json=sample_entity.dict(),
+                    )
+
+                # wrong sample id
+                with pytest.raises(HTTPException):
+                    client.put(f"/dataset/{dataset_name}/-1", json=sample_entity.dict())
+
+                # Update the sample with changed metadata
+                put_sample_request = client.put(
+                    f"/dataset/{dataset_name}/{sample_id}", json=sample_entity.dict()
+                )
+
+                assert put_sample_request.status_code == 200
+
+        for folder_name, folder in folders_map.items():
+            reader = UnderfolderReader(folder)
+            for sample in reader:
+                # checks for updateded metadata
+                assert sample["metadata"]["put"] == "inception"
+                assert len(sample["metadatay"]) == 5
+
+    def test_api_auth(self, sample_underfolder_minimnist, tmp_path):
+
+        # secrets
+        shared_secret = "secr3t!"
+        invalid_secret = "invalid_secret!"
+
+        # Fake hardcoded auth callback
+        def auth_callback(token: str) -> bool:
+            nonlocal shared_secret
+            return token == shared_secret
+
+        # creates underfolders from minimnist sample data
+        folders_names = ["protected_dataset"]
+        folders_map = {}
+        for folder_name in folders_names:
+            folder_path = tmp_path / folder_name
+            shutil.copytree(sample_underfolder_minimnist["folder"], folder_path)
+            folders_map[folder_name] = str(folder_path)
+
+        # creates underfolder api
+        api = UnderfolderAPI(underfolders_map=folders_map, auth_callback=auth_callback)
+
+        # creates the API client
+        client = TestClient(api)
+        assert client is not None
+
+        baked_requests = [
+            {"method": "GET", "url": "/datasets"},
+            {"method": "GET", "url": "/dataset/protected_dataset"},
+            {"method": "GET", "url": "/dataset/protected_dataset/0"},
+            {"method": "GET", "url": "/dataset/protected_dataset/0/image"},
+            {
+                "method": "PUT",
+                "url": "/dataset/protected_dataset/0",
+                "json": EntitySample(
+                    id=0,
+                    metadata={"metadata": {"put": "inception"}},
+                    data={},
+                ).dict(),
+            },
+        ]
+
+        for backed_request in baked_requests:
+
+            # No Auth mechanism set, should fail with 401 UNATHORIZED
+            with pytest.raises(HTTPException) as ex_info:
+                response = client.request(**backed_request)
+            assert isinstance(ex_info.value, HTTPException)
+            assert ex_info.value.status_code == 401
+            rich.print(backed_request, ex_info.value.status_code)
+
+            #  Auth mechanism set but invalid TOKEN
+            with pytest.raises(HTTPException) as ex_info:
+                response = client.request(
+                    **backed_request,
+                    headers={"Authorization": "Bearer " + invalid_secret},
+                )
+            assert isinstance(ex_info.value, HTTPException)
+            rich.print(backed_request, ex_info.value.status_code)
+
+            # Auth mechanism set and valid token
+            response = client.request(
+                **backed_request,
+                headers={"Authorization": "Bearer " + shared_secret},
+            )
+            assert response.status_code == 200
+            rich.print(backed_request, response.status_code)
+
+
+class TestUnderfolderAPISearch:
+    @pytest.mark.parametrize(
+        "search_item",
+        [
+            {
+                "metadata": {
+                    "info": {
+                        "sample_id": f"{DictSearch.KEY_PLACEHOLDER} >= 1 AND {DictSearch.KEY_PLACEHOLDER} <= 9"
+                    },
+                },
+                "expected": 3,
+                "pagination": {"offset": 5, "limit": 3},
+            },
+            {
+                "metadata": {
+                    "info": {
+                        "sample_id": f"{DictSearch.KEY_PLACEHOLDER} >= 1 AND {DictSearch.KEY_PLACEHOLDER} <= 9"
+                    },
+                },
+                "expected": 0,
+                "pagination": {"offset": 20, "limit": 30},
+            },
+            {
+                "metadata": {
+                    "info": {
+                        "sample_id": f"{DictSearch.KEY_PLACEHOLDER} >= 1 AND {DictSearch.KEY_PLACEHOLDER} <= 9"
+                    },
+                },
+                "expected": 3,
+                "pagination": {"offset": 0, "limit": 3},
+            },
+        ],
+    )
+    def test_api_search_samples_basic_pagination(
+        self, sample_underfolder_minimnist_queries, tmp_path, search_item
+    ):
+
+        # creates underfolders from minimnist sample data
+        dataset_name = "searchable"
+        folder = str(tmp_path / dataset_name)
+        shutil.copytree(sample_underfolder_minimnist_queries["folder"], folder)
+
+        # creates the api
+        api = UnderfolderAPI(
+            underfolders_map={dataset_name: folder}, auth_callback=None
+        )
+
+        # creates the API client
+        client = TestClient(api)
+
+        # Search ALL
+        metadata = search_item["metadata"]
+        expected_count = search_item["expected"]
+        pagination = search_item["pagination"]
+
+        search_entity = EntitySampleSearchRequest(
+            proto_sample=EntitySample(id=-1, metadata=metadata, data={}),
+            pagination=EntityPagination(**pagination),
+        )
+
+        search_response = client.post(
+            f"/search/{dataset_name}", json=search_entity.dict()
+        )
+        assert search_response.status_code == 200
+        response = EntitySampleSearchResponse(**search_response.json())
+        assert len(response.samples) == expected_count
+        for sample_entity in response.samples:
+            assert isinstance(sample_entity, EntitySample)
+
+        direct_samples_entities = UnderfolderInterface(
+            "interface", folder
+        ).search_samples(search_entity.proto_sample)
+
+        expected_ids = [x.id for x in direct_samples_entities]
+        for response_entity_sample in response.samples:
+            assert response_entity_sample.id in expected_ids
+
+    def test_api_search_samples_pagination_consistency(
+        self, sample_underfolder_minimnist_queries, tmp_path
+    ):
+
+        # creates underfolders from minimnist sample data
+        dataset_name = "searchable"
+        folder = str(tmp_path / dataset_name)
+        shutil.copytree(sample_underfolder_minimnist_queries["folder"], folder)
+
+        # creates the api
+        api = UnderfolderAPI(
+            underfolders_map={dataset_name: folder}, auth_callback=None
+        )
+
+        reader = UnderfolderReader(folder)
+
+        whole_samples_ids = set([x.id for x in reader])
+
+        # creates the API client
+        client = TestClient(api)
+
+        # Search ALL
+        total = len(reader)
+
+        # NO data request for pagination discovery
+        search_entity = EntitySampleSearchRequest(
+            proto_sample=EntitySample(id=-1, metadata={}, data={}),
+            pagination=None,
+            only_pagination=True,
+        )
+
+        search_response = client.post(
+            f"/search/{dataset_name}", json=search_entity.dict()
+        )
+        assert search_response.status_code == 200
+        response = EntitySampleSearchResponse(**search_response.json())
+
+        # total count
+        total_count = response.pagination.total_count
+        assert total_count == total
+
+        # split request in several paginations modes, and check consistency
+        page_sizes = list(range(1, total_count))
+        for page_size in page_sizes:
+
+            collected_samples = []
+            for offset in range(0, total_count, page_size):
+                search_entity = EntitySampleSearchRequest(
+                    proto_sample=EntitySample(id=-1, metadata={}, data={}),
+                    pagination=EntityPagination(offset=offset, limit=page_size),
+                )
+
+                search_response = client.post(
+                    f"/search/{dataset_name}", json=search_entity.dict()
+                )
+                assert search_response.status_code == 200
+                response = EntitySampleSearchResponse(**search_response.json())
+                collected_samples.extend(response.samples)
+
+            assert len(collected_samples) == total_count
+
+            # checks all ids are present
+            collected_ids = set([x.id for x in collected_samples])
+            assert collected_ids == whole_samples_ids
diff --git a/tests/pipelime/sequences/api/test_underfolder_interface.py b/tests/pipelime/sequences/api/test_underfolder_interface.py
new file mode 100644
index 0000000..08460d9
--- /dev/null
+++ b/tests/pipelime/sequences/api/test_underfolder_interface.py
@@ -0,0 +1,252 @@
+import io
+import shutil
+import rich
+import pytest
+from pipelime.sequences.api.entities import EntitySample, EntitySampleData
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.api.underfolder import UnderfolderInterface
+from pipelime.tools.dictionaries import DictSearch
+
+
+class TestUnderfolderInterface:
+    def test_interface_interaction(self, sample_underfolder_minimnist, tmp_path):
+
+        # creates a copy of minimnist dataset under a new folder
+        folder = tmp_path / "dataset"
+        shutil.copytree(sample_underfolder_minimnist["folder"], folder)
+        print("folder:", folder)
+
+        # base reader
+        dataset = UnderfolderReader(folder=folder)
+
+        # corresponding interface
+        interface = UnderfolderInterface("inter", folder, allowed_keys=None)
+
+        assert len(dataset) > 0
+        ref_sample = dataset[0]
+        keys = list(ref_sample.keys())
+
+        # retrieve dataset entity
+        dataset_entity = interface.get_dataset()
+
+        # assert manifest consistency
+        assert len(dataset) == dataset_entity.manifest.size
+        assert set(keys) == set(dataset_entity.manifest.keys)
+        assert len(dataset_entity.manifest.sample_ids) == len(dataset)
+
+        # store metadata keys
+        meta_keys = []
+
+        # match ids
+        for sample in dataset:
+            assert sample.id in dataset_entity.manifest.sample_ids
+
+            sample_entity = interface.get_sample(sample.id)
+            assert sample_entity is not None
+
+            assert sample_entity.id == sample.id
+
+            for key in keys:
+                assert key in sample_entity.metadata or key in sample_entity.data
+
+                if key in sample_entity.data:
+                    sample_data_entity = sample_entity.data[key]
+                    assert isinstance(sample_data_entity, EntitySampleData)
+
+                    data, mimetype = interface.get_sample_data(
+                        sample_id=sample.id, item_name=key
+                    )
+                    assert isinstance(data, io.BytesIO)
+
+                if key in sample_entity.metadata:
+                    meta_keys.append(key)
+                    sample_entity.metadata[key] = {"overwrite": True}
+
+                    interface.put_sample(sample.id, sample_entity)
+
+        new_dataset = UnderfolderReader(folder=folder)
+        for sample in new_dataset:
+            for key in meta_keys:
+                assert key in sample
+                assert "overwrite" in sample[key]
+                assert sample[key]["overwrite"]
+
+    @pytest.mark.parametrize(
+        "allowed_item",
+        [
+            {
+                "allowed_keys": None,
+                "to_write": ["metadata", "metadatay"],
+                "allowed": True,
+            },
+            {
+                "allowed_keys": ["metadata"],
+                "to_write": ["metadata", "metadatay"],
+                "allowed": False,
+            },
+            {
+                "allowed_keys": ["metadatay"],
+                "to_write": ["metadata", "metadatay"],
+                "allowed": False,
+            },
+            {
+                "allowed_keys": ["metadata"],
+                "to_write": ["metadata"],
+                "allowed": True,
+            },
+            {
+                "allowed_keys": ["metadatay"],
+                "to_write": ["metadatay"],
+                "allowed": True,
+            },
+        ],
+    )
+    def test_keys_protection(
+        self,
+        sample_underfolder_minimnist,
+        tmp_path,
+        allowed_item,
+    ):
+
+        allowed_keys = allowed_item["allowed_keys"]
+        to_write = allowed_item["to_write"]
+        allowed = allowed_item["allowed"]
+
+        # creates a copy of minimnist dataset under a new folder
+        folder = tmp_path / "dataset"
+        shutil.copytree(sample_underfolder_minimnist["folder"], folder)
+        print("folder:", folder)
+
+        # base reader
+        dataset = UnderfolderReader(folder=folder)
+
+        # corresponding interface
+        interface = UnderfolderInterface("inter", folder, allowed_keys=allowed_keys)
+
+        assert len(dataset) > 0
+        ref_entity = interface.get_sample(dataset[0].id)
+
+        metadata = {}
+        for key in to_write:
+            metadata[key] = {"overwrite": True}
+
+        sample_entity = EntitySample(id=ref_entity.id, data={}, metadata=metadata)
+
+        rich.print(allowed, "metadata", metadata, "allowed", allowed_keys)
+        if allowed:
+            interface.put_sample(ref_entity.id, sample_entity)
+        else:
+            with pytest.raises(PermissionError):
+                interface.put_sample(ref_entity.id, sample_entity)
+
+    # TEST SEARCH SAMPLES
+    @pytest.mark.parametrize(
+        "search_item",
+        [
+            {"metadata": {"info": {}}, "expected": 20},
+            {"metadata": {"info": {"sample_id": "== 0"}}, "expected": 1},
+            {"metadata": {"info": {"sample_id": "> 0"}}, "expected": 19},
+            {"metadata": {"info": {"sample_id": ">= 0"}}, "expected": 20},
+            {"metadata": {"info": {"double": "<= 20"}}, "expected": 11},
+            {
+                "metadata": {"info": {"double": "<= 20", "sample_id": "< 6"}},
+                "expected": 6,
+            },
+            {
+                "metadata": {
+                    "info": {"double": "<= 20", "sample_id": "< 6", "half": "== 0"}
+                },
+                "expected": 1,
+            },
+            {
+                "metadata": {"categories": {"main": "== 'category_odd'"}},
+                "expected": 10,
+            },
+            {
+                "metadata": {"categories": {"main": "!= 'category_odd'"}},
+                "expected": 10,
+            },
+            {
+                "metadata": {"categories": {"main": "LIKE 'category*'"}},
+                "expected": 20,
+            },
+            {
+                "metadata": {"categories": {"main": "LIKE '*odd'"}},
+                "expected": 10,
+            },
+            {
+                "metadata": {
+                    "info": {"sample_id": "< 5"},
+                    "categories": {"main": "LIKE '*odd'"},
+                },
+                "expected": 2,
+            },
+            {
+                "metadata": {
+                    "categories": {"others": "CONTAINS 'alpha'"},
+                },
+                "expected": 10,
+            },
+            {
+                "metadata": {
+                    "info": None,
+                    "categories": {"others": "CONTAINS 'alpha'"},
+                },
+                "expected": 10,
+            },
+            {
+                "metadata": {
+                    "categories": {
+                        "others": f"{DictSearch.KEY_PLACEHOLDER} CONTAINS 'alpha' and {DictSearch.KEY_PLACEHOLDER} CONTAINS 'beta'"
+                    },
+                },
+                "expected": 10,
+            },
+            {
+                "metadata": {
+                    "categories": {
+                        "others": f"{DictSearch.KEY_PLACEHOLDER} CONTAINS 'alpha' and {DictSearch.KEY_PLACEHOLDER} CONTAINS 'zeta'"
+                    },
+                },
+                "expected": 0,
+            },
+            {
+                "metadata": {
+                    "categories": {
+                        "others": f"{DictSearch.KEY_PLACEHOLDER} CONTAINS 'alpha' OR {DictSearch.KEY_PLACEHOLDER} CONTAINS 'zeta'"
+                    },
+                },
+                "expected": 20,
+            },
+            {
+                "metadata": {
+                    "info": {
+                        "sample_id": f"{DictSearch.KEY_PLACEHOLDER} >= 1 AND {DictSearch.KEY_PLACEHOLDER} <= 2"
+                    },
+                },
+                "expected": 2,
+            },
+        ],
+    )
+    def test_search_samples(
+        self, sample_underfolder_minimnist_queries, tmp_path, search_item
+    ):
+
+        # creates underfolders from minimnist sample data
+        dataset_name = "searchable"
+        folder = str(tmp_path / dataset_name)
+        shutil.copytree(sample_underfolder_minimnist_queries["folder"], folder)
+
+        # creates the api
+        interface = UnderfolderInterface(dataset_name, folder)
+
+        # Search ALL
+        metadata = search_item["metadata"]
+        expected_count = search_item["expected"]
+
+        rich.print("Search item", search_item)
+        samples_entities = interface.search_samples(
+            EntitySample(id=-1, metadata=metadata, data={})
+        )
+
+        assert len(samples_entities) == expected_count
diff --git a/tests/pipelime/sequences/conftest.py b/tests/pipelime/sequences/conftest.py
index a163aa2..2121f2f 100644
--- a/tests/pipelime/sequences/conftest.py
+++ b/tests/pipelime/sequences/conftest.py
@@ -31,7 +31,7 @@ def generate_plain_samples(
                     "deep": {"super_deep": 0, "groupby_field": int(i / group_each)},
                 },
             }
-            samples.append(PlainSample(data=data))
+            samples.append(PlainSample(data=data, id=i))
 
         return samples
 
diff --git a/tests/pipelime/sequences/io/test_h5.py b/tests/pipelime/sequences/io/test_h5.py
index e35a645..0b7128f 100644
--- a/tests/pipelime/sequences/io/test_h5.py
+++ b/tests/pipelime/sequences/io/test_h5.py
@@ -62,7 +62,7 @@ def test_reader(self, h5_datasets: dict, tmpdir):
                 assert isinstance(sample, H5Sample)
                 assert len(sample.keys()) > 0
 
-            out_filename = output_folder / "output_dataset.h5"
+            out_filename = output_folder / (dataset_name + ".h5")
             writer = H5Writer(filename=out_filename)
             _plug_test_writer(writer)
             writer(reader)
@@ -77,8 +77,6 @@ def test_reader(self, h5_datasets: dict, tmpdir):
                 for key in sample:
                     assert key in proto_sample
 
-            out_filename.unlink()
-
     def test_reader_copy_root(self, h5_datasets: dict, tmpdir):
 
         for dataset_name, cfg in h5_datasets.items():
@@ -126,6 +124,7 @@ def test_flush(self, h5_datasets):
                 sample: H5Sample
                 for key in sample.keys():
                     temp = sample[key]
+                    assert temp is not None
                     assert sample.is_cached(key)
 
             reader.flush()
@@ -147,6 +146,7 @@ def test_copy(self, h5_datasets):
                 keys = list(sample.keys())[:2]
                 for key in keys:
                     temp = sample[key]
+                    assert temp is not None
                     assert sample.is_cached(key)
 
                 sample_copy = sample.copy()
diff --git a/tests/pipelime/sequences/io/test_mixed.py b/tests/pipelime/sequences/io/test_mixed.py
index d128cf8..d55cf57 100644
--- a/tests/pipelime/sequences/io/test_mixed.py
+++ b/tests/pipelime/sequences/io/test_mixed.py
@@ -1,5 +1,3 @@
-
-
 from pathlib import Path
 from pipelime.sequences.writers.h5 import H5Writer
 from pipelime.sequences.readers.filesystem import UnderfolderReader
@@ -8,7 +6,6 @@
 
 
 class TestH5Underfolder:
-
     def test_cascade_huh(self, h5_datasets: dict, tmpdir):
 
         output_folder = Path(tmpdir.mkdir("test_h5reader_huh"))
@@ -21,11 +18,13 @@ def test_cascade_huh(self, h5_datasets: dict, tmpdir):
                 for key in sample.keys():
                     assert sample[key] is not None
 
-            underfolder_0 = output_folder / '0'
+            underfolder_0 = output_folder / "0"
 
             underfolder_writer = UnderfolderWriter(folder=underfolder_0)
             underfolder_writer(h5_reader)
-            underfolder_reader = UnderfolderReader(folder=underfolder_0, copy_root_files=True)
+            underfolder_reader = UnderfolderReader(
+                folder=underfolder_0, copy_root_files=True
+            )
 
             assert len(underfolder_reader) == len(h5_reader)
             assert len(underfolder_reader) > 0
@@ -41,9 +40,11 @@ def test_cascade_huh(self, h5_datasets: dict, tmpdir):
                     assert key in sample_h5
 
                 for key in sample_h5.keys():
-                    assert h5_reader.is_root_key(key) == underfolder_reader.is_root_key(key)
+                    assert h5_reader.is_root_key(key) == underfolder_reader.is_root_key(
+                        key
+                    )
 
-            h5filename_0 = output_folder / 'h5.py'
+            h5filename_0 = output_folder / "h5.py"
             h5_writer = H5Writer(filename=h5filename_0)
 
             h5_writer(underfolder_reader)
@@ -61,19 +62,20 @@ def test_cascade_uhu(self, toy_dataset_small: dict, tmpdir):
 
         output_folder = Path(tmpdir.mkdir("test_h5reader_uhu"))
 
-        folder = toy_dataset_small['folder']
-        keys = toy_dataset_small['expected_keys']
-        root_keys = toy_dataset_small['root_keys']
+        folder = toy_dataset_small["folder"]
+        keys = toy_dataset_small["expected_keys"]
+        root_keys = toy_dataset_small["root_keys"]
+        print(keys, root_keys)
 
         underfolder_reader = UnderfolderReader(folder=folder, copy_root_files=True)
 
-        h5_0 = output_folder / 'h5py'
+        h5_0 = output_folder / "h5py"
 
         h5_writer = H5Writer(filename=h5_0)
         h5_writer(underfolder_reader)
 
         h5_reader = H5Reader(filename=h5_0)
 
-        underfolder_0 = output_folder / '0'
+        underfolder_0 = output_folder / "0"
         underfolder_writer = UnderfolderWriter(folder=underfolder_0)
         underfolder_writer(h5_reader)
diff --git a/tests/pipelime/sequences/io/test_summary.py b/tests/pipelime/sequences/io/test_summary.py
index fec1f58..a9e6d6e 100644
--- a/tests/pipelime/sequences/io/test_summary.py
+++ b/tests/pipelime/sequences/io/test_summary.py
@@ -2,6 +2,7 @@
 from pipelime.sequences.readers.summary import ReaderSummary, TypeInfo
 import numpy as np
 
+
 class TestSummary:
     def test_summary(self, toy_dataset_small):
         path = toy_dataset_small["folder"]
@@ -18,10 +19,11 @@ def test_summary(self, toy_dataset_small):
         key = sorted(all_keys)[0]
         assert iteminfo.name == key
         assert iteminfo.count == len(reader)
-        assert np.ndarray in iteminfo.typeinfo.types 
+        assert np.ndarray in iteminfo.typeinfo.types
         assert iteminfo.root_item == (key in root_keys)
         assert iteminfo.encoding == "npy"
 
+
 class TestTypeInfo:
     def test_typeinfo(self):
         # Compatible but different shape
@@ -37,7 +39,7 @@ def test_typeinfo(self):
         b = TypeInfo(np.ndarray, shape=[100, 100], dtype="uint8")
         c = a + b
         assert c.types == {np.ndarray}
-        assert c.shape == None
+        assert c.shape is None
         assert c.dtype == "uint8"
 
         # None shape
@@ -45,7 +47,7 @@ def test_typeinfo(self):
         b = TypeInfo(np.ndarray, shape=[200, 100, 100], dtype="uint8")
         c = a + b
         assert c.types == {np.ndarray}
-        assert c.shape == None
+        assert c.shape is None
         assert c.dtype == "uint8"
 
         # Incompatible dtypes
@@ -53,13 +55,13 @@ def test_typeinfo(self):
         b = TypeInfo(np.ndarray, shape=None, dtype="uint8")
         c = a + b
         assert c.types == {np.ndarray}
-        assert c.shape == None
-        assert c.dtype == None
+        assert c.shape is None
+        assert c.dtype is None
 
         # Incompatible types
         a = TypeInfo(list, shape=None, dtype="uint8")
         b = TypeInfo(np.ndarray, shape=[200, 100, 100], dtype="uint8")
         c = a + b
         assert c.types == {np.ndarray, list}
-        assert c.shape == None
-        assert c.dtype == "uint8"
\ No newline at end of file
+        assert c.shape is None
+        assert c.dtype == "uint8"
diff --git a/tests/pipelime/sequences/io/test_filesystem.py b/tests/pipelime/sequences/io/test_underfolder.py
similarity index 51%
rename from tests/pipelime/sequences/io/test_filesystem.py
rename to tests/pipelime/sequences/io/test_underfolder.py
index 4a4d921..44a0c0d 100644
--- a/tests/pipelime/sequences/io/test_filesystem.py
+++ b/tests/pipelime/sequences/io/test_underfolder.py
@@ -1,15 +1,20 @@
 import uuid
 from itertools import product
 from pathlib import Path
-
+import os
 import numpy as np
 from choixe.spooks import Spook
-
+import rich
+import networkx as nx
 from pipelime.sequences.operations import OperationFilterKeys
 from pipelime.sequences.readers.base import BaseReader, ReaderTemplate
-from pipelime.sequences.readers.filesystem import UnderfolderReader
-from pipelime.sequences.samples import FilesystemItem, FileSystemSample, Sample
-from pipelime.sequences.writers.filesystem import UnderfolderWriter
+from pipelime.sequences.readers.filesystem import (
+    UnderfolderLinksPlugin,
+    UnderfolderReader,
+)
+from pipelime.sequences.samples import FileSystemItem, FileSystemSample, Sample
+from pipelime.sequences.writers.filesystem import UnderfolderWriter, UnderfolderWriterV2
+import pytest
 
 
 def _plug_test(reader: BaseReader):
@@ -49,12 +54,12 @@ def test_reader(self, toy_dataset_small):
 
                 for key in keys:
                     assert key in sample
-                    assert isinstance(sample.metaitem(key), FilesystemItem)
+                    assert isinstance(sample.metaitem(key), FileSystemItem)
 
                 if copy:
                     for key in root_keys:
                         assert key in sample
-                        assert isinstance(sample.metaitem(key), FilesystemItem)
+                        assert isinstance(sample.metaitem(key), FileSystemItem)
 
 
 class TestUnderfolderReaderWriterTemplating(object):
@@ -131,7 +136,7 @@ def test_filtered_reader_writer_without_explicit_template(
 
         folder = toy_dataset_small["folder"]
         keys = toy_dataset_small["expected_keys"]
-        root_keys = toy_dataset_small["root_keys"]
+        # root_keys = toy_dataset_small["root_keys"]
 
         reader = UnderfolderReader(folder=folder, copy_root_files=True)
         op = OperationFilterKeys(keys=keys[0], negate=False)
@@ -151,7 +156,7 @@ def test_filtered_reader_writer_without_explicit_template(
     def test_writer_copy_correct_extension(self, toy_dataset_small, tmpdir_factory):
 
         folder = toy_dataset_small["folder"]
-        keys = toy_dataset_small["expected_keys"]
+        # keys = toy_dataset_small["expected_keys"]
 
         for lazy_samples, copy_files, use_symlinks in product([True, False], repeat=3):
             reader = UnderfolderReader(folder=folder, lazy_samples=lazy_samples)
@@ -192,8 +197,8 @@ def test_reader_writer_content(self, toy_dataset_small, tmpdir_factory):
         # ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️
 
         folder = toy_dataset_small["folder"]
-        keys = toy_dataset_small["expected_keys"]
-        root_keys = toy_dataset_small["root_keys"]
+        # keys = toy_dataset_small["expected_keys"]
+        # root_keys = toy_dataset_small["root_keys"]
 
         combo_items = [
             {"copy_files": True, "use_symlinks": False},
@@ -205,7 +210,7 @@ def test_reader_writer_content(self, toy_dataset_small, tmpdir_factory):
         for combo in combo_items:
 
             reader = UnderfolderReader(folder=folder, copy_root_files=True)
-            template = reader.get_reader_template()
+            reader.get_reader_template()
 
             print("\nCombo", combo)
             writer_folder = Path(tmpdir_factory.mktemp(str(uuid.uuid1())))
@@ -218,7 +223,7 @@ def test_reader_writer_content(self, toy_dataset_small, tmpdir_factory):
             writer(reader)
 
             re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
-            re_template = re_reader.get_reader_template()
+            re_reader.get_reader_template()
 
             for idx in range(len(re_reader)):
                 data = reader[idx]["image"]
@@ -256,7 +261,6 @@ def test_reader_writer_force_copy(self, toy_dataset_small, tmpdir_factory):
 class TestUnderfolderWriterMultiprocessing(object):
     def test_reader_writer_force_copy(self, toy_dataset_small, tmpdir_factory):
         folder = toy_dataset_small["folder"]
-        image_key = "image"
 
         reader = UnderfolderReader(folder=folder, copy_root_files=True)
 
@@ -275,6 +279,7 @@ def test_reader_writer_force_copy(self, toy_dataset_small, tmpdir_factory):
 
             re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
             re_sample = re_reader[0]
+            assert isinstance(re_sample, Sample)
             assert len(reader) == len(re_reader)
             for sample_id in range(len(re_reader)):
                 sample = reader[sample_id]
@@ -285,7 +290,7 @@ def test_reader_writer_force_copy(self, toy_dataset_small, tmpdir_factory):
 class TestUnderfolderReaderMultiprocessing(object):
     def test_reader_writer_force_copy(self, toy_dataset_small, tmpdir_factory):
         folder = toy_dataset_small["folder"]
-        image_key = "image"
+        # image_key = "image"
 
         workers_options = [1, 0, 1, 2, 3, 4]
 
@@ -305,8 +310,272 @@ def test_reader_writer_force_copy(self, toy_dataset_small, tmpdir_factory):
 
             re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
             re_sample = re_reader[0]
+            assert isinstance(re_sample, Sample)
             assert len(reader) == len(re_reader)
             for sample_id in range(len(re_reader)):
                 sample = reader[sample_id]
                 resample = re_reader[sample_id]
                 assert sample["metadata"]["id"] == resample["metadata"]["id"]
+
+
+class TestUnderfolderWriterSymlinks(object):
+    def test_symlinks_not_broken(self, toy_dataset_small, tmpdir_factory):
+        folder = toy_dataset_small["folder"]
+
+        os.chdir(folder.parent)
+        folder = folder.name
+
+        reader = UnderfolderReader(folder=folder)
+        writer_folder = Path(tmpdir_factory.mktemp(str(uuid.uuid1())))
+        writer = UnderfolderWriter(
+            folder=writer_folder,
+            copy_files=True,
+            use_symlinks=True,
+        )
+        writer(reader)
+
+        # check for broken symlinks
+        import platform
+
+        i_am_on_windows = platform.system() == "Windows"
+
+        re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
+        for sample in re_reader:
+            for k, v in sample.filesmap.items():
+                path = Path(v)
+                assert i_am_on_windows or path.is_symlink()
+                assert path.is_file()
+
+
+class TestUnderfolderLinkPlugin:
+    def test_linking(self, tmpdir, plain_samples_sequence_generator):
+
+        N = 32
+        samples = plain_samples_sequence_generator("", N)
+
+        # retrieve keys
+        assert len(samples) > 0
+        keys = list(samples[0].keys())
+
+        # creates root keys as copy of normal keys
+        root_keys = [f"{k}_root" for k in keys]
+        for sample in samples:
+            for rkey in root_keys:
+                sample[rkey] = sample[rkey.replace("_root", "")]
+
+        # Split datasets in N datasets for each key, write them to disk
+        subsamples = {}
+        writers = {}
+        subfolders = {}
+        for key, root_key in zip(keys, root_keys):
+            subsamples[key] = OperationFilterKeys([key, root_key])(samples)
+            subfolders[key] = Path(tmpdir.mkdir(key))
+            writers[key] = UnderfolderWriter(
+                folder=subfolders[key], root_files_keys=root_keys
+            )
+            writers[key](subsamples[key])
+            rich.print(key, "Writing", subfolders[key])
+
+        # Creates a tree to represent the linking structure among the subfolders
+        g = nx.full_rary_tree(3, len(keys))
+        for u, v, a in g.edges(data=True):
+            key_source = keys[u]
+            key_target = keys[v]
+            folder_source = str(subfolders[key_source])
+            folder_target = str(subfolders[key_target])
+            UnderfolderLinksPlugin.link_underfolders(folder_source, folder_target)
+            print(u, v, keys[u], keys[v])
+
+        # Creates the reader of the Root Underfolder
+        root_reader = UnderfolderReader(folder=subfolders[keys[0]])
+        assert len(root_reader) > 0
+
+        # Write the merged dataset for debug
+        merged_folder = Path(tmpdir.mkdir("_merged"))
+        writer = UnderfolderWriter(folder=merged_folder, root_files_keys=root_keys)
+        writer(root_reader)
+        rich.print("Merged output to", merged_folder)
+
+        # Checks for keys/root_keys consistency
+        for key in keys:
+            assert not root_reader.is_root_key(key)
+        for rkey in root_keys:
+            assert root_reader.is_root_key(rkey)
+
+        # Checks for merged keys presence
+        ref_sample = root_reader[0]
+        loaded_keys = set(list(ref_sample.keys()))
+        assert loaded_keys == set(keys) | set(root_keys)
+
+        # Content alignment test
+        for sample in root_reader:
+            assert int(sample["idx"]) == sample["number"]
+            assert sample["odd"] == (sample["number"] % 2 == 1)
+            assert np.isclose(sample["fraction"], sample["number"] / 1000.0)
+            assert sample["reverse_number"] == N - sample["number"]
+            assert sample["metadata"]["even"] == (sample["number"] % 2 == 0)
+            assert sample["metadata"]["N"] == sample["number"]
+
+        # Connect a Leaf to the root underfolder in order to build a cycle
+        dg = nx.to_directed(g)
+        leafs = [
+            x for x in dg.nodes() if dg.out_degree(x) == 1 and dg.in_degree(x) == 1
+        ]
+        assert len(leafs) > 0
+        folder_source = str(subfolders[keys[leafs[0]]])
+        folder_target = str(subfolders[keys[0]])
+        UnderfolderLinksPlugin.link_underfolders(folder_source, folder_target)
+
+        # Checks for cycle when UnderfolderReader is called
+        with pytest.raises(RuntimeError):
+            UnderfolderReader(folder=subfolders[keys[0]])
+
+
+class TestUnderfolderWriterV2(object):
+    def _read_write_data(self, source_folder, **writer_kwargs):
+        os.chdir(source_folder.parent)
+        source_folder = source_folder.name
+
+        reader = UnderfolderReader(folder=source_folder)
+        writer = UnderfolderWriterV2(**writer_kwargs)
+        writer(reader)
+
+    def test_deep_copy(self, toy_dataset_small, tmpdir_factory):
+        source_folder = toy_dataset_small["folder"]
+        writer_folder = str(Path(tmpdir_factory.mktemp(str(uuid.uuid1()))))
+
+        self._read_write_data(
+            source_folder,
+            folder=writer_folder,
+            file_handling=UnderfolderWriterV2.FileHandling.ALWAYS_COPY_FROM_DISK,
+            copy_mode=UnderfolderWriterV2.CopyMode.DEEP_COPY,
+        )
+
+        re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
+        for sample in re_reader:
+            for k, v in sample.filesmap.items():
+                path = Path(v)
+                assert not path.is_symlink()
+                assert path.is_file()
+                assert path.stat().st_nlink == 1
+
+    def test_symlink(self, toy_dataset_small, tmpdir_factory):
+        source_folder = toy_dataset_small["folder"]
+        writer_folder = str(Path(tmpdir_factory.mktemp(str(uuid.uuid1()))))
+
+        self._read_write_data(
+            source_folder,
+            folder=writer_folder,
+            file_handling=UnderfolderWriterV2.FileHandling.ALWAYS_COPY_FROM_DISK,
+            copy_mode=UnderfolderWriterV2.CopyMode.SYM_LINK,
+        )
+
+        # check for broken symlinks
+        import platform
+
+        i_am_on_windows = platform.system() == "Windows"
+
+        re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
+        for sample in re_reader:
+            for k, v in sample.filesmap.items():
+                path = Path(v)
+                assert i_am_on_windows or path.is_symlink()
+                assert path.is_file()
+                assert path.stat().st_nlink == 1
+
+    def test_hardlink(self, toy_dataset_small, tmpdir_factory):
+        source_folder = toy_dataset_small["folder"]
+        writer_folder = str(Path(tmpdir_factory.mktemp(str(uuid.uuid1()))))
+
+        self._read_write_data(
+            source_folder,
+            folder=writer_folder,
+            file_handling=UnderfolderWriterV2.FileHandling.ALWAYS_COPY_FROM_DISK,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+        )
+
+        re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
+        for sample in re_reader:
+            for k, v in sample.filesmap.items():
+                path = Path(v)
+                assert not path.is_symlink()
+                assert path.is_file()
+                assert path.stat().st_nlink == 2
+
+    def test_always_write_from_cache(self, toy_dataset_small, tmpdir_factory):
+        source_folder = toy_dataset_small["folder"]
+        writer_folder = str(Path(tmpdir_factory.mktemp(str(uuid.uuid1()))))
+
+        self._read_write_data(
+            source_folder,
+            folder=writer_folder,
+            file_handling=UnderfolderWriterV2.FileHandling.ALWAYS_WRITE_FROM_CACHE,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+        )
+
+        re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
+        for sample in re_reader:
+            for k, v in sample.filesmap.items():
+                path = Path(v)
+                assert not path.is_symlink()
+                assert path.is_file()
+                assert path.stat().st_nlink == 1
+
+    def test_copy_if_not_cached(self, toy_dataset_small, tmpdir_factory):
+        source_folder = toy_dataset_small["folder"]
+        writer_folder = str(Path(tmpdir_factory.mktemp(str(uuid.uuid1()))))
+
+        self._read_write_data(
+            source_folder,
+            folder=writer_folder,
+            file_handling=UnderfolderWriterV2.FileHandling.COPY_IF_NOT_CACHED,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+        )
+
+        re_reader = UnderfolderReader(folder=writer_folder, copy_root_files=True)
+        for sample in re_reader:
+            for k, v in sample.filesmap.items():
+                path = Path(v)
+                assert not path.is_symlink()
+                assert path.is_file()
+                assert path.stat().st_nlink == 2
+
+    @pytest.mark.parametrize("num_threads", [0, 1, 2, 4, -1])
+    def test_changed_ext(self, toy_dataset_small, tmpdir_factory, num_threads):
+        source_folder = toy_dataset_small["folder"]
+        os.chdir(source_folder.parent)
+        source_folder = source_folder.name
+        reader = UnderfolderReader(folder=source_folder)
+
+        reader_template = reader.get_reader_template()
+        changed_keys = []
+        old_ext = "png"
+        new_ext = "jpg"
+        for k, ext in reader_template.extensions_map.items():
+            if ext == old_ext:
+                reader_template.extensions_map[k] = new_ext
+                changed_keys.append(k)
+
+        writer_folder = str(Path(tmpdir_factory.mktemp(str(uuid.uuid1()))))
+        writer = UnderfolderWriterV2(
+            folder=writer_folder,
+            file_handling=UnderfolderWriterV2.FileHandling.ALWAYS_COPY_FROM_DISK,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+            reader_template=reader_template,
+            num_workers=num_threads,
+        )
+        writer(reader)
+
+        for sample in reader:
+            for k, v in sample.filesmap.items():
+                path = Path(v)
+                if k in changed_keys:
+                    assert num_threads == -1 or num_threads > 1 or sample.is_cached(k)
+                    assert not path.is_symlink()
+                    assert path.is_file()
+                    assert path.stat().st_nlink == 1
+                else:
+                    assert not sample.is_cached(k)
+                    assert not path.is_symlink()
+                    assert path.is_file()
+                    assert path.stat().st_nlink == 2
diff --git a/tests/pipelime/sequences/io/underfolder/test_plugin_stage.py b/tests/pipelime/sequences/io/underfolder/test_plugin_stage.py
new file mode 100644
index 0000000..d5efe4b
--- /dev/null
+++ b/tests/pipelime/sequences/io/underfolder/test_plugin_stage.py
@@ -0,0 +1,43 @@
+from pipelime.sequences.readers.filesystem import (
+    UnderfolderReader,
+    UnderfolderStagePlugin,
+)
+from pipelime.sequences.stages import (
+    StageCompose,
+    StageIdentity,
+    StageKeysFilter,
+    StageRemap,
+)
+from pathlib import Path
+from pipelime.sequences.writers.filesystem import UnderfolderWriter
+import rich
+
+
+class TestUnderfolderStagePlugin:
+    def test_stages(self, tmpdir, plain_samples_sequence_generator):
+
+        N = 32
+        samples = plain_samples_sequence_generator("", N)
+
+        stages = [
+            StageIdentity(),
+            StageRemap(remap={"idx": "a"}, remove_missing=False),
+            StageRemap(remap={"number": "b"}, remove_missing=False),
+            StageRemap(remap={"reverse_number": "c"}, remove_missing=False),
+            StageRemap(remap={"odd": "d"}, remove_missing=False),
+            StageKeysFilter(key_list=["a", "b"], negate=False),
+        ]
+        stage = StageCompose(stages=stages)
+
+        output_folder = Path(tmpdir.mkdir("staged_underfolder"))
+        UnderfolderWriter(folder=output_folder)(samples)
+        UnderfolderStagePlugin.set_stages(source_folder=output_folder, stage=stage)
+
+        rich.print("Output written to:", output_folder)
+        reader = UnderfolderReader(folder=output_folder)
+        for sample in reader:
+            rich.print(list(sample.keys()))
+            assert "a" in sample
+            assert "b" in sample
+            assert "c" not in sample
+            assert "d" not in sample
diff --git a/tests/pipelime/sequences/streams/__init__.py b/tests/pipelime/sequences/streams/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/pipelime/sequences/streams/test_underfolder.py b/tests/pipelime/sequences/streams/test_underfolder.py
new file mode 100644
index 0000000..01e24de
--- /dev/null
+++ b/tests/pipelime/sequences/streams/test_underfolder.py
@@ -0,0 +1,258 @@
+from pathlib import Path
+import imageio
+from pipelime.sequences.operations import OperationResetIndices
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.samples import FileSystemSample, Sample
+from pipelime.sequences.streams.base import ItemConverter
+from pipelime.sequences.streams.underfolder import UnderfolderStream
+import io
+import pytest
+import shutil
+import numpy as np
+
+from pipelime.sequences.writers.filesystem import UnderfolderWriter
+from pipelime.tools.idgenerators import IdGeneratorUUID
+
+
+class TestUnderfolderStreams:
+    def _create_dataset(sample_underfolder_minimnist, tmp_path):
+        folder = tmp_path / "dataset"
+        shutil.copytree(sample_underfolder_minimnist["folder"], folder)
+        return folder
+
+    def test_stream_empty(self, sample_underfolder_empty):
+
+        view = UnderfolderStream(sample_underfolder_empty["folder"])
+        assert len(view) == 0
+
+        assert view.manifest()["size"] == 0
+
+    def test_stream_read(self, sample_underfolder_minimnist, tmp_path):
+        folder = sample_underfolder_minimnist["folder"]
+
+        dataset = UnderfolderReader(folder=folder)
+        view = UnderfolderStream(folder)
+        assert len(view) > 0
+        assert len(dataset) == len(view)
+
+        manifest = view.manifest()
+        assert manifest is not None
+        assert isinstance(manifest, dict)
+        assert "keys" in manifest
+        keys = manifest["keys"]
+
+        for sample_id in range(len(view)):
+
+            sample = view.get_sample(sample_id)
+            assert isinstance(sample, Sample)
+
+            for key in keys:
+                assert key in sample
+
+                if key == "image":
+                    for format in ItemConverter.IMAGE_FORMATS:
+                        image, mimetype = view.get_data(sample_id, key, format=format)
+                        assert isinstance(image, io.BytesIO)
+
+                    with pytest.raises(ValueError):
+                        view.get_data(sample_id, key, format="invalid")
+                    with pytest.raises(ValueError):
+                        ItemConverter.item_to_data(sample[key], "invalid")
+
+                elif key == "label" or key == "points":
+                    for format in ItemConverter.MATRIX_FORMATS:
+                        label, mimetype = view.get_data(sample_id, key, format=format)
+                        assert isinstance(label, dict)
+                        assert "data" in label
+
+                elif key == "metadata" or key == "metadatay":
+                    for format in ItemConverter.DICT_FORMATS:
+                        metadata, mimetype = view.get_data(
+                            sample_id, key, format=format
+                        )
+                        assert isinstance(metadata, dict)
+
+            key = "IMPOSSIBLE_KEY!"
+            assert key not in sample
+
+            with pytest.raises(KeyError):
+                view.get_item(sample_id, key)
+
+    def test_stream_write(self, sample_underfolder_minimnist, tmp_path):
+        folder = tmp_path / "dataset"
+        shutil.copytree(sample_underfolder_minimnist["folder"], folder)
+        print("folder:", folder)
+        dataset = UnderfolderReader(folder=folder)
+        view = UnderfolderStream(folder)
+        assert len(view) > 0
+        assert len(dataset) == len(view)
+
+        for sample_id in view.get_sample_ids():
+
+            view.set_data(
+                sample_id, "new_metadata", {"data": [1, 2, 3.0], "flag": True}, "dict"
+            )
+            view.set_data(sample_id, "new_matrix", {"data": [1, 2, 3.0]}, "matrix")
+            view.set_data(
+                sample_id,
+                "new_points",
+                {"data": [1, 2, 3.0, 4, 5, 6]},
+                "matrix",
+            )
+
+            image = np.random.rand(28, 28, 3)
+            image_bytes = io.BytesIO()
+            imageio.imwrite(image_bytes, image, format="jpg")
+            view.set_data(sample_id, "new_image", image_bytes, "jpg")
+
+        view.flush()
+        for sample in view.reader:
+            assert isinstance(sample, FileSystemSample)
+            for key in sample:
+                assert not sample.is_cached(key)
+
+        dataset = UnderfolderReader(folder=folder)
+        for sample in dataset:
+            assert "new_metadata" in sample
+            assert "new_matrix" in sample
+            assert "new_points" in sample
+            assert "new_image" in sample
+
+    def test_stream_write_uuid(self, sample_underfolder_minimnist, tmp_path):
+        temp_reader = UnderfolderReader(sample_underfolder_minimnist["folder"])
+        filtered = OperationResetIndices(generator=IdGeneratorUUID())(temp_reader)
+
+        folder = tmp_path / "dataset"
+        UnderfolderWriter(
+            folder=folder,
+            extensions_map=temp_reader.get_reader_template().extensions_map,
+            root_files_keys=temp_reader.get_reader_template().root_files_keys,
+        )(filtered)
+
+        dataset = UnderfolderReader(folder=folder)
+        view = UnderfolderStream(folder)
+        assert len(view) > 0
+        assert len(dataset) == len(view)
+
+        for sample_id in view.get_sample_ids():
+
+            view.set_data(
+                sample_id, "new_metadata", {"data": [1, 2, 3.0], "flag": True}, "dict"
+            )
+            view.set_data(sample_id, "new_matrix", {"data": [1, 2, 3.0]}, "matrix")
+            view.set_data(
+                sample_id,
+                "new_points",
+                {"data": [1, 2, 3.0, 4, 5, 6]},
+                "matrix",
+            )
+
+            image = np.random.rand(28, 28, 3)
+            image_bytes = io.BytesIO()
+            imageio.imwrite(image_bytes, image, format="jpg")
+            view.set_data(sample_id, "new_image", image_bytes, "jpg")
+
+        view.flush()
+        for sample in view.reader:
+            assert isinstance(sample, FileSystemSample)
+            for key in sample:
+                assert not sample.is_cached(key)
+
+        dataset = UnderfolderReader(folder=folder)
+        for sample in dataset:
+            assert "new_metadata" in sample
+            assert "new_matrix" in sample
+            assert "new_points" in sample
+            assert "new_image" in sample
+
+    def test_stream_notallowed_keys(self, sample_underfolder_minimnist, tmp_path):
+        folder = tmp_path / "dataset"
+        shutil.copytree(sample_underfolder_minimnist["folder"], folder)
+
+        allowed_keys = ["image", "points"]
+        view = UnderfolderStream(folder, allowed_keys=allowed_keys)
+        assert len(view) > 0
+
+        for sample_id in range(len(view)):
+
+            with pytest.raises(PermissionError):
+                view.set_data(
+                    sample_id, "metadata", {"data": [1, 2, 3.0], "flag": True}, "dict"
+                )
+
+            with pytest.raises(PermissionError):
+                view.set_data(sample_id, "matrix", {"data": [1, 2, 3.0]}, "matrix")
+
+            view.set_data(
+                sample_id,
+                "points",
+                {"data": [1, 2, 3.0, 4, 5, 6]},
+                "matrix",
+            )
+
+            image = np.random.rand(28, 28, 3)
+            image_bytes = io.BytesIO()
+            imageio.imwrite(image_bytes, image, format="jpg")
+            view.set_data(sample_id, "image", image_bytes, "jpg")
+
+    def test_stream_write_format(self, sample_underfolder_minimnist, tmp_path):
+        folder = tmp_path / "dataset"
+        shutil.copytree(sample_underfolder_minimnist["folder"], folder)
+        print("folder:", folder)
+        view = UnderfolderStream(folder)
+
+        ##############################
+        # Write without further specification means DEFAULT FILE EXTENSION
+        ##############################
+        for sample_id in range(len(view)):
+            view.set_data(
+                sample_id, "new_metadata", {"data": [1, 2, 3.0], "flag": True}, "dict"
+            )
+
+        dataset = UnderfolderReader(folder=folder)
+        for sample in dataset:
+            assert isinstance(sample, FileSystemSample)
+            filepath = Path(sample.filesmap["new_metadata"])
+            default_extension = UnderfolderWriter.DEFAULT_EXTENSION
+            assert filepath.suffix.replace(".", "") == default_extension
+
+        ##############################
+        # Write with specification, overwriting default extension multiple times
+        ##############################
+        for file_format in ["json", "yml", "pkl"]:
+            view.add_extensions_map({"last_metadata": file_format})
+            for sample_id in range(len(view)):
+                view.set_data(
+                    sample_id,
+                    "last_metadata",
+                    {"data": [1, 2, 3.0], "flag": True},
+                    "dict",
+                )
+
+            dataset = UnderfolderReader(folder=folder)
+            for sample in dataset:
+                assert "last_metadata" in sample
+                assert isinstance(sample, FileSystemSample)
+                filepath = Path(sample.filesmap["last_metadata"])
+                assert filepath.suffix.replace(".", "") == file_format
+                assert sample["last_metadata"]["flag"]
+
+    def test_stream_write_rootkeys(self, sample_underfolder_minimnist, tmp_path):
+        folder = tmp_path / "dataset"
+        shutil.copytree(sample_underfolder_minimnist["folder"], folder)
+        print("folder:", folder)
+        view = UnderfolderStream(folder)
+
+        view.add_root_files_keys(["new_metadata"])
+        ##############################
+        # Write without further specification means DEFAULT FILE EXTENSION
+        ##############################
+        for sample_id in range(len(view)):
+            view.set_data(
+                sample_id, "new_metadata", {"data": [1, 2, 3.0], "flag": True}, "dict"
+            )
+            break
+
+        dataset = UnderfolderReader(folder=folder)
+        for sample in dataset:
+            assert "new_metadata" in sample
diff --git a/tests/pipelime/sequences/test_operations.py b/tests/pipelime/sequences/test_operations.py
index 3cd9315..402b49b 100644
--- a/tests/pipelime/sequences/test_operations.py
+++ b/tests/pipelime/sequences/test_operations.py
@@ -1,6 +1,8 @@
 import functools
 import hashlib
+from itertools import count
 from typing import Dict, Optional, Sequence
+import uuid
 
 import pydash
 import pytest
@@ -15,9 +17,10 @@
     OperationFilterKeys,
     OperationGroupBy,
     OperationIdentity,
-    OperationMix,
+    # OperationMix,
     OperationOrderBy,
     OperationPort,
+    OperationRemapKeys,
     OperationResetIndices,
     OperationShuffle,
     OperationSplitByQuery,
@@ -78,8 +81,8 @@ def test_sum(self, plain_samples_sequence_generator):
                 assert isinstance(out, SamplesSequence)
                 assert N * D == len(out)
             else:
-                with pytest.raises(AssertionError):
-                    out = op(datasets)
+                assert op.input_port().is_valid_data(datasets)
+                out = op(datasets)
 
 
 class TestOperationSubsample(object):
@@ -113,6 +116,71 @@ def test_subsample(self, plain_samples_sequence_generator):
                         if F == 1.0:
                             assert len(out) == N
 
+    def test_subsample_start(self, plain_samples_sequence_generator):
+
+        sizes = [32, 10, 128, 16, 1]
+        factors_int = [2, 10, 20, 1]
+        factors_float = [0.1, -0.1, 10.2, 1.0]
+        starts_int = [-1, 0, 1, 2, 10, 0.5]
+        starts_float = [-0.1, 0.0, 0.1, 0.5, 0.9]
+
+        # test with integer inputs
+        for F in factors_int:
+            for N in sizes:
+                for S in starts_int:
+                    dataset = plain_samples_sequence_generator("d0_", N)
+
+                    if S < 0:
+                        with pytest.raises(Exception):
+                            op = OperationSubsample(factor=F, start=S)
+                        continue
+
+                    if isinstance(S, float):
+                        with pytest.raises(Exception):
+                            op = OperationSubsample(factor=F, start=S)
+                        continue
+
+                    op = OperationSubsample(factor=F, start=S)
+                    _plug_test(op)
+
+                    if N > 0:
+                        out = op(dataset)
+
+                        assert isinstance(out, SamplesSequence)
+
+                        if F > 1 and N > 1:
+                            assert len(out) < N
+                        if F == 1 and S == 0:
+                            assert len(out) == N
+
+        # test with float inputs
+        for F in factors_float:
+            for N in sizes:
+                for S in starts_float:
+                    dataset = plain_samples_sequence_generator("d0_", N)
+
+                    if S < 0:
+                        with pytest.raises(Exception):
+                            op = OperationSubsample(factor=F, start=S)
+                        continue
+
+                    op = OperationSubsample(factor=F, start=S)
+                    _plug_test(op)
+
+                    if N > 0:
+                        out = op(dataset)
+
+                        assert isinstance(out, SamplesSequence)
+
+                        op = OperationSubsample(factor=F, start=S)
+                        _plug_test(op)
+
+                        if 0.0 < F < 1.0:
+                            assert len(out) < N
+
+                        if F == 1.0 and S == 0.0:
+                            assert len(out) == N
+
 
 class TestOperationIdentity(object):
     def test_subsample(self, plain_samples_sequence_generator):
@@ -168,6 +236,12 @@ def test_reset(self, plain_samples_sequence_generator):
             dataset = plain_samples_sequence_generator("d0_", N)
             dataset_clone = plain_samples_sequence_generator("d0_", N)
 
+            # Generate common uuid for samples ensuring they are the same
+            for sample_index, _ in enumerate(dataset):
+                common_id = str(uuid.uuid1())
+                dataset[sample_index].id = common_id
+                dataset_clone[sample_index].id = common_id
+
             op = OperationResetIndices(generator=generator)
             _plug_test(op)
             out = op(dataset)
@@ -516,6 +590,13 @@ def check_sample_onthefly(sample, sequence):
 
 
 class TestMappableOperation(object):
+    class CountCallback:
+        def __init__(self):
+            self.counter = 0
+
+        def __call__(self, data: dict) -> None:
+            self.counter += 1
+
     class OperationPlusOne(MappableOperation):
         def __init__(self, key: str, **kwargs) -> None:
             super().__init__(**kwargs)
@@ -543,11 +624,17 @@ def test_mappable_operation(self, plain_samples_sequence_generator, pb, workers)
         key = "number"
 
         dataset = plain_samples_sequence_generator("d0_", N)
-        op = self.OperationPlusOne(key, progress_bar=pb, num_workers=workers)
+        callback = self.CountCallback()
+        op = self.OperationPlusOne(
+            key, progress_bar=pb, num_workers=workers, progress_callback=callback
+        )
 
         out = op(dataset)
         _plug_test(op, check_serialization=False)
 
+        # Check callback was called the right number of times
+        assert callback.counter == len(dataset)
+
         expected_numbers = [x[key] + 1 for x in dataset]
         out_numbers = [x[key] for x in out]
 
@@ -556,10 +643,12 @@ def test_mappable_operation(self, plain_samples_sequence_generator, pb, workers)
         for x, y in zip(expected_numbers, out_numbers):
             assert x == y, (expected_numbers, out_numbers)
 
-        op = self.OperationLessThan10(key)
+        callback = self.CountCallback()
+        op = self.OperationLessThan10(key, progress_callback=callback)
         out = op(dataset)
 
         assert len(out) == 10
+        assert callback.counter == len(dataset)
 
 
 class TestOperationStage(object):
@@ -582,3 +671,27 @@ def test_operation_stage(self, plain_samples_sequence_generator, pb, workers):
         for x in out:
             assert key_2 in x
             assert key not in x
+
+
+class TestOperationRemapKeys(object):
+    def test_operation_stage(self, plain_samples_sequence_generator):
+        N = 32
+        key = "number"
+        key_2 = "number_2"
+
+        dataset = plain_samples_sequence_generator("d0_", N)
+
+        for remove_missing in [True, False]:
+            op = OperationRemapKeys(remap={key: key_2}, remove_missing=remove_missing)
+            _plug_test(op, check_serialization=False)
+            out = op(dataset)
+
+            assert isinstance(out, SamplesSequence)
+            assert N == len(out)
+            for x in out:
+                assert key_2 in x
+                assert key not in x
+                if remove_missing:
+                    assert len(x.keys()) == 1
+                else:
+                    assert len(x.keys()) > 1
diff --git a/tests/pipelime/sequences/test_proxies.py b/tests/pipelime/sequences/test_proxies.py
new file mode 100644
index 0000000..df5d066
--- /dev/null
+++ b/tests/pipelime/sequences/test_proxies.py
@@ -0,0 +1,272 @@
+from pipelime.sequences.samples import Sample, PlainSample, SamplesSequence
+from pipelime.sequences.stages import StageKeysFilter
+import pipelime.sequences.proxies as sp
+
+from typing import Sequence, Tuple, Set
+
+
+class SideEffectSample(PlainSample):
+    def get_bypass(self, key):
+        return self._data.get(key, None)
+
+    def __getitem__(self, key):
+        if self._data[key] > 0:
+            self._data[key] *= -1
+        return self._data[key]
+
+    def merge(self, other: "SideEffectSample") -> "SideEffectSample":
+        new_data = self._data.copy()
+        new_data.update(other._data.copy())
+        return SideEffectSample(data=new_data, id=self.id)
+
+    def copy(self):
+        return SideEffectSample(data=self._data.copy(), id=self.id)
+
+
+class TestSequenceProxy:
+    def _make_random_source(self, max_idx: int):
+        import random
+
+        def rnd_fn():
+            return random.randint(0, 100)
+
+        return SamplesSequence(
+            samples=[
+                PlainSample(
+                    {
+                        "idx": i,
+                        "data_1": rnd_fn(),
+                        "data_2": rnd_fn(),
+                        "data_3": rnd_fn(),
+                        "data_4": rnd_fn(),
+                    }
+                )
+                for i in range(0, max_idx)
+            ],
+            stage=StageKeysFilter(["idx", "data_1", "data_2", "data_3"]),
+        )
+
+    def _make_cached_sequence(
+        self, sample_cache: sp.CachedSamplesSequence.SampleCache, max_idx: int
+    ) -> Tuple[sp.CachedSamplesSequence, Set[str], Set[str], Set[str], Set[str]]:
+        source_sseq = SamplesSequence(
+            samples=[
+                SideEffectSample(
+                    {"data_1": i, "data_2": i * 2, "data_3": i * 3, "data_4": i * 4}
+                )
+                for i in range(1, max_idx)
+            ],
+            stage=StageKeysFilter(["data_1", "data_2", "data_3"]),
+        )
+        cached_sseq = sp.CachedSamplesSequence(
+            source=source_sseq,
+            sample_cache=sample_cache,
+            forced_keys={"data_1"},
+            stage=StageKeysFilter(["data_1", "data_2"]),
+        )
+
+        forced_keys = {"data_1"}
+        last_stage_keys = forced_keys | {"data_2"}
+        hidden_stage_keys = last_stage_keys | {"data_3"}
+        all_keys = hidden_stage_keys | {"data_4"}
+        return cached_sseq, all_keys, hidden_stage_keys, last_stage_keys, forced_keys
+
+    def _check_samples(
+        self,
+        kset: Set[str],
+        ksrc: Set[str],
+        sseq: Sequence[Sample],
+        negative_keys: Set[str],
+    ):
+        for s in sseq:
+            for k in kset:
+                if k in negative_keys:
+                    assert s.get_bypass(k) < 0  # type: ignore
+                else:
+                    assert s.get_bypass(k) > 0  # type: ignore
+            for k in ksrc - kset:
+                assert s.get_bypass(k) is None  # type: ignore
+
+    def _check_cache_buffer(
+        self, sseq: sp.CachedSamplesSequence, max_idx: int, cached_idx: Set[int]
+    ):
+        for idx in range(1, max_idx):
+            item = sseq._sample_cache._cache_map.get(idx)  # type: ignore
+            if idx in cached_idx:
+                assert item is not None
+            else:
+                assert item is None
+
+    def test_endless_cache_proxy(self):
+        max_idx = 5
+        sseq, k0, k1, k2, fk = self._make_cached_sequence(
+            sp.CachedSamplesSequence.EndlessCachePolicy(), max_idx
+        )
+
+        # raw source
+        self._check_samples(k0, k0, sseq.samples.samples, set())  # type: ignore
+        # copy after first stage
+        self._check_samples(k1, k0, sseq.samples, set())
+        # copy after second stage, then accessed and cached
+        self._check_samples(k2, k0, sseq, fk)
+        # these have never been accessed
+        self._check_samples(k1, k0, sseq.samples, set())
+
+        # all samples have been cached
+        self._check_cache_buffer(sseq, max_idx, {i for i in range(len(sseq))})
+
+    def test_bounded_cache_proxy(self):
+        cache_size = 2
+        max_idx = 5
+        sseq, k0, k1, k2, fk = self._make_cached_sequence(
+            sp.CachedSamplesSequence.BoundedCachePolicy(cache_size), max_idx
+        )
+
+        # raw source
+        self._check_samples(k0, k0, sseq.samples.samples, set())  # type: ignore
+        # copy after first stage
+        self._check_samples(k1, k0, sseq.samples, set())
+        # copy after second stage, then accessed and cached
+        self._check_samples(k2, k0, sseq, fk)
+        # these have never been accessed
+        self._check_samples(k1, k0, sseq.samples, set())
+
+        # only last samples have been cached
+        self._check_cache_buffer(
+            sseq, max_idx, {i for i in range(len(sseq) - cache_size, len(sseq))}
+        )
+
+    def test_persistent_cache_proxy(self, tmp_path):
+        def make_cache_test(cache_name):
+            cache_dir = tmp_path / cache_name
+            cache_dir.mkdir(parents=True, exist_ok=True)
+
+            buffer_size = 2
+            max_idx = 5
+            sseq, k0, k1, k2, fk = self._make_cached_sequence(
+                sp.CachedSamplesSequence.PersistentCachePolicy(
+                    cache_dir=cache_dir, max_buffer_size=buffer_size, verbose=True
+                ),
+                max_idx,
+            )
+
+            # raw source
+            self._check_samples(k0, k0, sseq.samples.samples, set())  # type: ignore
+            # copy after first stage
+            self._check_samples(k1, k0, sseq.samples, set())
+            # copy after second stage, then accessed and cached
+            self._check_samples(k2, k0, sseq, fk)
+            # these have never been accessed
+            self._check_samples(k1, k0, sseq.samples, set())
+
+            # only last samples have been cached
+            self._check_cache_buffer(
+                sseq, max_idx, {i for i in range(len(sseq) - buffer_size, len(sseq))}
+            )
+
+            return cache_dir
+
+        cache0_full_path = make_cache_test("cache0")
+        cache1_full_path = make_cache_test("cache1")
+        make_cache_test("cache1")
+
+        from filecmp import dircmp
+        from pathlib import Path
+        import json
+        import joblib
+
+        def file_diff(dcmp: dircmp):
+            assert len(dcmp.left_only) == 0
+            assert len(dcmp.right_only) == 0
+            left_path, right_path = Path(dcmp.left), Path(dcmp.right)
+            for f in dcmp.common_files:
+                left_f = left_path / Path(f)
+                right_f = right_path / Path(f)
+                if f == "func_code.py":
+                    assert f in dcmp.same_files
+                elif f == "metadata.json":
+                    with left_f.open() as file:
+                        left_data = json.load(file)
+                        assert "input_args" in left_data
+                    with right_f.open() as file:
+                        right_data = json.load(file)
+                        assert "input_args" in right_data
+                    assert left_data["input_args"] == right_data["input_args"]
+                else:
+                    assert f == "output.pkl"
+                    left_obj = joblib.load(left_f)
+                    right_obj = joblib.load(right_f)
+                    assert isinstance(left_obj, SideEffectSample)
+                    assert isinstance(right_obj, SideEffectSample)
+                    for k in left_obj.keys() | right_obj.keys():
+                        assert left_obj.get_bypass(k) == right_obj.get_bypass(k)
+            for sub_dcmp in dcmp.subdirs.values():
+                file_diff(sub_dcmp)
+
+        file_diff(dircmp(str(cache0_full_path), str(cache1_full_path)))
+
+    def test_concat_seq_proxy(self):
+        n_seq = 3
+        n_items = 5
+        srcs = [self._make_random_source(n_items) for i in range(n_seq)]
+        cat_sseq = sp.ConcatSamplesSequence(srcs, StageKeysFilter(["data_1", "data_2"]))
+
+        out_iter = iter(cat_sseq)
+        for src_seq in srcs:
+            for item in src_seq:
+                out_item = next(out_iter)
+                for k, v in out_item.items():
+                    assert k in item
+                    assert v == item[k]
+
+    def test_filtered_seq_proxy(self):
+        n_items = 5
+        src = self._make_random_source(n_items)
+
+        def filter_fn(x: Sample) -> bool:
+            return x["idx"] in (2, 4)
+
+        flt_sseq = sp.FilteredSamplesSequence(
+            src, filter_fn, StageKeysFilter(["data_1", "data_2"])
+        )
+
+        assert len(flt_sseq) == 2
+        flt_iter = iter(flt_sseq)
+        for src_item in src:
+            if filter_fn(src_item):
+                flt_item = next(flt_iter)
+                for k, v in flt_item.items():
+                    assert k in src_item
+                    assert v == src_item[k]
+
+    def test_sorted_seq_proxy(self):
+        n_items = 5
+        src = self._make_random_source(n_items)
+
+        def keyfn(x: Sample):
+            return x["data_1"]
+
+        srt_sseq = sp.SortedSamplesSequence(
+            src, keyfn, StageKeysFilter(["data_1", "data_2"])
+        )
+
+        sorted_src = sorted(src.samples, key=keyfn)
+        for src_item, out_item in zip(sorted_src, srt_sseq):
+            for k, v in out_item.items():
+                assert k in src_item
+                assert v == src_item[k]
+
+    def test_sliced_seq_proxy(self):
+        n_items = 10
+        src = self._make_random_source(n_items)
+
+        slc_sseq = sp.SlicedSamplesSequence(
+            src, 2, 9, 3
+        )
+
+        sliced_src = src.samples[2:9:3]
+        assert len(slc_sseq) == len(sliced_src)
+        for src_item, out_item in zip(sliced_src, slc_sseq):
+            for k, v in out_item.items():
+                assert k in src_item
+                assert v == src_item[k]
diff --git a/tests/pipelime/sequences/test_samples.py b/tests/pipelime/sequences/test_samples.py
index 8d09286..cb4e0d5 100644
--- a/tests/pipelime/sequences/test_samples.py
+++ b/tests/pipelime/sequences/test_samples.py
@@ -2,7 +2,7 @@
 
 from pipelime.sequences.readers.filesystem import UnderfolderReader
 from pipelime.sequences.samples import (
-    FilesystemItem,
+    FileSystemItem,
     FileSystemSample,
     GroupedSample,
     MemoryItem,
@@ -61,7 +61,7 @@ def test_filesystem_sample(self, filesystem_datasets, tmp_path_factory):
         sample = reader[0]
 
         for key in sample.keys():
-            assert isinstance(sample.metaitem(key), FilesystemItem)
+            assert isinstance(sample.metaitem(key), FileSystemItem)
 
         assert len(sample) > 0
         assert isinstance(sample["image"], np.ndarray)
@@ -72,6 +72,22 @@ def test_filesystem_sample(self, filesystem_datasets, tmp_path_factory):
 
         del sample["_NEW_IMAGE_"]
 
+    def test_filesystem_sample_copy(self, filesystem_datasets):
+
+        dataset_folder = filesystem_datasets["minimnist_underfolder"]["folder"]
+        reader = UnderfolderReader(folder=dataset_folder)
+
+        sample = reader[0]
+        old_keys = list(sample.keys())
+
+        sample_copy = sample.copy()
+        for key in sample_copy.keys():
+            del sample_copy[key]
+
+        for key in old_keys:
+            assert key not in sample_copy
+            assert key in sample
+
     def test_flush(self, filesystem_datasets, tmp_path_factory):
 
         dataset_folder = filesystem_datasets["minimnist_underfolder"]["folder"]
@@ -84,6 +100,7 @@ def test_flush(self, filesystem_datasets, tmp_path_factory):
             for key in sample.keys():
                 temp = sample[key]
                 assert sample.is_cached(key)
+                assert temp is not None
 
         reader.flush()
 
@@ -95,7 +112,7 @@ def test_flush(self, filesystem_datasets, tmp_path_factory):
     def test_filesystem_sample_nonlazy(self, filesystem_datasets, tmp_path_factory):
 
         dataset_folder = filesystem_datasets["minimnist_underfolder"]["folder"]
-        reader = UnderfolderReader(folder=dataset_folder, lazy_samples=False)
+        UnderfolderReader(folder=dataset_folder, lazy_samples=False)
 
     def test_update(self, filesystem_datasets, tmp_path_factory):
         dataset_folder = filesystem_datasets["minimnist_underfolder"]["folder"]
diff --git a/tests/pipelime/sequences/test_samples_merge.py b/tests/pipelime/sequences/test_samples_merge.py
new file mode 100644
index 0000000..c4b7d5d
--- /dev/null
+++ b/tests/pipelime/sequences/test_samples_merge.py
@@ -0,0 +1,160 @@
+from pathlib import Path
+from pipelime.sequences.readers.filesystem import UnderfolderReader
+from pipelime.sequences.readers.h5 import H5Reader
+from pipelime.sequences.samples import GroupedSample, PlainSample, SamplesSequence
+from pipelime.sequences.writers.filesystem import UnderfolderWriter
+from pipelime.sequences.writers.h5 import H5Writer
+from pipelime.tools.toydataset import ToyDatasetGenerator
+import rich
+import functools
+import shutil
+
+
+class TestSampleMergesGroupedSample:
+    def test_merge(self, plain_samples_generator):
+
+        dataset = plain_samples_generator("d0_", 10)
+
+        g0 = GroupedSample(samples=dataset[:5])
+        g1 = GroupedSample(samples=dataset[5:])
+
+        for k in ["added_0", "added_1"]:
+            g1[k] = "new_value"
+
+        merged = g0.merge(g1)
+        assert set(merged.keys()) == set(g0.keys()) | set(g1.keys())
+
+
+class TestSampleMergesPlainSample:
+    def test_merge(self):
+
+        keys = ["A", "B", "C"]
+        plain_samples = [PlainSample({key: 1.0}) for key in keys]
+        merged_sample = functools.reduce(lambda x, y: x.merge(y), plain_samples)
+        assert set(merged_sample.keys()) == set(keys)
+
+
+class TestSampleMergesUnderfolder:
+    def test_merges(self, tmpdir):
+
+        # Builds Folders and Filenames
+        size = 32
+        sub_datasets = ["A", "B", "C"]
+        root_folders = [Path(tmpdir.mkdir(x)) for x in sub_datasets]
+        data_folders = [x / UnderfolderReader.DATA_SUBFOLDER for x in root_folders]
+        rich.print("Folders:")
+        [rich.print(x) for x in root_folders]
+
+        # Generates Toy Dataset
+        for name, folder in zip(sub_datasets, data_folders):
+            ToyDatasetGenerator.generate_toy_dataset(folder, size, suffix=name)
+
+        # Builds Reader for each toy dataset
+        readers = {x: UnderfolderReader(y) for x, y in zip(sub_datasets, root_folders)}
+
+        # Adds an overlapping key to each sample in each dataset
+        for reader_name, reader in readers.items():
+            for sample in reader:
+                sample["added_key"] = reader_name
+
+        # computes the set of merged keys (account for overlapping keys)
+        keys_set = set()
+        for reader_name, reader in readers.items():
+            assert len(reader) == size
+            assert len(reader) > 0
+            keys_set.update(reader[0].keys())
+
+        # merge all the datasets
+        merged_reader = SamplesSequence.merge_sequences(readers.values())
+
+        assert len(merged_reader) == size
+        assert len(merged_reader) > 0
+
+        # checks that all the keys are present
+        ref_sample = merged_reader[0]
+        assert len(keys_set) > 0
+        assert len(ref_sample.keys()) == len(keys_set)
+
+        # Checks if overlapping keys values are set equal to the last sample in merging lists
+        assert ref_sample["added_key"] == sub_datasets[-1]
+
+        # write merged dataset to underfolder
+        merged_folder = Path(tmpdir.mkdir("merged"))
+        writer = UnderfolderWriter(folder=merged_folder)
+        writer(merged_reader)
+        rich.print("Output written to:", merged_folder)
+
+        # read merged dataset from underfolder
+        r_reader = UnderfolderReader(folder=merged_folder)
+        assert len(r_reader) > 0
+        assert len(r_reader) == len(merged_reader)
+
+
+class TestSampleMergesH5:
+    def test_merges(self, tmpdir):
+
+        # Builds Folders and Filenames
+        size = 32
+        sub_datasets = ["A", "B", "C"]
+        root_folders = [Path(tmpdir.mkdir(x)) for x in sub_datasets]
+        data_folders = [x / UnderfolderReader.DATA_SUBFOLDER for x in root_folders]
+        h5_files = [x / "dataset.h5" for x in root_folders]
+
+        rich.print("Files:")
+        [rich.print(x) for x in root_folders]
+
+        # Generate toy data
+        for name, folder in zip(sub_datasets, data_folders):
+            ToyDatasetGenerator.generate_toy_dataset(folder, size, suffix=name)
+
+        # Underfolder readers for toy datasets
+        readers = {x: UnderfolderReader(y) for x, y in zip(sub_datasets, root_folders)}
+
+        # H5Writer for each dataset
+        writers = {x: H5Writer(y) for x, y in zip(sub_datasets, h5_files)}
+        for name, reader in readers.items():
+            writer = writers[name]
+            writer(reader)
+
+        # Removed unused underfolder reader of toy datasets
+        for d in data_folders:
+            shutil.rmtree(d)
+
+        # Reader for h5 toy data
+        readers = {x: H5Reader(y) for x, y in zip(sub_datasets, h5_files)}
+
+        # Checks for size of each dataset
+        for reader_name, reader in readers.items():
+            assert len(reader) == size
+
+        # Merge datasets
+        merged_reader = SamplesSequence.merge_sequences(readers.values())
+
+        assert len(merged_reader) == size
+        assert len(merged_reader) > 0
+
+        # Adds an overlapping key to each sample in each dataset
+        for reader_name, reader in readers.items():
+            for sample in reader:
+                sample["added_key"] = reader_name
+
+        # Builds a set of merged keys (account for overlapping keys)
+        keys_set = set()
+        for reader_name, reader in readers.items():
+            assert len(reader) == size
+            assert len(reader) > 0
+            keys_set.update(reader[0].keys())
+
+        ref_sample = merged_reader[0]
+        assert len(keys_set) > 0
+        assert len(ref_sample.keys()) == len(keys_set)
+
+        # Write merged dataset to h5 file
+        merged_h5_file = Path(tmpdir.mkdir("merged")) / "dataset.h5"
+        writer = H5Writer(filename=merged_h5_file)
+        writer(merged_reader)
+        rich.print("Output written to:", merged_h5_file)
+
+        # Read merged dataset from h5 file
+        r_reader = H5Reader(filename=merged_h5_file)
+        assert len(r_reader) > 0
diff --git a/tests/pipelime/sequences/test_stages.py b/tests/pipelime/sequences/test_stages.py
index 405bf38..cb95f68 100644
--- a/tests/pipelime/sequences/test_stages.py
+++ b/tests/pipelime/sequences/test_stages.py
@@ -1,7 +1,9 @@
 from choixe.spooks import Spook
 
 from pipelime.sequences.readers.filesystem import UnderfolderReader
-from pipelime.sequences.samples import PlainSample
+from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+from pipelime.sequences.readers.base import ReaderTemplate
+from pipelime.sequences.samples import PlainSample, SamplesSequence
 from pipelime.sequences.stages import (
     SampleStage,
     StageAugmentations,
@@ -9,6 +11,9 @@
     StageIdentity,
     StageKeysFilter,
     StageRemap,
+    RemoteParams,
+    StageUploadToRemote,
+    StageRemoveRemote,
 )
 
 
@@ -80,7 +85,7 @@ def test_filter(self):
 
         negates = [True, False]
         for negate in negates:
-            stage = StageKeysFilter(keys=["name", "idx"], negate=negate)
+            stage = StageKeysFilter(key_list=["name", "idx"], negate=negate)
             _plug_test(stage)
 
             out = stage(s)
@@ -91,8 +96,7 @@ def test_filter(self):
 
 
 class TestStageAugmentations(object):
-    def test_augmentations(self, toy_dataset_small):
-
+    def test_augmentations(self, toy_dataset_small, tmp_path):
         import albumentations as A
 
         folder = toy_dataset_small["folder"]
@@ -129,10 +133,23 @@ def test_augmentations(self, toy_dataset_small):
 
         for sample in reader:
             out = stage(sample)
-
             for key in sample.keys():
                 assert key in out
 
+        transform_file = str(tmp_path / "tr.json")
+        A.save(transform, transform_file)
+        stage_fromfile = StageAugmentations(
+            transform_cfg=transform_file,
+            targets={
+                "image": "image",
+                "mask": "mask",
+                "inst": "inst",
+                "keypoints": "keypoints",
+                "bboxes": "bboxes",
+            },
+        )
+        assert stage.to_dict() == stage_fromfile.to_dict()
+
 
 class TestStageCompose(object):
     def test_compose(self):
@@ -144,7 +161,7 @@ def test_compose(self):
             StageRemap(remap={"name": "a"}, remove_missing=False),
             StageRemap(remap={"idx": "b"}, remove_missing=False),
             StageRemap(remap={"float": "c"}, remove_missing=False),
-            StageKeysFilter(keys=["a", "b", "c"], negate=False),
+            StageKeysFilter(key_list=["a", "b", "c"], negate=False),
             StageIdentity(),
         ]
 
@@ -157,3 +174,370 @@ def test_compose(self):
         assert "b" in out
         assert "c" in out
         assert "tail" not in out
+
+
+class TestSampleSequenceStaged:
+    def test_samplessequence_staged(self):
+        samples = []
+        for index in range(10):
+            samples.append(
+                PlainSample(
+                    data={"name": "sample", "idx": index, "float": 2.3, "tail": True}
+                )
+            )
+
+        stages = [
+            StageIdentity(),
+            StageRemap(remap={"name": "a"}, remove_missing=False),
+            StageRemap(remap={"idx": "b"}, remove_missing=False),
+            StageRemap(remap={"float": "c"}, remove_missing=False),
+            StageKeysFilter(key_list=["a", "b"], negate=False),
+        ]
+
+        sequence = SamplesSequence(samples=samples)
+        sequence.stage = StageCompose(stages=stages)
+
+        for sample in sequence:
+            assert "a" in sample
+            assert "b" in sample
+            assert "c" not in sample
+            assert "tail" not in sample
+
+
+class TestStageRemote:
+    def _check_remote_list(self, data_folder, expected_remote_list):
+        from pipelime.filesystem.toolkit import FSToolkit
+        from urllib.parse import urlparse, ParseResult
+        from pathlib import Path
+
+        for x in UnderfolderReader(data_folder):
+            for k in x.keys():
+                if FSToolkit.is_remote_file(x.metaitem(k).source()):
+                    rmlist = FSToolkit.load_remote_list(x.metaitem(k).source())
+                    rmlist = [urlparse(rm) for rm in rmlist]
+                    rmlist = [
+                        ParseResult(
+                            scheme=rm.scheme,
+                            netloc="localhost",
+                            path=Path(rm.path[1:]).parent.as_posix(),
+                            params="",
+                            query="",
+                            fragment="",
+                        ).geturl()
+                        for rm in rmlist
+                    ]
+
+                    assert expected_remote_list == rmlist
+
+    def _upload_to_remote(
+        self, dataset, out_folder, remote_prms, filter_fn=None, check_data=True
+    ):
+        from pipelime.sequences.proxies import FilteredSamplesSequence
+        from pipelime.filesystem.toolkit import FSToolkit
+        import numpy as np
+
+        # read and upload
+        reader = UnderfolderReader(dataset)
+        filtered_seq = (
+            reader if filter_fn is None else FilteredSamplesSequence(reader, filter_fn)
+        )
+        sseq = SamplesSequence(
+            filtered_seq,
+            StageUploadToRemote(remote_prms, {"image": "png", "mask": "png"}),
+        )
+
+        # save after uploading
+        reader_template = reader.get_reader_template()
+        assert isinstance(reader_template, ReaderTemplate)
+        reader_template.extensions_map["image"] = "remote"
+        reader_template.extensions_map["mask"] = "remote"
+
+        out_folder.mkdir(parents=True)
+        writer = UnderfolderWriterV2(
+            out_folder,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+            reader_template=reader_template,
+        )
+
+        writer(sseq)
+
+        if check_data:
+            # high-level check
+            reader_out = UnderfolderReader(out_folder)
+            for x, y in zip(filtered_seq, reader_out):
+                assert FSToolkit.is_remote_file(y.metaitem("image").source())
+                assert FSToolkit.is_remote_file(y.metaitem("mask").source())
+
+                assert x.keys() == y.keys()
+                for k, v in x.items():
+                    if isinstance(v, np.ndarray):
+                        assert np.array_equal(v, y[k])
+                    else:
+                        assert v == y[k]
+
+        return len(sseq)
+
+    def test_file_upload(self, toy_dataset_small, tmp_path):
+        # data lake
+        remote_root = tmp_path / "remote"
+        remote_root.mkdir(parents=True)
+        remote_root = remote_root.as_posix()
+
+        self._upload_to_remote(
+            toy_dataset_small["folder"],
+            tmp_path / "output",
+            RemoteParams(scheme="file", netloc="localhost", base_path=remote_root),
+        )
+
+    def test_s3_upload(self, toy_dataset_small, tmp_path, minio):
+        # data lake
+        if not minio:
+            from pytest import skip
+
+            skip("MinIO unavailable")
+
+        self._upload_to_remote(
+            toy_dataset_small["folder"],
+            tmp_path / "output",
+            RemoteParams(
+                scheme="s3",
+                netloc="localhost:9000",
+                base_path="test-s3-upload",
+                init_args={
+                    "access_key": minio,
+                    "secret_key": minio,
+                    "secure_connection": False,
+                },
+            ),
+        )
+
+    def test_incremental_file_upload(self, toy_dataset_small, tmp_path):
+        from pipelime.sequences.proxies import FilteredSamplesSequence
+        from pipelime.filesystem.toolkit import FSToolkit
+        from shutil import rmtree
+        import pytest
+        import numpy as np
+        from filecmp import cmp
+
+        # data lake
+        remote_root = tmp_path / "remote"
+        remote_root.mkdir(parents=True)
+        remote_root = remote_root.as_posix()
+
+        input_dataset = toy_dataset_small["folder"]
+        output_dataset_a = tmp_path / "output_a"
+
+        stage_upload = RemoteParams(
+            scheme="file", netloc="localhost", base_path=remote_root
+        )
+
+        # upload even samples
+        counter_even = self._upload_to_remote(
+            input_dataset,
+            output_dataset_a,
+            stage_upload,
+            lambda x: int(x["metadata"]["index"]) % 2 == 0,
+        )
+
+        # manually copy the odd samples
+        reader = UnderfolderReader(input_dataset, copy_root_files=False)
+        sseq = FilteredSamplesSequence(
+            reader, lambda x: int(x["metadata"]["index"]) % 2 == 1
+        )
+        writer = UnderfolderWriterV2(
+            output_dataset_a,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+            reader_template=reader.get_reader_template(),
+        )
+        writer(sseq)
+
+        # clear all remote data
+        rmtree(remote_root, ignore_errors=True)
+        remote_root = tmp_path / "remote"
+        remote_root.mkdir(parents=True, exist_ok=True)
+        remote_root = remote_root.as_posix()
+
+        # upload all indices, but only the odd ones are actually copied to the remote
+        output_dataset_b = tmp_path / "output_b"
+        counter_all = self._upload_to_remote(
+            output_dataset_a, output_dataset_b, stage_upload, check_data=False
+        )
+
+        reader = UnderfolderReader(output_dataset_b)
+        assert counter_even == len(reader) // 2
+        assert counter_all == len(reader)
+
+        # only the odd samples are on the remote
+        for x, y in zip(UnderfolderReader(output_dataset_a), reader):
+            assert x.keys() == y.keys()
+            for k in x.keys():
+                if k in ("image", "mask") and int(x["metadata"]["index"]) % 2 == 0:
+                    # even sample
+                    assert FSToolkit.is_remote_file(x.metaitem(k).source())
+                    assert FSToolkit.is_remote_file(y.metaitem(k).source())
+                    assert cmp(
+                        x.metaitem(k).source(), y.metaitem(k).source(), shallow=False
+                    )
+                    with pytest.raises(Exception):
+                        _ = x[k]
+                    with pytest.raises(Exception):
+                        _ = y[k]
+                else:
+                    v = x[k]
+                    if isinstance(v, np.ndarray):
+                        assert np.array_equal(v, y[k])
+                    else:
+                        assert v == y[k]
+
+    def test_multiple_remote_upload(self, toy_dataset_small, tmp_path):
+        from urllib.parse import ParseResult
+        from shutil import rmtree
+        import numpy as np
+
+        # create two remotes
+        remote_a = tmp_path / "remote_a"
+        remote_a.mkdir(parents=True, exist_ok=True)
+        remote_a = remote_a.as_posix()
+
+        remote_b = tmp_path / "remote_b"
+        remote_b.mkdir(parents=True, exist_ok=True)
+        remote_b = remote_b.as_posix()
+
+        input_dataset = toy_dataset_small["folder"]
+        output_dataset_a = tmp_path / "output_a"
+
+        # upload to both remotes
+        self._upload_to_remote(
+            input_dataset,
+            output_dataset_a,
+            [
+                RemoteParams(scheme="file", netloc="localhost", base_path=remote_a),
+                RemoteParams(scheme="file", netloc="localhost", base_path=remote_b),
+            ],
+        )
+
+        # the .remote files must contains both remotes, remote_root first
+        expected_remote_list = [
+            ParseResult(
+                scheme="file",
+                netloc="localhost",
+                path=remote_a,
+                params="",
+                query="",
+                fragment="",
+            ).geturl(),
+            ParseResult(
+                scheme="file",
+                netloc="localhost",
+                path=remote_b,
+                params="",
+                query="",
+                fragment="",
+            ).geturl(),
+        ]
+        self._check_remote_list(output_dataset_a, expected_remote_list)
+
+        # now remove remote_a and check again the data
+        rmtree(remote_a, ignore_errors=True)
+
+        for x, y in zip(
+            UnderfolderReader(input_dataset), UnderfolderReader(output_dataset_a)
+        ):
+            for k, v in x.items():
+                if isinstance(v, np.ndarray):
+                    assert np.array_equal(v, y[k])
+                else:
+                    assert v == y[k]
+
+        # now upload to remote_c taking data from remote_b
+        remote_c = tmp_path / "remote_c"
+        remote_c.mkdir(parents=True, exist_ok=True)
+        remote_c = remote_c.as_posix()
+
+        output_dataset_b = tmp_path / "output_b"
+        self._upload_to_remote(
+            output_dataset_a,
+            output_dataset_b,
+            RemoteParams(scheme="file", netloc="localhost", base_path=remote_c),
+        )
+
+        expected_remote_list.append(
+            ParseResult(
+                scheme="file",
+                netloc="localhost",
+                path=remote_c,
+                params="",
+                query="",
+                fragment="",
+            ).geturl()
+        )
+
+        self._check_remote_list(output_dataset_b, expected_remote_list)
+
+    def test_remove_remote(self, toy_dataset_small, tmp_path):
+        from urllib.parse import ParseResult
+
+        # create two remotes
+        remote_a = tmp_path / "remote_a"
+        remote_a.mkdir(parents=True, exist_ok=True)
+        remote_a = remote_a.as_posix()
+
+        remote_b = tmp_path / "remote_b"
+        remote_b.mkdir(parents=True, exist_ok=True)
+        remote_b = remote_b.as_posix()
+
+        input_dataset = toy_dataset_small["folder"]
+        output_dataset_a = tmp_path / "output_a"
+
+        # upload to both remotes
+        self._upload_to_remote(
+            input_dataset,
+            output_dataset_a,
+            [
+                RemoteParams(scheme="file", netloc="localhost", base_path=remote_a),
+                RemoteParams(scheme="file", netloc="localhost", base_path=remote_b),
+            ],
+        )
+
+        # the .remote files must contains both remotes, remote_root first
+        expected_remote_list = [
+            ParseResult(
+                scheme="file",
+                netloc="localhost",
+                path=remote_a,
+                params="",
+                query="",
+                fragment="",
+            ).geturl(),
+            ParseResult(
+                scheme="file",
+                netloc="localhost",
+                path=remote_b,
+                params="",
+                query="",
+                fragment="",
+            ).geturl(),
+        ]
+        self._check_remote_list(output_dataset_a, expected_remote_list)
+
+        # remove remote_a
+        reader = UnderfolderReader(output_dataset_a)
+        sseq = SamplesSequence(
+            reader,
+            StageRemoveRemote(
+                RemoteParams(scheme="file", netloc="localhost", base_path=remote_a),
+                ["image", "mask"],
+            ),
+        )
+
+        output_dataset_b = tmp_path / "output_b"
+        output_dataset_b.mkdir(parents=True)
+        writer = UnderfolderWriterV2(
+            output_dataset_b,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+            reader_template=reader.get_reader_template(),
+        )
+        writer(sseq)
+
+        expected_remote_list.pop(0)
+        self._check_remote_list(output_dataset_b, expected_remote_list)
diff --git a/tests/pipelime/sequences/test_validation.py b/tests/pipelime/sequences/test_validation.py
index 4d61a94..c8c2dc7 100644
--- a/tests/pipelime/sequences/test_validation.py
+++ b/tests/pipelime/sequences/test_validation.py
@@ -1,8 +1,12 @@
 import pytest
-from schema import SchemaError
-
+import rich
 from pipelime.sequences.readers.filesystem import UnderfolderReader
-from pipelime.sequences.validation import SchemaLoader
+from pipelime.sequences.validation import (
+    SampleSchema,
+    SchemaLoader,
+    StageValidate,
+    OperationValidate,
+)
 
 
 class TestSampleSchemaValidation(object):
@@ -14,6 +18,7 @@ def test_samples_validation(self, filesystem_datasets, tmp_path_factory):
         reader = UnderfolderReader(folder=dataset_folder)
 
         for schema_name, schema in schemas.items():
+            rich.print("Validation", schema_name, schema)
             filename = schema["filename"]
             valid = schema["valid"]
             should_pass = schema["should_pass"]
@@ -29,5 +34,60 @@ def test_samples_validation(self, filesystem_datasets, tmp_path_factory):
                 if should_pass:
                     schema.validate(sample)
                 else:
-                    with pytest.raises(SchemaError):
+                    with pytest.raises(SampleSchema.ValidationError):
                         schema.validate(sample)
+
+    def test_samples_validation_with_stage(self, filesystem_datasets, tmp_path_factory):
+
+        dataset_folder = filesystem_datasets["minimnist_underfolder"]["folder"]
+        schemas = filesystem_datasets["minimnist_underfolder"]["schemas"]
+
+        reader = UnderfolderReader(folder=dataset_folder)
+
+        for schema_name, schema in schemas.items():
+            rich.print("Validation", schema_name, schema)
+            filename = schema["filename"]
+            valid = schema["valid"]
+            should_pass = schema["should_pass"]
+
+            if valid:
+                schema = SchemaLoader.load(filename)
+            else:
+                with pytest.raises(ModuleNotFoundError):
+                    SchemaLoader.load(filename)
+                continue
+
+            for sample in reader:
+                if should_pass:
+                    StageValidate(schema)(sample)
+                else:
+                    with pytest.raises(SampleSchema.ValidationError):
+                        StageValidate(schema)(sample)
+
+    def test_samples_validation_with_operation(
+        self, filesystem_datasets, tmp_path_factory
+    ):
+
+        dataset_folder = filesystem_datasets["minimnist_underfolder"]["folder"]
+        schemas = filesystem_datasets["minimnist_underfolder"]["schemas"]
+
+        reader = UnderfolderReader(folder=dataset_folder)
+
+        for schema_name, schema in schemas.items():
+            filename = schema["filename"]
+            valid = schema["valid"]
+            should_pass = schema["should_pass"]
+
+            if valid:
+                schema = SchemaLoader.load(filename)
+            else:
+                with pytest.raises(ModuleNotFoundError):
+                    SchemaLoader.load(filename)
+                continue
+
+            op = OperationValidate(sample_schema=schema)
+            if should_pass:
+                op(reader)
+            else:
+                with pytest.raises(SampleSchema.ValidationError):
+                    op(reader)
diff --git a/tests/pipelime/tools/__init__.py b/tests/pipelime/tools/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/pipelime/tools/test_click.py b/tests/pipelime/tools/test_click.py
new file mode 100644
index 0000000..9b3c4b2
--- /dev/null
+++ b/tests/pipelime/tools/test_click.py
@@ -0,0 +1,44 @@
+from pipelime.tools.click import ClickTools
+from types import SimpleNamespace
+from deepdiff import DeepDiff
+import pytest
+
+
+class TestClickTools:
+    @pytest.mark.parametrize(
+        "args, expected",
+        [
+            (
+                [
+                    "-i",
+                    "dag.yml",
+                    "-p",
+                    "params.yml",
+                    "-o",
+                    "output.yml",
+                    "--number",
+                    "2",
+                    "--boolean",
+                    "False",
+                ],
+                {
+                    "i": "dag.yml",
+                    "p": "params.yml",
+                    "o": "output.yml",
+                    "number": "2",
+                    "boolean": "False",
+                },
+            ),
+            (
+                [],
+                {},
+            ),
+        ],
+    )
+    def test_additional_arguments(self, args, expected):
+
+        # monkey patch
+        ctx = SimpleNamespace(**{"args": []})
+        ctx.args = args
+        result = ClickTools.parse_additional_args(ctx)
+        assert not DeepDiff(expected, result)
diff --git a/tests/pipelime/workflow/tools/test_cwl_template.py b/tests/pipelime/workflow/tools/test_cwl_template.py
index b5a05e0..b6c98f8 100644
--- a/tests/pipelime/workflow/tools/test_cwl_template.py
+++ b/tests/pipelime/workflow/tools/test_cwl_template.py
@@ -1,10 +1,6 @@
 from pathlib import Path
-from re import template
-
 import click
 import yaml
-
-from pipelime.cli.workflow.workflow import workflow
 from pipelime.workflow.cwl import (
     CwlNode,
     CwlNodesManager,
@@ -483,7 +479,7 @@ def test_cwl_template(self, tmpdir_factory):
         # cwl_filled2 = Click2Cwl.convert_click_to_cwl(cmd, commands, forwards)
         # assert cwl_filled == cwl_filled2
 
-        output_cwl_file = Path(tmpdir_factory.mktemp("cwl")) / f"output.cwl"
+        output_cwl_file = Path(tmpdir_factory.mktemp("cwl")) / "output.cwl"
         cwl_template.dumps(output_cwl_file)
         with open(output_cwl_file, "r") as f:
             lines = f.readlines()
@@ -507,7 +503,7 @@ def a_click_command(opt0, opt1, opt2, opt3, opt4, opt5, opt6, opt7, opt8):
     pass
         """
 
-        output_click_file = Path(tmpdir_factory.mktemp("click")) / f"output.py"
+        output_click_file = Path(tmpdir_factory.mktemp("click")) / "output.py"
         with open(output_click_file, "w") as f:
             f.write(click_command)
         loaded_template = CwlTemplate(output_click_file)
@@ -579,7 +575,7 @@ def test_cwl_workflow_template(self, tmpdir_factory):
         assert "node01_opt02" in cwl_workflow_template.template["inputs"]
         assert "string" in cwl_workflow_template.template["inputs"]["node01_opt02"]
 
-        output_template = Path(tmpdir_factory.mktemp("workflow")) / f"template.cwl"
+        output_template = Path(tmpdir_factory.mktemp("workflow")) / "template.cwl"
         cwl_workflow_template.dumps(output_template)
         loaded_template = CwlWorkflowTemplate.from_file(output_template)
         assert cwl_workflow_template.template == loaded_template.template
@@ -604,14 +600,14 @@ def test_cwl_nodes_manager(self, tmpdir_factory):
         nodes = CwlNodesManager.available_nodes(folder=output_folder)
         assert len(nodes) == 0
 
-        click_command1 = f"""
+        click_command1 = """
 import click
 @click.command('command1')
 @click.option('--opt0', required=True, type=int, help='help opt0')
 def command1(opt0):
     pass
         """
-        output_click_file1 = Path(tmpdir_factory.mktemp("click")) / f"command1.py"
+        output_click_file1 = Path(tmpdir_factory.mktemp("click")) / "command1.py"
         with open(output_click_file1, "w") as f:
             f.write(click_command1)
         cwl_template1 = CwlTemplate(
@@ -619,14 +615,14 @@ def command1(opt0):
         )  # , alias=['one', 'two'], forwards=['opt0'])
         CwlNodesManager.create_node("first", cwl_template1, folder=output_folder)
 
-        click_command2 = f"""
+        click_command2 = """
 import click
 @click.command('command2')
 @click.option('--opt0', required=True, type=int, help='help opt0')
 def command2(opt0):
     pass
         """
-        output_click_file2 = Path(tmpdir_factory.mktemp("click")) / f"command2.py"
+        output_click_file2 = Path(tmpdir_factory.mktemp("click")) / "command2.py"
         with open(output_click_file2, "w") as f:
             f.write(click_command2)
         cwl_template2 = CwlTemplate(script=str(output_click_file2))
diff --git a/tests/sample_data/augmentations/full_custom_augmentations.yml b/tests/sample_data/augmentations/full_custom_augmentations.yml
deleted file mode 100644
index 0ddfe14..0000000
--- a/tests/sample_data/augmentations/full_custom_augmentations.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-__version__: 0.5.1
-transform:
-  __class_fullname__: albumentations.core.composition.Compose
-  additional_targets: {}
-  bbox_params: null
-  keypoint_params: null
-  p: 1.0
-  transforms:
-  - __class_fullname__: pipelime.augmentations.transforms.PadIfNeededV2
-    min_height: 700
-    min_width: 700
diff --git a/tests/sample_data/augmentations/full_custom_augmentations_bboxes_keypoints.yml b/tests/sample_data/augmentations/full_custom_augmentations_bboxes_keypoints.yml
deleted file mode 100644
index 8460bea..0000000
--- a/tests/sample_data/augmentations/full_custom_augmentations_bboxes_keypoints.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-__version__: 0.5.1
-transform:
-  __class_fullname__: albumentations.core.composition.Compose
-  additional_targets: {}
-  bbox_params:
-    { 
-      check_each_transform: false,
-      format: albumentations,
-      label_fields: null,
-      min_area: 0.0,
-      min_visibility: 0.0,
-    }
-  keypoint_params:
-    angle_in_degrees: true
-    check_each_transform: true
-    format: xy
-    label_fields: null
-    remove_invisible: true
-  p: 1.0
-  transforms:
-  - __class_fullname__: pipelime.augmentations.transforms.PadIfNeededV2
-    min_height: 700
-    min_width: 700
diff --git a/tests/sample_data/datasets/underfolder_empty/data/.placeholder b/tests/sample_data/datasets/underfolder_empty/data/.placeholder
new file mode 100644
index 0000000..e69de29
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/cfg.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/cfg.yml
new file mode 100644
index 0000000..a5c6480
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/cfg.yml
@@ -0,0 +1,3 @@
+old: False
+base_folder: minimnist
+index: id
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_image.jpg
new file mode 100644
index 0000000..9852495
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_info.yml
new file mode 100644
index 0000000..5e7b4b7
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00000_info.yml
@@ -0,0 +1,3 @@
+double: 0.0
+half: 0.0
+sample_id: 0
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_image.jpg
new file mode 100644
index 0000000..d4ae539
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_info.yml
new file mode 100644
index 0000000..300ca9b
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00001_info.yml
@@ -0,0 +1,3 @@
+double: 2.0
+half: 0.5
+sample_id: 1
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_image.jpg
new file mode 100644
index 0000000..5a0f422
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_info.yml
new file mode 100644
index 0000000..35b3ae4
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00002_info.yml
@@ -0,0 +1,3 @@
+double: 4.0
+half: 1.0
+sample_id: 2
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_image.jpg
new file mode 100644
index 0000000..80cef02
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_info.yml
new file mode 100644
index 0000000..ec2809f
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00003_info.yml
@@ -0,0 +1,3 @@
+double: 6.0
+half: 1.5
+sample_id: 3
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_image.jpg
new file mode 100644
index 0000000..e740792
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_info.yml
new file mode 100644
index 0000000..604a485
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00004_info.yml
@@ -0,0 +1,3 @@
+double: 8.0
+half: 2.0
+sample_id: 4
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_image.jpg
new file mode 100644
index 0000000..64b58d8
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_info.yml
new file mode 100644
index 0000000..a94ea32
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00005_info.yml
@@ -0,0 +1,3 @@
+double: 10.0
+half: 2.5
+sample_id: 5
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_image.jpg
new file mode 100644
index 0000000..b79ba97
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_info.yml
new file mode 100644
index 0000000..1636b79
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00006_info.yml
@@ -0,0 +1,3 @@
+double: 12.0
+half: 3.0
+sample_id: 6
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_image.jpg
new file mode 100644
index 0000000..16a1e94
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_info.yml
new file mode 100644
index 0000000..626b9c8
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00007_info.yml
@@ -0,0 +1,3 @@
+double: 14.0
+half: 3.5
+sample_id: 7
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_image.jpg
new file mode 100644
index 0000000..82fe4d1
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_info.yml
new file mode 100644
index 0000000..1c590e4
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00008_info.yml
@@ -0,0 +1,3 @@
+double: 16.0
+half: 4.0
+sample_id: 8
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_image.jpg
new file mode 100644
index 0000000..11e15cc
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_info.yml
new file mode 100644
index 0000000..80ca4ee
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00009_info.yml
@@ -0,0 +1,3 @@
+double: 18.0
+half: 4.5
+sample_id: 9
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_image.jpg
new file mode 100644
index 0000000..97380d4
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_info.yml
new file mode 100644
index 0000000..b6ee60b
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00010_info.yml
@@ -0,0 +1,3 @@
+double: 20.0
+half: 5.0
+sample_id: 10
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_image.jpg
new file mode 100644
index 0000000..295c55a
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_info.yml
new file mode 100644
index 0000000..a992dd4
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00011_info.yml
@@ -0,0 +1,3 @@
+double: 22.0
+half: 5.5
+sample_id: 11
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_image.jpg
new file mode 100644
index 0000000..69a7c5d
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_info.yml
new file mode 100644
index 0000000..915695a
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00012_info.yml
@@ -0,0 +1,3 @@
+double: 24.0
+half: 6.0
+sample_id: 12
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_image.jpg
new file mode 100644
index 0000000..777ad10
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_info.yml
new file mode 100644
index 0000000..1b8d75d
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00013_info.yml
@@ -0,0 +1,3 @@
+double: 26.0
+half: 6.5
+sample_id: 13
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_image.jpg
new file mode 100644
index 0000000..1b721b1
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_info.yml
new file mode 100644
index 0000000..55f7a16
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00014_info.yml
@@ -0,0 +1,3 @@
+double: 28.0
+half: 7.0
+sample_id: 14
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_image.jpg
new file mode 100644
index 0000000..4dca0bf
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_info.yml
new file mode 100644
index 0000000..cfafe30
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00015_info.yml
@@ -0,0 +1,3 @@
+double: 30.0
+half: 7.5
+sample_id: 15
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_image.jpg
new file mode 100644
index 0000000..85af773
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_info.yml
new file mode 100644
index 0000000..381a981
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00016_info.yml
@@ -0,0 +1,3 @@
+double: 32.0
+half: 8.0
+sample_id: 16
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_image.jpg
new file mode 100644
index 0000000..b73655f
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_info.yml
new file mode 100644
index 0000000..f95ee7b
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00017_info.yml
@@ -0,0 +1,3 @@
+double: 34.0
+half: 8.5
+sample_id: 17
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_categories.yml
new file mode 100644
index 0000000..3be12db
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_categories.yml
@@ -0,0 +1,5 @@
+main: category_even
+others:
+- delta
+- epsilon
+- zeta
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_image.jpg
new file mode 100644
index 0000000..0867dd8
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_info.yml
new file mode 100644
index 0000000..d933afa
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00018_info.yml
@@ -0,0 +1,3 @@
+double: 36.0
+half: 9.0
+sample_id: 18
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_categories.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_categories.yml
new file mode 100644
index 0000000..4cb6579
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_categories.yml
@@ -0,0 +1,5 @@
+main: category_odd
+others:
+- alpha
+- beta
+- gamma
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_image.jpg b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_image.jpg
new file mode 100644
index 0000000..c0ce3fd
Binary files /dev/null and b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_image.jpg differ
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_info.yml b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_info.yml
new file mode 100644
index 0000000..c255e79
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/data/00019_info.yml
@@ -0,0 +1,3 @@
+double: 38.0
+half: 9.5
+sample_id: 19
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/numbers.txt b/tests/sample_data/datasets/underfolder_minimnist_queries/numbers.txt
new file mode 100644
index 0000000..0246698
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/numbers.txt
@@ -0,0 +1,3 @@
+0.1 0.1 0.2 0.2 0.3 0.4 0.5
+0.1 0.1 0.2 0.2 0.3 0.4 0.7
+0.1 0.1 0.2 0.2 0.3 0.4 0.8
\ No newline at end of file
diff --git a/tests/sample_data/datasets/underfolder_minimnist_queries/pose.txt b/tests/sample_data/datasets/underfolder_minimnist_queries/pose.txt
new file mode 100644
index 0000000..a067909
--- /dev/null
+++ b/tests/sample_data/datasets/underfolder_minimnist_queries/pose.txt
@@ -0,0 +1,4 @@
+9.975294325464288958e-01 6.760711053306790730e-02 -1.908690150248902595e-02 -3.584983950621601755e-03
+-6.734271198122841751e-02 9.976292508622911503e-01 1.417169597998962915e-02 -6.301910434275505291e-02
+1.999975866377137410e-02 -1.285132013864587891e-02 9.997173866768973571e-01 1.282372274249416466e-01
+0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 1.000000000000000000e+00
\ No newline at end of file
diff --git a/tests/sample_data/datasets/underfolder_minimnist_schemas/bad.schema b/tests/sample_data/datasets/underfolder_minimnist_schemas/bad.schema
index 1c4704b..a698084 100644
--- a/tests/sample_data/datasets/underfolder_minimnist_schemas/bad.schema
+++ b/tests/sample_data/datasets/underfolder_minimnist_schemas/bad.schema
@@ -1,1 +1,1 @@
-print("😈")
\ No newline at end of file
+print("😈")
diff --git a/tests/sample_data/datasets/underfolder_minimnist_schemas/deep.schema b/tests/sample_data/datasets/underfolder_minimnist_schemas/deep.schema
index bfe456c..9c5660d 100644
--- a/tests/sample_data/datasets/underfolder_minimnist_schemas/deep.schema
+++ b/tests/sample_data/datasets/underfolder_minimnist_schemas/deep.schema
@@ -3,10 +3,10 @@ from schema import And, Optional
 
 deep = True
 schema = {
-    'cfg': object,
-    'numbers': object,
-    'image': np.ndarray,
-    'image_mask': And(np.ndarray, lambda x: len(x.shape) >= 2),
-    'image_maskinv': And(np.ndarray, lambda x: x.shape[0] == x.shape[1]),
-    Optional(str): object
+    "cfg": object,
+    "numbers": object,
+    "image": np.ndarray,
+    "image_mask": And(np.ndarray, lambda x: len(x.shape) >= 2),
+    "image_maskinv": And(np.ndarray, lambda x: x.shape[0] == x.shape[1]),
+    Optional(str): object,
 }
diff --git a/tests/sample_data/piper/dags/complex/dag_parsed.yml b/tests/sample_data/piper/dags/complex/dag_parsed.yml
new file mode 100644
index 0000000..824e337
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/dag_parsed.yml
@@ -0,0 +1,210 @@
+nodes:
+  detection:
+    args:
+      fake_time: 0.01
+    command: python fake_detector.py
+    inputs:
+      input_folders:
+      - /tmp/pipe/output/A_remapped
+      - /tmp/pipe/output/B_remapped
+      - /tmp/pipe/output/C_remapped
+    inputs_schema:
+      input_folders: /path/to/validation_remapped.schema
+    outputs:
+      output_folders:
+      - /tmp/pipe/output/A_detected
+      - /tmp/pipe/output/B_detected
+      - /tmp/pipe/output/C_detected
+    outputs_schema:
+      output_folders: /path/to/validation_detected.schema
+  filter_keys@A:
+    args:
+      keys:
+      - image_A
+      - inst_A
+      - metadata_A
+    command: pipelime underfolder filter_keys
+    inputs:
+      input_folder: /tmp/pipe/output/A_generated
+    outputs:
+      output_folder: /tmp/pipe/output/A_filtered
+  filter_keys@B:
+    args:
+      keys:
+      - image_B
+      - inst_B
+      - metadata_B
+    command: pipelime underfolder filter_keys
+    inputs:
+      input_folder: /tmp/pipe/output/B_generated
+    outputs:
+      output_folder: /tmp/pipe/output/B_filtered
+  filter_keys@C:
+    args:
+      keys:
+      - image_C
+      - inst_C
+      - metadata_C
+    command: pipelime underfolder filter_keys
+    inputs:
+      input_folder: /tmp/pipe/output/C_generated
+    outputs:
+      output_folder: /tmp/pipe/output/C_filtered
+  generator@A:
+    args:
+      size: 10
+      suffix: _A
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/A_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@B:
+    args:
+      size: 10
+      suffix: _B
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/B_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@C:
+    args:
+      size: 10
+      suffix: _C
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/C_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  remap_keys@A:
+    args:
+      keys:
+      - - image_A
+        - image
+      - - inst_A
+        - inst
+      - - metadata_A
+        - metadata
+      remove: {}
+    command: pipelime underfolder remap_keys
+    inputs:
+      input_folder: /tmp/pipe/output/A_subsampled
+    inputs_schema:
+      input_folder: /path/to/validation.schema
+    outputs:
+      output_folder: /tmp/pipe/output/A_remapped
+  remap_keys@B:
+    args:
+      keys:
+      - - image_B
+        - image
+      - - inst_B
+        - inst
+      - - metadata_B
+        - metadata
+      remove: {}
+    command: pipelime underfolder remap_keys
+    inputs:
+      input_folder: /tmp/pipe/output/B_subsampled
+    inputs_schema:
+      input_folder: /path/to/validation.schema
+    outputs:
+      output_folder: /tmp/pipe/output/B_remapped
+  remap_keys@C:
+    args:
+      keys:
+      - - image_C
+        - image
+      - - inst_C
+        - inst
+      - - metadata_C
+        - metadata
+      remove: {}
+    command: pipelime underfolder remap_keys
+    inputs:
+      input_folder: /tmp/pipe/output/C_subsampled
+    inputs_schema:
+      input_folder: /path/to/validation.schema
+    outputs:
+      output_folder: /tmp/pipe/output/C_remapped
+  shuffle@0:
+    args:
+      seed: 10
+    command: pipelime underfolder shuffle
+    inputs:
+      input_folder: /tmp/pipe/output/split_train
+    outputs:
+      output_folder: /tmp/pipe/output/split_train_shuffled
+  shuffle@1:
+    args:
+      seed: 10
+    command: pipelime underfolder shuffle
+    inputs:
+      input_folder: /tmp/pipe/output/split_validation
+    outputs:
+      output_folder: /tmp/pipe/output/split_validation_shuffled
+  shuffle@2:
+    args:
+      seed: 10
+    command: pipelime underfolder shuffle
+    inputs:
+      input_folder: /tmp/pipe/output/split_test
+    outputs:
+      output_folder: /tmp/pipe/output/split_test_shuffled
+  split:
+    args:
+      splits:
+      - 0.8
+      - 0.1
+      - 0.1
+    command: pipelime underfolder split
+    inputs:
+      input_folder: /tmp/pipe/output/summed
+    outputs:
+      output_folders:
+      - /tmp/pipe/output/split_train
+      - /tmp/pipe/output/split_validation
+      - /tmp/pipe/output/split_test
+  subsample@A:
+    args:
+      factor: 0.5
+    command: pipelime underfolder subsample
+    inputs:
+      input_folder: /tmp/pipe/output/A_filtered
+    outputs:
+      output_folder: /tmp/pipe/output/A_subsampled
+  subsample@B:
+    args:
+      factor: 0.5
+    command: pipelime underfolder subsample
+    inputs:
+      input_folder: /tmp/pipe/output/B_filtered
+    outputs:
+      output_folder: /tmp/pipe/output/B_subsampled
+  subsample@C:
+    args:
+      factor: 0.5
+    command: pipelime underfolder subsample
+    inputs:
+      input_folder: /tmp/pipe/output/C_filtered
+    outputs:
+      output_folder: /tmp/pipe/output/C_subsampled
+  summation:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders:
+      - /tmp/pipe/output/A_detected
+      - /tmp/pipe/output/B_detected
+      - /tmp/pipe/output/C_detected
+    outputs:
+      output_folder: /tmp/pipe/output/summed
+  sumup:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders:
+      - /tmp/pipe/output/split_train_shuffled
+      - /tmp/pipe/output/split_validation_shuffled
+      - /tmp/pipe/output/split_test_shuffled
+    outputs:
+      output_folder: /tmp/pipe/final
diff --git a/tests/sample_data/piper/dags/complex/dag_to_parse.yml b/tests/sample_data/piper/dags/complex/dag_to_parse.yml
new file mode 100644
index 0000000..4e042c1
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/dag_to_parse.yml
@@ -0,0 +1,169 @@
+# --------------------------------------------------------------------------------------
+# ROOT key is "nodes", a mapping of {node_name:str -> node_params:dict}
+nodes:
+  # ------------------------------------------------------------------------------------
+  # GENERATOR NODE, creates several ToyDatasets as input
+  # ------------------------------------------------------------------------------------
+  
+  # ↓ this is the node name, has to be unique
+  generator:
+    # ↓ this keyword indicates that the node will be repeated several times based on 
+    # ↓ values in the 'items' field
+    foreach:
+      # ↓ 'items' field is a list of values to be used in the 'foreach' loop
+      items: $var(params.flags)
+      # ↓ 'do' field contains the real node definition with placeholders for the 
+      # ↓ elements that will be replaced by the 'foreach' loop (e.e. $iter(item) will
+      # ↓ be replaced by the value of 'item' in the 'foreach' loop)
+      do:
+        command: pipelime datasets generate_toy_dataset
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_generated
+        args:
+          suffix: _$iter(item)
+          size: $var(params.generated_dataset_size)
+        # ↓ these are the schemas of the outputs of the node
+        outputs_schema:
+          output_folder: $var(params.schemas.generated)
+  # ------------------------------------------------------------------------------------
+  # FILTER_KEYS NODE, filter some keys for each sample, for each dataset
+  # ------------------------------------------------------------------------------------
+  filter_keys:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime underfolder filter_keys
+        inputs:
+          input_folder: $var(params.output_folder)/$iter(item)_generated
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_filtered
+        args:
+          keys:
+            foreach:
+              items: $var(params.filter_keys)
+              do: $argiter(item)_$iter(item)
+  # ------------------------------------------------------------------------------------
+  # SUBSAMPLE NODE, reduce size of each dataset
+  # ------------------------------------------------------------------------------------
+  subsample:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime underfolder subsample
+        inputs:
+          input_folder: $var(params.output_folder)/$iter(item)_filtered
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_subsampled
+        args:
+          factor: 0.5
+        
+  # ------------------------------------------------------------------------------------
+  # REMAP_KEYS NODE, rename keys of each dataset into a common naming scheme
+  # ------------------------------------------------------------------------------------
+  remap_keys:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime underfolder remap_keys
+        inputs:
+          input_folder: $var(params.output_folder)/$iter(item)_subsampled
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_remapped
+        args:
+          keys@0:
+            foreach:
+              items: $var(params.filter_keys)
+              do: $argiter(item)_$iter(item)
+          keys@1:
+            foreach:
+              items: $var(params.filter_keys)
+              do: $argiter(item)
+          remove: {}
+        # ↓ these are the schemas of the inputs of the node
+        inputs_schema:
+          input_folder: $var(params.schemas.generated)
+  # ------------------------------------------------------------------------------------
+  # DETECTION NODE, launch custom detection node on each input dataset
+  # ------------------------------------------------------------------------------------
+  detection:
+    # ↓ this is the base command to be executed
+    command: $var(params.custom_command)
+    # ↓ this is mapping of inputs arguments {arg_name:str -> values:str}
+    inputs:
+      input_folders:
+        # ↓ this is a for each for arguments itself, it is a list of values to be used
+        # ↓ in the 'foreach' loop with the same behavior as the 'foreach' keyword used
+        # ↓ above for nodes
+        foreach:
+          items: $var(params.flags)
+          do: $var(params.output_folder)/$argiter(item)_remapped
+    # ↓ this is mapping of outputs arguments {arg_name:str -> values:str}
+    outputs:
+      output_folders:
+        foreach:
+          items: $var(params.flags)
+          do: $var(params.output_folder)/$argiter(item)_detected
+    # ↓ this is mapping of generic arguments {arg_name:str -> values:str}     
+    args:
+      fake_time: 0.01
+    inputs_schema:
+      input_folders: $var(params.schemas.remapped)
+    outputs_schema:
+      output_folders: $var(params.schemas.detected)
+  # ------------------------------------------------------------------------------------
+  # SUMMATION NODE, sum each dataset into a single dataset
+  # ------------------------------------------------------------------------------------
+  summation:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders: 
+        foreach:
+          items: $var(params.flags)
+          do: $var(params.output_folder)/$argiter(item)_detected
+    outputs:
+      output_folder: $var(params.output_folder)/summed
+  # ------------------------------------------------------------------------------------
+  # SPLIT NODE, split datasets into train, validation and test
+  # ------------------------------------------------------------------------------------
+  split:
+    command: pipelime underfolder split
+    args:
+      splits:
+        foreach:
+          items: $var(params.splits)
+          do: $argiter(item.p)
+    inputs:
+      input_folder: $var(params.output_folder)/summed
+    outputs:
+      output_folders:
+        foreach:
+          items: $var(params.splits)
+          do: $var(params.output_folder)/split_$argiter(item.name)
+  # ------------------------------------------------------------------------------------
+  # SHUFFLE NODE, useless shuffle after split
+  # ------------------------------------------------------------------------------------
+  shuffle:
+    foreach:
+      items: $var(params.splits)
+      do:
+        command: pipelime underfolder shuffle
+        inputs:
+          input_folder: $var(params.output_folder)/split_$iter(item.name)
+        outputs:
+          output_folder: $var(params.output_folder)/split_$iter(item.name)_shuffled
+        args:
+          seed: 10
+  # ------------------------------------------------------------------------------------
+  # SUMUP NODE, final merge
+  # ------------------------------------------------------------------------------------
+  sumup:
+    command: pipelime underfolder sum
+    inputs:
+      input_folders:
+        foreach:
+          items: $var(params.splits)
+          do: $var(params.output_folder)/split_$argiter(item.name)_shuffled
+    outputs:
+      output_folder: $var(params.final_folder)
+
+
diff --git a/tests/sample_data/piper/dags/complex/final_validation.py b/tests/sample_data/piper/dags/complex/final_validation.py
new file mode 100644
index 0000000..e748d48
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/final_validation.py
@@ -0,0 +1,18 @@
+def final_validation(**kwargs):
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+
+    final_folder = kwargs.get("FINAL_FOLDER", None)
+    size = kwargs.get("SIZE", None)
+    assert final_folder is not None
+    assert size is not None
+
+    reader = UnderfolderReader(folder=final_folder)
+    assert len(reader) == (size * 3) // 2
+
+    suffixes_set = set()
+    for sample in reader:
+        print(list(sample.keys()))
+        print(sample["metadata"])
+        suffixes_set.add(sample["metadata"]["suffix"])
+
+    assert len(suffixes_set) == 3
diff --git a/tests/sample_data/piper/dags/complex/params.yml b/tests/sample_data/piper/dags/complex/params.yml
new file mode 100644
index 0000000..879ea71
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/params.yml
@@ -0,0 +1,24 @@
+params:
+  filter_keys:
+  - image
+  - inst
+  - metadata
+  flags:
+  - A
+  - B
+  - C
+  custom_command: python fake_detector.py
+  output_folder: /tmp/pipe/output
+  final_folder: /tmp/pipe/final
+  generated_dataset_size: 10
+  splits:
+  - name: train
+    p: 0.8
+  - name: validation
+    p: 0.1
+  - name: test
+    p: 0.1
+  schemas:
+    generated: /path/to/validation.schema
+    remapped: /path/to/validation_remapped.schema
+    detected: /path/to/validation_detected.schema
diff --git a/tests/sample_data/piper/dags/complex/params_with_placeholders.yml b/tests/sample_data/piper/dags/complex/params_with_placeholders.yml
new file mode 100644
index 0000000..ffc42f4
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/params_with_placeholders.yml
@@ -0,0 +1,24 @@
+params:
+  filter_keys:
+  - image
+  - inst
+  - metadata
+  flags:
+  - A
+  - B
+  - C
+  custom_command: '@str(CUSTOM_COMMAND)'
+  output_folder: '@str(OUTPUT_FOLDER)'
+  final_folder: '@str(FINAL_FOLDER)'
+  generated_dataset_size: '@int(SIZE)'
+  splits:
+  - name: train
+    p: 0.8
+  - name: validation
+    p: 0.1
+  - name: test
+    p: 0.1
+  schemas:
+    generated: '@str(SCHEMA_GENERATED)'
+    remapped: '@str(SCHEMA_REMAPPED)'
+    detected: '@str(SCHEMA_DETECTED)'
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/complex/schema_detected.schema b/tests/sample_data/piper/dags/complex/schema_detected.schema
new file mode 100644
index 0000000..6ed3202
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/schema_detected.schema
@@ -0,0 +1,8 @@
+from schema import Optional
+
+deep = True
+schema = {
+    "image": object,
+    "fake_detection": {"keypoints": object},
+    Optional(str): object,
+}
diff --git a/tests/sample_data/piper/dags/complex/schema_generated.schema b/tests/sample_data/piper/dags/complex/schema_generated.schema
new file mode 100644
index 0000000..8c377d3
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/schema_generated.schema
@@ -0,0 +1,4 @@
+from schema import Optional, Regex
+
+deep = False
+schema = {Regex("image_*"): object, Optional(str): object}
diff --git a/tests/sample_data/piper/dags/complex/schema_remapped.schema b/tests/sample_data/piper/dags/complex/schema_remapped.schema
new file mode 100644
index 0000000..e600501
--- /dev/null
+++ b/tests/sample_data/piper/dags/complex/schema_remapped.schema
@@ -0,0 +1,4 @@
+from schema import Optional
+
+deep = False
+schema = {"image": object, Optional(str): object}
diff --git a/tests/sample_data/piper/dags/invalid_arg_foreach/dag_parsed.yml b/tests/sample_data/piper/dags/invalid_arg_foreach/dag_parsed.yml
new file mode 100644
index 0000000..a1cac11
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_arg_foreach/dag_parsed.yml
@@ -0,0 +1,8 @@
+nodes:
+  generator:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: ['item_1', 'item_2', 'item_3']
+    args:
+      suffix: suffix
+      size: 50
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_arg_foreach/dag_to_parse.yml b/tests/sample_data/piper/dags/invalid_arg_foreach/dag_to_parse.yml
new file mode 100644
index 0000000..20fab0d
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_arg_foreach/dag_to_parse.yml
@@ -0,0 +1,11 @@
+nodes:
+  generator:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder:
+        foreach:
+          items: [1,2,3]
+          dos: item_$iter(item)
+    args:
+      suffix: suffix
+      size: 50
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_arg_foreach_content/dag_parsed.yml b/tests/sample_data/piper/dags/invalid_arg_foreach_content/dag_parsed.yml
new file mode 100644
index 0000000..a1cac11
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_arg_foreach_content/dag_parsed.yml
@@ -0,0 +1,8 @@
+nodes:
+  generator:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: ['item_1', 'item_2', 'item_3']
+    args:
+      suffix: suffix
+      size: 50
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_arg_foreach_content/dag_to_parse.yml b/tests/sample_data/piper/dags/invalid_arg_foreach_content/dag_to_parse.yml
new file mode 100644
index 0000000..51fce0f
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_arg_foreach_content/dag_to_parse.yml
@@ -0,0 +1,11 @@
+nodes:
+  generator:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder:
+        foreach:
+          items: 10
+          do: 4
+    args:
+      suffix: suffix
+      size: 50
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_node_foreach/dag_parsed.yml b/tests/sample_data/piper/dags/invalid_node_foreach/dag_parsed.yml
new file mode 100644
index 0000000..e69de29
diff --git a/tests/sample_data/piper/dags/invalid_node_foreach/dag_to_parse.yml b/tests/sample_data/piper/dags/invalid_node_foreach/dag_to_parse.yml
new file mode 100644
index 0000000..2942bc1
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_node_foreach/dag_to_parse.yml
@@ -0,0 +1,11 @@
+nodes:
+  generator:
+    foreach:
+      items: 'A'
+      dos:
+        command: pipelime datasets generate_toy_dataset
+        outputs:
+          output_folder: /tmp/output
+        args:
+          suffix: _$iter(item)
+          size: 50
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_node_foreach_do_content/dag_parsed.yml b/tests/sample_data/piper/dags/invalid_node_foreach_do_content/dag_parsed.yml
new file mode 100644
index 0000000..e69de29
diff --git a/tests/sample_data/piper/dags/invalid_node_foreach_do_content/dag_to_parse.yml b/tests/sample_data/piper/dags/invalid_node_foreach_do_content/dag_to_parse.yml
new file mode 100644
index 0000000..0f02267
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_node_foreach_do_content/dag_to_parse.yml
@@ -0,0 +1,5 @@
+nodes:
+  generator:
+    foreach:
+      items: 'A'
+      do: 'what?'
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_nodes/dag_parsed.yml b/tests/sample_data/piper/dags/invalid_nodes/dag_parsed.yml
new file mode 100644
index 0000000..6c6762a
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_nodes/dag_parsed.yml
@@ -0,0 +1,3 @@
+nodes:
+  main:
+    commands: pipelime datasets generate_toy_dataset
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_nodes/dag_to_parse.yml b/tests/sample_data/piper/dags/invalid_nodes/dag_to_parse.yml
new file mode 100644
index 0000000..6c6762a
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_nodes/dag_to_parse.yml
@@ -0,0 +1,3 @@
+nodes:
+  main:
+    commands: pipelime datasets generate_toy_dataset
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_params/dag_parsed.yml b/tests/sample_data/piper/dags/invalid_params/dag_parsed.yml
new file mode 100644
index 0000000..5cbd816
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_params/dag_parsed.yml
@@ -0,0 +1,8 @@
+nodes:
+  main:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/folder
+    args:
+      suffix: suffix
+      size: 100
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/invalid_params/dag_to_parse.yml b/tests/sample_data/piper/dags/invalid_params/dag_to_parse.yml
new file mode 100644
index 0000000..2399e9d
--- /dev/null
+++ b/tests/sample_data/piper/dags/invalid_params/dag_to_parse.yml
@@ -0,0 +1,8 @@
+nodes:
+  main:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/folder
+    args:
+      suffix: suffix
+      size: $var(params.size)
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/no_params/dag_parsed.yml b/tests/sample_data/piper/dags/no_params/dag_parsed.yml
new file mode 100644
index 0000000..5cbd816
--- /dev/null
+++ b/tests/sample_data/piper/dags/no_params/dag_parsed.yml
@@ -0,0 +1,8 @@
+nodes:
+  main:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/folder
+    args:
+      suffix: suffix
+      size: 100
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/no_params/dag_to_parse.yml b/tests/sample_data/piper/dags/no_params/dag_to_parse.yml
new file mode 100644
index 0000000..5cbd816
--- /dev/null
+++ b/tests/sample_data/piper/dags/no_params/dag_to_parse.yml
@@ -0,0 +1,8 @@
+nodes:
+  main:
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/folder
+    args:
+      suffix: suffix
+      size: 100
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error/dag_parsed.yml b/tests/sample_data/piper/dags/validation_error/dag_parsed.yml
new file mode 100644
index 0000000..b05fdb5
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error/dag_parsed.yml
@@ -0,0 +1,28 @@
+nodes:
+  generator@A:
+    args:
+      size: 10
+      suffix: _A
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/A_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@B:
+    args:
+      size: 10
+      suffix: _B
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/B_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@C:
+    args:
+      size: 10
+      suffix: _C
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/C_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error/dag_to_parse.yml b/tests/sample_data/piper/dags/validation_error/dag_to_parse.yml
new file mode 100644
index 0000000..7e3016a
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error/dag_to_parse.yml
@@ -0,0 +1,14 @@
+nodes:
+  generator:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime datasets generate_toy_dataset
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_generated
+        args:
+          suffix: _$iter(item)
+          size: $var(params.generated_dataset_size)
+        # ↓ these are the schemas of the outputs of the node
+        outputs_schema:
+          output_folder: $var(params.schemas.generated)
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error/params.yml b/tests/sample_data/piper/dags/validation_error/params.yml
new file mode 100644
index 0000000..52197e0
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error/params.yml
@@ -0,0 +1,9 @@
+params:
+  flags:
+  - A
+  - B
+  - C
+  output_folder: /tmp/pipe/output
+  generated_dataset_size: 10
+  schemas:
+    generated: /path/to/validation.schema
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error/params_with_placeholders.yml b/tests/sample_data/piper/dags/validation_error/params_with_placeholders.yml
new file mode 100644
index 0000000..7f203fa
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error/params_with_placeholders.yml
@@ -0,0 +1,9 @@
+params:
+  flags:
+  - A
+  - B
+  - C
+  output_folder: '@str(OUTPUT_FOLDER)'
+  generated_dataset_size: '@int(SIZE)'
+  schemas:
+    generated: '@str(SCHEMA_GENERATED)'
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error/schema_generated.schema b/tests/sample_data/piper/dags/validation_error/schema_generated.schema
new file mode 100644
index 0000000..314233b
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error/schema_generated.schema
@@ -0,0 +1,4 @@
+from schema import Optional, Regex
+
+deep = False
+schema = {"image_not_present": object}
diff --git a/tests/sample_data/piper/dags/validation_error_noschema_file/dag_parsed.yml b/tests/sample_data/piper/dags/validation_error_noschema_file/dag_parsed.yml
new file mode 100644
index 0000000..b05fdb5
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error_noschema_file/dag_parsed.yml
@@ -0,0 +1,28 @@
+nodes:
+  generator@A:
+    args:
+      size: 10
+      suffix: _A
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/A_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@B:
+    args:
+      size: 10
+      suffix: _B
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/B_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
+  generator@C:
+    args:
+      size: 10
+      suffix: _C
+    command: pipelime datasets generate_toy_dataset
+    outputs:
+      output_folder: /tmp/pipe/output/C_generated
+    outputs_schema:
+      output_folder: /path/to/validation.schema
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error_noschema_file/dag_to_parse.yml b/tests/sample_data/piper/dags/validation_error_noschema_file/dag_to_parse.yml
new file mode 100644
index 0000000..7e3016a
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error_noschema_file/dag_to_parse.yml
@@ -0,0 +1,14 @@
+nodes:
+  generator:
+    foreach:
+      items: $var(params.flags)
+      do:
+        command: pipelime datasets generate_toy_dataset
+        outputs:
+          output_folder: $var(params.output_folder)/$iter(item)_generated
+        args:
+          suffix: _$iter(item)
+          size: $var(params.generated_dataset_size)
+        # ↓ these are the schemas of the outputs of the node
+        outputs_schema:
+          output_folder: $var(params.schemas.generated)
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error_noschema_file/params.yml b/tests/sample_data/piper/dags/validation_error_noschema_file/params.yml
new file mode 100644
index 0000000..52197e0
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error_noschema_file/params.yml
@@ -0,0 +1,9 @@
+params:
+  flags:
+  - A
+  - B
+  - C
+  output_folder: /tmp/pipe/output
+  generated_dataset_size: 10
+  schemas:
+    generated: /path/to/validation.schema
\ No newline at end of file
diff --git a/tests/sample_data/piper/dags/validation_error_noschema_file/params_with_placeholders.yml b/tests/sample_data/piper/dags/validation_error_noschema_file/params_with_placeholders.yml
new file mode 100644
index 0000000..7f203fa
--- /dev/null
+++ b/tests/sample_data/piper/dags/validation_error_noschema_file/params_with_placeholders.yml
@@ -0,0 +1,9 @@
+params:
+  flags:
+  - A
+  - B
+  - C
+  output_folder: '@str(OUTPUT_FOLDER)'
+  generated_dataset_size: '@int(SIZE)'
+  schemas:
+    generated: '@str(SCHEMA_GENERATED)'
\ No newline at end of file
diff --git a/tests/sample_data/piper/piper_commands/fake_detector.py b/tests/sample_data/piper/piper_commands/fake_detector.py
new file mode 100644
index 0000000..fddb64e
--- /dev/null
+++ b/tests/sample_data/piper/piper_commands/fake_detector.py
@@ -0,0 +1,87 @@
+import click
+from pipelime.pipes.piper import Piper, PiperCommand
+from pipelime.tools.progress import pipelime_track
+
+
+@click.command("fake_detector")
+@click.option(
+    "-i",
+    "--input_folders",
+    type=click.Path(exists=True),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@click.option(
+    "-o",
+    "--output_folders",
+    type=click.Path(),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@click.option("-t", "--fake_time", type=float, default=0.0)
+@Piper.command(
+    inputs=["input_folders"],
+    outputs=["output_folders"],
+)
+def fake_detector(input_folders: str, output_folders: str, fake_time: float):
+
+    from pipelime.sequences.readers.filesystem import UnderfolderReader
+    from pipelime.sequences.writers.filesystem import UnderfolderWriterV2
+    from pipelime.sequences.samples import SamplesSequence
+    import numpy as np
+
+    # checks size of input/output folders
+    if len(input_folders) != len(output_folders):
+        raise click.UsageError(
+            "The number of input folders must be equal to the number of output folders"
+        )
+
+    for index, input_folder in enumerate(input_folders):
+
+        reader = UnderfolderReader(input_folder)
+
+        # ADD fake detections to each sample
+        out_samples = []
+        for sample in pipelime_track(
+            reader,
+            track_callback=PiperCommand.instance.generate_progress_callback(
+                index * 2,
+                len(reader),
+            ),
+        ):
+            sample["fake_detection"] = {
+                "keypoints": [np.random.randint(0, 100, (10, 4)).tolist()]
+            }
+            out_samples.append(sample)
+
+            if fake_time > 0:
+                # wait for fake_time
+                import time
+
+                time.sleep(fake_time)
+
+        # build out sequence
+        out_sequence = SamplesSequence(out_samples)
+
+        # add fake detections to the extensions map
+        template = reader.get_reader_template()
+        template.extensions_map.update({"fake_detection": "yml"})
+
+        # write out sequence
+        writer = UnderfolderWriterV2(
+            output_folders[index],
+            reader_template=template,
+            file_handling=UnderfolderWriterV2.FileHandling.COPY_IF_NOT_CACHED,
+            copy_mode=UnderfolderWriterV2.CopyMode.HARD_LINK,
+            progress_callback=PiperCommand.instance.generate_progress_callback(
+                chunk_index=index * 2 + 1,
+                total_chunks=len(input_folders),
+            ),
+        )
+        writer(out_sequence)
+
+
+if __name__ == "__main__":
+    fake_detector()
diff --git a/tests/sample_data/piper/piper_commands/no_command_name.py b/tests/sample_data/piper/piper_commands/no_command_name.py
new file mode 100644
index 0000000..ea035c7
--- /dev/null
+++ b/tests/sample_data/piper/piper_commands/no_command_name.py
@@ -0,0 +1,34 @@
+import click
+from pipelime.pipes.piper import Piper
+
+
+@click.command()
+@click.option(
+    "-i",
+    "--input_folders",
+    type=click.Path(exists=True),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@click.option(
+    "-o",
+    "--output_folders",
+    type=click.Path(),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@Piper.command(
+    inputs=["input_folders"],
+    outputs=["output_folders"],
+)
+def no_command_name(
+    input_folders: str,
+    output_folders: str,
+):
+    pass
+
+
+if __name__ == "__main__":
+    no_command_name()
diff --git a/tests/sample_data/piper/piper_commands/not_a_piper.py b/tests/sample_data/piper/piper_commands/not_a_piper.py
new file mode 100644
index 0000000..a7c7a20
--- /dev/null
+++ b/tests/sample_data/piper/piper_commands/not_a_piper.py
@@ -0,0 +1,29 @@
+import click
+
+
+@click.command("not_a_piper")
+@click.option(
+    "-i",
+    "--input_folders",
+    type=click.Path(exists=True),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@click.option(
+    "-o",
+    "--output_folders",
+    type=click.Path(),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+def not_a_piper(
+    input_folders: str,
+    output_folders: str,
+):
+    pass
+
+
+if __name__ == "__main__":
+    not_a_piper()
diff --git a/tests/sample_data/piper/piper_commands/wrong_inputs.py b/tests/sample_data/piper/piper_commands/wrong_inputs.py
new file mode 100644
index 0000000..39907a4
--- /dev/null
+++ b/tests/sample_data/piper/piper_commands/wrong_inputs.py
@@ -0,0 +1,35 @@
+import click
+from pipelime.pipes.piper import Piper
+
+
+@click.command("wrong_outputs")
+@click.option(
+    "-i",
+    "--input_folders",
+    type=click.Path(exists=True),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@click.option(
+    "-o",
+    "--output_folders",
+    type=click.Path(),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@Piper.command(
+    inputs=["input_folder"],
+    outputs=["output_folders"],
+)
+def wrong_outputs(
+    input_folders: str,
+    output_folders: str,
+):
+
+    pass
+
+
+if __name__ == "__main__":
+    wrong_outputs()
diff --git a/tests/sample_data/piper/piper_commands/wrong_outputs.py b/tests/sample_data/piper/piper_commands/wrong_outputs.py
new file mode 100644
index 0000000..12ecee4
--- /dev/null
+++ b/tests/sample_data/piper/piper_commands/wrong_outputs.py
@@ -0,0 +1,35 @@
+import click
+from pipelime.pipes.piper import Piper
+
+
+@click.command("wrong_outputs")
+@click.option(
+    "-i",
+    "--input_folders",
+    type=click.Path(exists=True),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@click.option(
+    "-o",
+    "--output_folders",
+    type=click.Path(),
+    required=True,
+    multiple=True,
+    help="The input folder",
+)
+@Piper.command(
+    inputs=["input_folders"],
+    outputs=["output_folder"],
+)
+def wrong_outputs(
+    input_folders: str,
+    output_folders: str,
+):
+
+    pass
+
+
+if __name__ == "__main__":
+    wrong_outputs()
diff --git a/tests/test_pipelime.py b/tests/test_pipelime.py
index 9e7ac6e..a41a941 100644
--- a/tests/test_pipelime.py
+++ b/tests/test_pipelime.py
@@ -3,9 +3,6 @@
 """Tests for `pipelime` package."""
 
 import pytest
-from click.testing import CliRunner
-
-from pipelime import cli, pipelime
 
 
 @pytest.fixture
@@ -24,17 +21,6 @@ def test_content(response):
     # assert 'GitHub' in BeautifulSoup(response.content).title.string
 
 
-# def test_command_line_interface():
-#     """Test the CLI."""
-#     runner = CliRunner()
-#     result = runner.invoke(cli.main)
-#     assert result.exit_code == 0
-#     assert 'pipelime.cli.main' in result.output
-#     help_result = runner.invoke(cli.main, ['--help'])
-#     assert help_result.exit_code == 0
-#     assert '--help  Show this message and exit.' in help_result.output
-
-
 def test_toy_dataset_small(toy_dataset_small):
 
     for x in [