Repository: catalyst-team/catalyst

Issue:
Multiple dataset source for tag2label
It will be nice if I can choose not one input dir here https://github.com/catalyst-team/catalyst/blob/master/catalyst/contrib/scripts/tag2label.py

For example it can looks like:
`catalyst-contrib tag2label --in-dir=dataset1,dataset2`


Discussion:

Resolving patch:
diff --git a/catalyst/contrib/scripts/tag2label.py b/catalyst/contrib/scripts/tag2label.py
--- a/catalyst/contrib/scripts/tag2label.py
+++ b/catalyst/contrib/scripts/tag2label.py
@@ -6,15 +6,35 @@
     prepare_dataset_labeling, separate_tags
 
 
-def prepare_df_from_dirs(in_dir, tag_column_name):
-    if not in_dir.endswith("/"):
-        in_dir = f"{in_dir}/"
-
-    dataset = create_dataset(
-        f"{in_dir}/**", process_fn=lambda x: x.replace(f"{in_dir}", "")
-    )
-    df = create_dataframe(dataset, columns=[tag_column_name, "filepath"])
-
+def prepare_df_from_dirs(in_dirs, tag_column_name):
+    dfs = []
+    splitted_dirs = in_dirs.strip(',').split(',')
+
+    def process_fn(x):
+        if len(splitted_dirs) == 1:
+            # remove all in_dir part from path
+            return x.replace(f"{in_dir}", "")
+        else:
+            # leaves last part of in_dir path,
+            #  which identifies separate in_dir
+            return x.replace(
+                f"{in_dir}",
+                f"{in_dir.split('/')[-2]}/")
+
+    for in_dir in splitted_dirs:
+        if not in_dir.endswith("/"):
+            in_dir = f"{in_dir}/"
+
+        dataset = create_dataset(
+            f"{in_dir}/**",
+            process_fn=process_fn)
+
+        dfs.append(
+            create_dataframe(
+                dataset,
+                columns=[tag_column_name, "filepath"]))
+
+    df = pd.concat(dfs).reset_index(drop=True)
     return df
 
 
@@ -29,7 +49,8 @@ def build_args(parser):
         "--in-dir",
         type=str,
         default=None,
-        help="Path to directory with dataset."
+        help="Path to directory with dataset"
+             "or paths separated by commas for several datasets"
     )
 
     parser.add_argument(


Test patch:
diff --git a/catalyst/contrib/scripts/tests/__init__.py b/catalyst/contrib/scripts/tests/__init__.py
new file mode 100644
diff --git a/catalyst/contrib/scripts/tests/test_tag2label.py b/catalyst/contrib/scripts/tests/test_tag2label.py
new file mode 100644
--- /dev/null
+++ b/catalyst/contrib/scripts/tests/test_tag2label.py
@@ -0,0 +1,70 @@
+import shutil
+from pathlib import Path
+from ..tag2label import prepare_df_from_dirs
+
+
+def prepare_dataset():
+    shutil.rmtree('datasets', ignore_errors=True)
+
+    # dummy datasets e.g. root1 and root2
+    root1 = Path('datasets/root1')
+    root1.mkdir(parents=True, exist_ok=True)
+    root2 = Path('datasets/root2')
+    root2.mkdir(parents=True, exist_ok=True)
+
+    # dummy labels folders for root1
+    root1_act1 = root1 / 'act1'
+    root1_act2 = root1 / 'act2'
+    root1_act1.mkdir()
+    root1_act2.mkdir()
+
+    # dummy labels folders for root2
+    root2_act1 = root2 / 'act1'
+    root2_act2 = root2 / 'act2'
+    root2_act1.mkdir()
+    root2_act2.mkdir()
+
+    # dummy files for root1
+    a = root1_act1 / 'a.txt'
+    b = root1_act1 / 'b.txt'
+    c = root1_act2 / 'c.txt'
+
+    # dummy files for root2
+    d = root2_act1 / 'd.txt'
+    e = root2_act2 / 'e.txt'
+    f = root2_act2 / 'f.txt'
+
+    for file in [a, b, c, d, e, f]:
+        file.touch()
+
+
+def test_prepare_df_from_dirs_one():
+    def check_filepath(f):
+        return f.startswith('act1') or f.startswith('act2')
+
+    prepare_dataset()
+    df = prepare_df_from_dirs('datasets/root1', 'label')
+
+    assert df.shape[0] == 3
+    assert df.filepath.apply(check_filepath).sum().all()
+    assert df.label.isin(['act1', 'act2']).all()
+
+    shutil.rmtree('datasets', ignore_errors=True)
+
+
+def test_prepare_df_from_dirs_multi():
+    def check_filepath(f):
+        return f.startswith('root1/act1') or \
+               f.startswith('root1/act2') or \
+               f.startswith('root2/act1') or \
+               f.startswith('root2/act2')
+
+    prepare_dataset()
+    df = prepare_df_from_dirs(
+        'datasets/root1,datasets/root2',
+        'label')
+    assert df.shape[0] == 6
+    assert df.filepath.apply(check_filepath).sum().all()
+    assert df.label.isin(['act1', 'act2']).all()
+
+    shutil.rmtree('datasets', ignore_errors=True)