Repository: nosarthur/gita

Issue:
Allow automatic group creation on recursive add
Use case: I've got tons of git repos I need to manage, but they are already nicely grouped into folders (e.g. `Work/Client-5/Project-4/Service-1/<code>`). Currently I'm using `gita add -r` to add them all, then grouping using bash (e.g. `for i in $(gita ls); do echo "target-group" | gita group add $i; done`

With #93 we got a lovely `-r` flag on the add command. Would it be possible to add a `--auto-group` flag that only works if `-r` was passed. Preferably, this would cause groups to be created after the add, with one group per parent folder (e.g. a repo stored at `Foo/Bar/Code` would be in two different groups, `Foo`, `Bar`. It would also be fine if the tool auto-created groups as `Foo` and `Foo-Bar` (to avoid issues where subfolders might have the same names as other projects). 

Thoughts? 



Discussion:

Patch:
diff --git a/README.md b/README.md
index d260fac..4b4c516 100644
--- a/README.md
+++ b/README.md
@@ -62,6 +62,8 @@ The additional status symbols denote
 The bookkeeping sub-commands are
 
 - `gita add <repo-path(s)>`: add repo(s) to `gita`
+- `gita add -a <repo-parent-path(s)>`: add repo(s) in <repo-parent-path(s)> recursively
+  and automatically generate hierarchical groups. See the [customization section](#custom) for more details.
 - `gita add -b <bare-repo-path(s)>`: add bare repo(s) to `gita`. See the [customization section](#custom) for more details on setting custom worktree.
 - `gita add -m <main-repo-path(s)>`: add main repo(s) to `gita`. See the [customization section](#custom) for more details.
 - `gita add -r <repo-parent-path(s)>`: add repo(s) in <repo-parent-path(s)> recursively
@@ -207,6 +209,32 @@ gita ll
 gita pull
 ```
 
+It is also possible to recursively add repos within a directory and
+generate hierarchical groups automatically. For example, running
+
+```
+gita add -a src
+```
+on the following folder structure
+```
+src
+├── project1
+│   ├── repo1
+│   └── repo2
+├── repo3
+├── project2
+│   ├── repo4
+│   └── repo5
+└── repo6
+```
+gives rise to
+```
+src:repo1,repo2,repo3,repo4,repo5,repo6
+src-project1:repo1,repo2
+src-project2:repo4,repo5
+```
+
+
 ### define main repos and shadow the global configuration setting with local setting
 
 The so-called main repos contain `.gita` folder for local configurations.
diff --git a/doc/README_CN.md b/doc/README_CN.md
index 45f5926..03d96a7 100644
--- a/doc/README_CN.md
+++ b/doc/README_CN.md
@@ -46,6 +46,7 @@
 基础指令：
 
 - `gita add <repo-path(s)>`: 添加库
+- `gita add -a <repo-parent-path(s)>`:
 - `gita add -b <bare-repo-path(s)>`:
 - `gita add -m <main-repo-path(s)>`:
 - `gita add -r <repo-parent-path(s)>`:
diff --git a/gita/__main__.py b/gita/__main__.py
index 5288632..ee4e7e7 100644
--- a/gita/__main__.py
+++ b/gita/__main__.py
@@ -52,11 +52,16 @@ def f_add(args: argparse.Namespace):
             sub_paths = glob.glob(os.path.join(main_path,'**/'), recursive=True)
             utils.add_repos({}, sub_paths, root=main_path)
     else:
-        if args.recursive:
+        if args.recursive or args.auto_group:
             paths = chain.from_iterable(
                     glob.glob(os.path.join(p, '**/'), recursive=True)
                     for p in args.paths)
-        utils.add_repos(repos, paths, is_bare=args.bare)
+        new_repos = utils.add_repos(repos, paths, is_bare=args.bare)
+        if args.auto_group:
+            new_groups = utils.auto_group(new_repos, args.paths)
+            if new_groups:
+                print(f'Created {len(new_groups)} new group(s).')
+                utils.write_to_groups_file(new_groups, 'a+')
 
 
 def f_rename(args: argparse.Namespace):
@@ -377,12 +382,15 @@ def main(argv=None):
     # bookkeeping sub-commands
     p_add = subparsers.add_parser('add', description='add repo(s)',
             help='add repo(s)')
-    p_add.add_argument('paths', nargs='+', help="repo(s) to add")
+    p_add.add_argument('paths', nargs='+', type=os.path.abspath, help="repo(s) to add")
     xgroup = p_add.add_mutually_exclusive_group()
     xgroup.add_argument('-r', '--recursive', action='store_true',
-            help="recursively add repo(s) in the given path.")
+            help="recursively add repo(s) in the given path(s).")
     xgroup.add_argument('-m', '--main', action='store_true',
             help="make main repo(s), sub-repos are recursively added.")
+    xgroup.add_argument('-a', '--auto-group', action='store_true',
+            help="recursively add repo(s) in the given path(s) "
+                "and create hierarchical groups based on folder structure.")
     xgroup.add_argument('-b', '--bare', action='store_true',
             help="add bare repo(s)")
     p_add.set_defaults(func=f_add)
diff --git a/gita/utils.py b/gita/utils.py
index 21d7a8f..34fc435 100644
--- a/gita/utils.py
+++ b/gita/utils.py
@@ -6,8 +6,8 @@ import platform
 import subprocess
 from functools import lru_cache, partial
 from pathlib import Path
-from typing import List, Dict, Coroutine, Union, Iterator
-from collections import Counter
+from typing import List, Dict, Coroutine, Union, Iterator, Tuple
+from collections import Counter, defaultdict
 
 from . import info
 from . import common
@@ -187,7 +187,7 @@ def _make_name(path: str, repos: Dict[str, Dict[str, str]],
     Given a new repo `path`, create a repo name. By default, basename is used.
     If name collision exists, further include parent path name.
 
-    @param path: It should not be in `repos`
+    @param path: It should not be in `repos` and is absolute
     """
     name = os.path.basename(os.path.normpath(path))
     if name in repos or name_counts[name] > 1:
@@ -202,7 +202,7 @@ def _get_repo_type(path, repo_type, root) -> str:
     """
     if repo_type != '':  # explicitly set
         return repo_type
-    if root == path:  # main repo
+    if root is not None and os.path.normpath(root) == os.path.normpath(path):
         return 'm'
     return ''
 
@@ -215,7 +215,7 @@ def add_repos(repos: Dict[str, Dict[str, str]], new_paths: List[str],
     @param repos: name -> path
     """
     existing_paths = {prop['path'] for prop in repos.values()}
-    new_paths = {os.path.abspath(p) for p in new_paths if is_git(p, is_bare)}
+    new_paths = {p for p in new_paths if is_git(p, is_bare)}
     new_paths = new_paths - existing_paths
     new_repos = {}
     if new_paths:
@@ -236,6 +236,42 @@ def add_repos(repos: Dict[str, Dict[str, str]], new_paths: List[str],
     return new_repos
 
 
+def _generate_dir_hash(repo_path: str, paths: List[str]) -> Tuple[str, ...]:
+    """
+    Return relative parent strings
+
+    For example, if `repo_path` is /a/b/c/d/here, and one of `paths` is /a/b/
+    then return (b, c, d)
+    """
+    for p in paths:
+        if is_relative_to(repo_path, p):
+            break
+    else:
+        return ()
+    return (os.path.basename(p),
+            *os.path.normpath(os.path.relpath(repo_path, p)).split(os.sep)[:-1])
+
+
+def auto_group(repos: Dict[str, Dict[str, str]], paths: List[str]
+        ) -> Dict[str, List[str]]:
+    """
+
+    """
+    # FIXME: the upstream code should make sure that paths are all independent
+    #        i.e., each repo should be contained in one and only one path
+    new_groups = defaultdict(list)
+    for repo_name, prop in repos.items():
+        hash = _generate_dir_hash(prop['path'], paths)
+        if not hash:
+            continue
+        for i in range(1, len(hash)+1):
+            group_name = '-'.join(hash[:i])
+            new_groups[group_name].append(repo_name)
+    # FIXME: need to make sure the new group names don't clash with old ones
+    #        or repo names
+    return new_groups
+
+
 def parse_clone_config(fname: str) -> Iterator[List[str]]:
     """
     Return the url, name, and path of all repos in `fname`.
diff --git a/setup.py b/setup.py
index 8bf0fc9..6fb92eb 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@ with open('README.md', encoding='utf-8') as f:
 setup(
     name='gita',
     packages=['gita'],
-    version='0.15.0',
+    version='0.15.1',
     license='MIT',
     description='Manage multiple git repos with sanity',
     long_description=long_description,


Test patch:
diff --git a/tests/test_main.py b/tests/test_main.py
index 6223660..490f5d2 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -29,7 +29,7 @@ class TestAdd:
         (['add', '-m', '.'], 'm'),
         ])
     @patch('gita.common.get_config_fname')
-    def testAdd(self, mock_path_fname, tmp_path, input, expected):
+    def test_add(self, mock_path_fname, tmp_path, input, expected):
         def side_effect(input, _=None):
             return tmp_path / f'{input}.txt'
         mock_path_fname.side_effect = side_effect
@@ -88,7 +88,7 @@ def test_flags(mock_path_fname, _, __, path_fname, expected, capfd):
 
 class TestLsLl:
     @patch('gita.common.get_config_fname')
-    def testLl(self, mock_path_fname, capfd, tmp_path):
+    def test_ll(self, mock_path_fname, capfd, tmp_path):
         """
         functional test
         """
@@ -128,7 +128,7 @@ class TestLsLl:
         assert err == ''
         assert out.strip() == utils.get_repos()['gita']['path']
 
-    def testLs(self, monkeypatch, capfd):
+    def test_ls(self, monkeypatch, capfd):
         monkeypatch.setattr(utils, 'get_repos',
                 lambda: {'repo1': {'path': '/a/'}, 'repo2': {'path': '/b/'}})
         monkeypatch.setattr(utils, 'describe', lambda x: x)
@@ -157,7 +157,7 @@ class TestLsLl:
     @patch('gita.info.get_commit_msg', return_value="msg")
     @patch('gita.info.get_commit_time', return_value="")
     @patch('gita.common.get_config_fname')
-    def testWithPathFiles(self, mock_path_fname, _0, _1, _2, _3, _4, path_fname,
+    def test_with_path_files(self, mock_path_fname, _0, _1, _2, _3, _4, path_fname,
                           expected, capfd):
         def side_effect(input, _=None):
             if input == 'repos.csv':
@@ -301,21 +301,21 @@ class TestContext:
 
     @patch('gita.utils.get_context', return_value=Path('gname.context'))
     @patch('gita.utils.get_groups', return_value={'gname': ['a', 'b']})
-    def testDisplayContext(self, _, __, capfd):
+    def test_display_context(self, _, __, capfd):
         __main__.main(['context'])
         out, err = capfd.readouterr()
         assert err == ''
         assert 'gname: a b\n' == out
 
     @patch('gita.utils.get_context')
-    def testReset(self, mock_ctx):
+    def test_reset(self, mock_ctx):
         __main__.main(['context', 'none'])
         mock_ctx.return_value.unlink.assert_called()
 
     @patch('gita.utils.get_context', return_value=None)
     @patch('gita.common.get_config_dir', return_value=TEST_DIR)
     @patch('gita.utils.get_groups', return_value={'lala': ['b'], 'kaka': []})
-    def testSetFirstTime(self, *_):
+    def test_set_first_time(self, *_):
         ctx = TEST_DIR / 'lala.context'
         assert not ctx.is_file()
         __main__.main(['context', 'lala'])
@@ -325,7 +325,7 @@ class TestContext:
     @patch('gita.common.get_config_dir', return_value=TEST_DIR)
     @patch('gita.utils.get_groups', return_value={'lala': ['b'], 'kaka': []})
     @patch('gita.utils.get_context')
-    def testSetSecondTime(self, mock_ctx, *_):
+    def test_set_second_time(self, mock_ctx, *_):
         __main__.main(['context', 'kaka'])
         mock_ctx.return_value.rename.assert_called()
 
@@ -333,7 +333,7 @@ class TestContext:
 class TestGroupCmd:
 
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
-    def testLs(self, _, capfd):
+    def test_ls(self, _, capfd):
         args = argparse.Namespace()
         args.to_group = None
         args.group_cmd = 'ls'
@@ -344,7 +344,7 @@ class TestGroupCmd:
         assert 'xx yy\n' == out
 
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
-    def testLl(self, _, capfd):
+    def test_ll(self, _, capfd):
         args = argparse.Namespace()
         args.to_group = None
         args.group_cmd = None
@@ -356,7 +356,7 @@ class TestGroupCmd:
 
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
     @patch('gita.utils.write_to_groups_file')
-    def testRename(self, mock_write, _):
+    def test_rename(self, mock_write, _):
         args = argparse.Namespace()
         args.gname = 'xx'
         args.new_name = 'zz'
@@ -367,7 +367,7 @@ class TestGroupCmd:
         mock_write.assert_called_once_with(expected, 'w')
 
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
-    def testRenameError(self, *_):
+    def test_rename_error(self, *_):
         args = argparse.Namespace()
         args.gname = 'xx'
         args.new_name = 'yy'
@@ -383,7 +383,7 @@ class TestGroupCmd:
     @patch('gita.utils.get_repos', return_value={'a': '', 'b': '', 'c': '', 'd': ''})
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
     @patch('gita.utils.write_to_groups_file')
-    def testRm(self, mock_write, _, __, input, expected):
+    def test_rm(self, mock_write, _, __, input, expected):
         utils.get_groups.cache_clear()
         args = ['group', 'rm'] + shlex.split(input)
         __main__.main(args)
@@ -392,7 +392,7 @@ class TestGroupCmd:
     @patch('gita.utils.get_repos', return_value={'a': '', 'b': '', 'c': '', 'd': ''})
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
     @patch('gita.utils.write_to_groups_file')
-    def testAdd(self, mock_write, *_):
+    def test_add(self, mock_write, *_):
         args = argparse.Namespace()
         args.to_group =  ['a', 'c']
         args.group_cmd = 'add'
@@ -404,7 +404,7 @@ class TestGroupCmd:
     @patch('gita.utils.get_repos', return_value={'a': '', 'b': '', 'c': '', 'd': ''})
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
     @patch('gita.utils.write_to_groups_file')
-    def testAddToExisting(self, mock_write, *_):
+    def test_add_to_existing(self, mock_write, *_):
         args = argparse.Namespace()
         args.to_group =  ['a', 'c']
         args.group_cmd = 'add'
@@ -417,7 +417,7 @@ class TestGroupCmd:
     @patch('gita.utils.get_repos', return_value={'a': '', 'b': '', 'c': '', 'd': ''})
     @patch('gita.common.get_config_fname', return_value=GROUP_FNAME)
     @patch('gita.utils.write_to_groups_file')
-    def testRmRepo(self, mock_write, *_):
+    def test_rm_repo(self, mock_write, *_):
         args = argparse.Namespace()
         args.from_group = ['a', 'c']
         args.group_cmd = 'rmrepo'
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 78c496b..39430b0 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -10,6 +10,27 @@ from conftest import (
 )
 
 
+@pytest.mark.parametrize('repo_path, paths, expected', [
+    ('/a/b/c/repo', ['/a/b'], ('b', 'c')),
+    ])
+def test_generate_dir_hash(repo_path, paths, expected):
+    got = utils._generate_dir_hash(repo_path, paths)
+    assert got == expected
+
+
+@pytest.mark.parametrize('repos, paths, expected', [
+    ({'r1': {'path': '/a/b//repo1'}, 'r2': {'path': '/a/b/repo2'}},
+        ['/a/b'], {'b': ['r1', 'r2']}),
+    ({'r1': {'path': '/a/b//repo1'}, 'r2': {'path': '/a/b/c/repo2'}},
+        ['/a/b'], {'b': ['r1', 'r2'], 'b-c': ['r2']}),
+    ({'r1': {'path': '/a/b/c/repo1'}, 'r2': {'path': '/a/b/c/repo2'}},
+        ['/a/b'], {'b-c': ['r1', 'r2'], 'b': ['r1', 'r2']}),
+    ])
+def test_auto_group(repos, paths, expected):
+    got = utils.auto_group(repos, paths)
+    assert got == expected
+
+
 @pytest.mark.parametrize('test_input, diff_return, expected', [
     ([{'abc': {'path': '/root/repo/', 'type': '', 'flags': []}}, False],
         True, 'abc \x1b[31mrepo *+_  \x1b[0m msg xx'),
@@ -83,7 +104,7 @@ def test_custom_push_cmd(*_):
 @pytest.mark.parametrize(
     'path_input, expected',
     [
-        (['/home/some/repo/'], '/home/some/repo,some/repo,,\r\n'),  # add one new
+        (['/home/some/repo'], '/home/some/repo,some/repo,,\r\n'),  # add one new
         (['/home/some/repo1', '/repo2'],
             {'/repo2,repo2,,\r\n',  # add two new
             '/home/some/repo1,repo1,,\r\n'}),  # add two new