Repository: asottile/all-repos

Issue:
remote branches
After running the `all-repos-clone` command, I am not able to checkout any remote branches for the cloned repositories.

I have tried running `git fetch --all` though that does not do anything.

_(discussed briefly on stream)_

Discussion:

Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f0780b3..c70c996 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.1.0
+    rev: v2.2.3
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -15,25 +15,25 @@ repos:
     hooks:
     -   id: flake8
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.4.3
+    rev: v1.4.4
     hooks:
     -   id: autopep8
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.4.0
+    rev: v1.5.0
     hooks:
     -   id: reorder-python-imports
         args: [--py3-plus]
 -   repo: https://github.com/asottile/add-trailing-comma
-    rev: v1.0.0
+    rev: v1.3.0
     hooks:
     -   id: add-trailing-comma
         args: [--py36-plus]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v1.12.0
+    rev: v1.19.0
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.670-1
+    rev: v0.711
     hooks:
     -   id: mypy
diff --git a/README.md b/README.md
index c3c0ecf..24dc0aa 100644
--- a/README.md
+++ b/README.md
@@ -175,6 +175,8 @@ A configuration file looks roughly like this:
   repository names which match this regex will be included.
 - `exclude` (default `"^$"`): python regex for excluding repositories.
   Repository names which match this regex will be excluded.
+- `all_branches` (default `false`): whether to clone all of the branches or
+  just the default upstream branch.
 
 ## Source modules
 
diff --git a/all_repos/clone.py b/all_repos/clone.py
index 395ae73..4a12971 100644
--- a/all_repos/clone.py
+++ b/all_repos/clone.py
@@ -1,4 +1,5 @@
 import argparse
+import functools
 import json
 import os.path
 import shutil
@@ -70,14 +71,20 @@ def _default_branch(remote: str) -> str:
     return line[len(start):-1 * len(end)]
 
 
-def _fetch_reset(path: str) -> None:
+def _fetch_reset(path: str, *, all_branches: bool) -> None:
     def _git(*cmd: str) -> None:
         subprocess.check_call(('git', '-C', path, *cmd))
 
     try:
         branch = _default_branch(git.remote(path))
-        _git('remote', 'set-branches', 'origin', branch)
-        _git('fetch', 'origin', branch)
+        if all_branches:
+            _git(
+                'config', 'remote.origin.fetch',
+                '+refs/heads/*:refs/remotes/origin/*',
+            )
+        else:
+            _git('remote', 'set-branches', 'origin', branch)
+        _git('fetch', 'origin')
         _git('checkout', branch)
         _git('reset', '--hard', f'origin/{branch}')
     except subprocess.CalledProcessError:
@@ -121,9 +128,10 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     for path, remote in filtered_repos - current_repos:
         _init(config.output_dir, path, remote)
 
+    fn = functools.partial(_fetch_reset, all_branches=config.all_branches)
     todo = [os.path.join(config.output_dir, p) for p in repos_filtered]
     with mapper.thread_mapper(args.jobs) as do_map:
-        mapper.exhaust(do_map(_fetch_reset, todo))
+        mapper.exhaust(do_map(fn, todo))
 
     # write these last
     os.makedirs(config.output_dir, exist_ok=True)
diff --git a/all_repos/config.py b/all_repos/config.py
index c13a284..8037ad5 100644
--- a/all_repos/config.py
+++ b/all_repos/config.py
@@ -16,6 +16,7 @@ class Config(NamedTuple):
     source_settings: Any
     push: Callable[[Any, str], None]
     push_settings: Any
+    all_branches: bool
 
     def _path(self, *paths: str) -> str:
         return os.path.abspath(os.path.join(self.output_dir, *paths))
@@ -55,8 +56,10 @@ def load_config(filename: str) -> Config:
     push_settings = push_module.Settings(**contents['push_settings'])
     include = re.compile(contents.get('include', ''))
     exclude = re.compile(contents.get('exclude', '^$'))
+    all_branches = contents.get('all_branches', False)
     return Config(
         output_dir=output_dir, include=include, exclude=exclude,
         list_repos=source_module.list_repos, source_settings=source_settings,
         push=push_module.push, push_settings=push_settings,
+        all_branches=all_branches,
     )


Test patch:
diff --git a/tests/clone_test.py b/tests/clone_test.py
index 2712216..c091c18 100644
--- a/tests/clone_test.py
+++ b/tests/clone_test.py
@@ -91,3 +91,34 @@ def test_no_crash_repo_without_branch(file_config):
 
     # should not crash
     assert not main(('--config-filename', str(file_config.cfg)))
+
+
+def test_clones_all_branches_true(file_config):
+    subprocess.check_call((
+        'git', '-C', str(file_config.dir1), 'checkout', 'master', '-b', 'b2',
+    ))
+    subprocess.check_call((
+        'git', '-C', str(file_config.dir1), 'checkout', 'master',
+    ))
+
+    assert not main(('--config-file', str(file_config.cfg)))
+
+    # initially we should not see multiple branches
+    branch_out = subprocess.check_output((
+        'git', '-C', str(file_config.output_dir.join('repo1')),
+        'branch', '--remote',
+    )).decode()
+    assert branch_out == '  origin/master\n'
+
+    # set that we want to clone all branches
+    cfg_contents = json.loads(file_config.cfg.read())
+    cfg_contents['all_branches'] = True
+    file_config.cfg.write(json.dumps(cfg_contents))
+
+    assert not main(('--config-file', str(file_config.cfg)))
+
+    branch_out = subprocess.check_output((
+        'git', '-C', str(file_config.output_dir.join('repo1')),
+        'branch', '--remote',
+    )).decode()
+    assert branch_out == '  origin/b2\n  origin/master\n'