Repository: mgedmin/ghcloneall

Issue:
clone private repositories
In #6 you wrote...

> No, I'm using anonymous access.  This means, of course, that ghcloneall is only suitable for cloning open source repositories that are publicly available.  

1. oh no!
2. how does this go together with the optional argument "--include-private     include private repositories (default)"? (see readme)

Discussion:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index a128502..23c9f9f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -5,8 +5,8 @@ Changelog
 1.9.3 (unreleased)
 ------------------
 
+- Allow authentication with github token
 - Add support for Python 3.9.
-
 - Drop support for Python 3.5.
 
 
diff --git a/README.rst b/README.rst
index d7f7175..149408f 100644
--- a/README.rst
+++ b/README.rst
@@ -80,11 +80,13 @@ Other command-line options::
     $ ghcloneall --help
     usage: ghcloneall [-h] [--version] [-c CONCURRENCY] [-n] [-q] [-v]
                       [--start-from REPO] [--organization ORGANIZATION]
-                      [--user USER] [--gists] [--repositories] [--pattern PATTERN]
-                      [--include-forks] [--exclude-forks] [--include-archived]
-                      [--exclude-archived] [--include-private] [--exclude-private]
-                      [--include-disabled] [--exclude-disabled] [--init]
-                      [--http-cache DBNAME] [--no-http-cache]
+                      [--user USER] [--github-token GITHUB_TOKEN] [--gists]
+                      [--repositories] [--pattern PATTERN] [--include-forks]
+                      [--exclude-forks] [--include-archived] [--exclude-archived]
+                      [--include-private] [--exclude-private] [--include-disabled]
+                      [--exclude-disabled] [--init] [--http-cache DBNAME]
+                      [--no-http-cache]
+
 
     Clone/update all user/org repositories from GitHub.
 
@@ -101,6 +103,8 @@ Other command-line options::
       --organization ORGANIZATION
                             specify the GitHub organization
       --user USER           specify the GitHub user
+      --github-token GITHUB_TOKEN
+                            specify the GitHub token
       --gists               clone user's gists
       --repositories        clone user's or organisation's repositories (default)
       --pattern PATTERN     specify repository name glob pattern to filter
@@ -130,6 +134,8 @@ should look like this::
     # github_org = ZopeFoundation
     github_user = mgedmin
     pattern = *.vim
+    # Provide github token for authentication
+    # github_token = <my-github-token>
     # You can also uncomment and change these if you wish
     # gists = False
     # include_forks = False
diff --git a/ghcloneall.py b/ghcloneall.py
index 43f7cd0..e5bd1e2 100755
--- a/ghcloneall.py
+++ b/ghcloneall.py
@@ -83,7 +83,7 @@ def get_github_list(url, batch_size=100, progress_callback=None, session=None):
     while 'next' in links:
         if progress_callback:
             progress_callback(len(res))
-        more, links = get_json_and_links(links['next']['url'])
+        more, links = get_json_and_links(links['next']['url'], session)
         res += more
     return res
 
@@ -383,7 +383,8 @@ class Repo(object):
 
 class RepoWrangler(object):
 
-    def __init__(self, dry_run=False, verbose=0, progress=None, quiet=False):
+    def __init__(self, dry_run=False, verbose=0, progress=None, quiet=False,
+                 token=None):
         self.n_repos = 0
         self.n_updated = 0
         self.n_new = 0
@@ -394,13 +395,18 @@ class RepoWrangler(object):
         self.progress = progress if progress else Progress()
         self.lock = threading.Lock()
 
+        self.session = requests.Session()
+        if token:
+            self.session.auth = ('', token)
+
     def get_github_list(self, list_url, message):
         self.progress.status(message)
 
         def progress_callback(n):
             self.progress.status("{} ({})".format(message, n))
 
-        return get_github_list(list_url, progress_callback=progress_callback)
+        return get_github_list(list_url, progress_callback=progress_callback,
+                               session=self.session)
 
     def list_gists(self, user, pattern=None):
         list_url = 'https://api.github.com/users/{}/gists'.format(user)
@@ -777,6 +783,9 @@ def _main():
     parser.add_argument(
         '--user',
         help='specify the GitHub user')
+    parser.add_argument(
+        '--github-token',
+        help='specify the GitHub token')
     parser.add_argument(
         '--gists', action='store_true', default=None,
         help="clone user's gists")
@@ -834,6 +843,9 @@ def _main():
             args.user = config.get(CONFIG_SECTION, 'github_user')
         if config.has_option(CONFIG_SECTION, 'github_org'):
             args.organization = config.get(CONFIG_SECTION, 'github_org')
+    if not args.github_token:
+        if config.has_option(CONFIG_SECTION, 'github_token'):
+            args.github_token = config.get(CONFIG_SECTION, 'github_token')
     if not args.pattern:
         if config.has_option(CONFIG_SECTION, 'pattern'):
             args.pattern = config.get(CONFIG_SECTION, 'pattern')
@@ -874,6 +886,8 @@ def _main():
             config.set(CONFIG_SECTION, 'github_user', args.user)
         if args.organization:
             config.set(CONFIG_SECTION, 'github_org', args.organization)
+        if args.github_token:
+            config.set(CONFIG_SECTION, 'github_token', args.github_token)
         if args.pattern:
             config.set(CONFIG_SECTION, 'pattern', args.pattern)
         if args.gists is not None:
@@ -913,7 +927,8 @@ def _main():
 
     with Progress() as progress:
         wrangler = RepoWrangler(dry_run=args.dry_run, verbose=args.verbose,
-                                progress=progress, quiet=args.quiet)
+                                progress=progress, quiet=args.quiet,
+                                token=args.github_token)
         if args.gists:
             repos = wrangler.list_gists(
                 user=args.user,


Test patch:
diff --git a/tests.py b/tests.py
index 4542bdf..c5f04c6 100644
--- a/tests.py
+++ b/tests.py
@@ -695,6 +695,12 @@ def Gist(name, **kwargs):
     return ghcloneall.Repo.from_gist(gist(name, **kwargs))
 
 
+def test_RepoWrangler_auth():
+    token = 'UNITTEST'
+    wrangler = ghcloneall.RepoWrangler(token=token)
+    assert wrangler.session.auth == ('', token)
+
+
 def test_RepoWrangler_list_gists(mock_requests_get):
     mock_requests_get.update(mock_multi_page_api_responses(
         url='https://api.github.com/users/test_user/gists',
@@ -1417,6 +1423,23 @@ def test_main_init_org(monkeypatch, capsys, config_writes_allowed):
     )
 
 
+def test_main_init_org_token(monkeypatch, capsys, config_writes_allowed):
+    monkeypatch.setattr(sys, 'argv', [
+        'ghcloneall', '--init', '--org', 'gtimelog', '--github-token',
+        'UNITTEST'
+    ])
+    ghcloneall.main()
+    assert capsys.readouterr().out == (
+        'Wrote .ghcloneallrc\n'
+    )
+    assert config_writes_allowed.read_text() == (
+        '[ghcloneall]\n'
+        'github_org = gtimelog\n'
+        'github_token = UNITTEST\n'
+        '\n'
+    )
+
+
 def test_main_init_filter_flags(monkeypatch, capsys, config_writes_allowed):
     monkeypatch.setattr(sys, 'argv', [
         'ghcloneall', '--init', '--org', 'gtimelog',
@@ -1443,6 +1466,7 @@ def test_main_reads_config_file(monkeypatch, capsys, config_writes_allowed):
         u'[ghcloneall]\n'
         u'github_user = mgedmin\n'
         u'github_org = gtimelog\n'
+        u'github_token = UNITTEST\n'
         u'gists = False\n'
         u'pattern = *.vim\n'
         u'include_forks = True\n'