Repository: Justintime50/github-archive

Issue:
Add a "Forks" flag to clone/pull forks
# Summary

By default, forks are included in cloning/pulling. This setting cannot be altered. Let's add a flag that allows the user to choose whether to include forks or not.

## Acceptance Criteria

- [ ] Add a flag that allows users to include forks in what is cloned/pulled. This will be `off` by default.

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bff191d..cf5782d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@
 * Added the `--threads` flag which can specify the number of concurrent threads to run at once (closes #22)
 * Adds a new `--view` flag which allows you to "dry run" the application, seeing the entire list of repos and gists based on the input provided (closes #25)
 * Adds a new `--stars` flag which you can pass a comma separated list of users to and GitHub Archive will retrieve all of their starred repos which you can then view, clone, or pull (closes #26)
+* Adds a new `--forks` flag which will include forks for whatever lists and operations you provide (closes #17))
 
 ### Fixes
 
diff --git a/README_v4.md b/README_v4.md
index d8e6d5b..29b3bda 100644
--- a/README_v4.md
+++ b/README_v4.md
@@ -72,6 +72,7 @@ Options:
     -v, --view            Pass this flag to view git assets (dry run).
     -c, --clone           Pass this flag to clone git assets.
     -p, --pull            Pass this flag to pull git assets.
+    -f, --forks           Pass this flag to include forked git assets.
     -u USERS, --users USERS
                             Pass a comma separated list of users to get repos for.
     -o ORGS, --orgs ORGS  Pass a comma separated list of orgs to get repos for.
diff --git a/github_archive/archive.py b/github_archive/archive.py
index 828bac9..ae3bbae 100644
--- a/github_archive/archive.py
+++ b/github_archive/archive.py
@@ -28,6 +28,7 @@ class GithubArchive:
         view=False,
         clone=False,
         pull=False,
+        forks=False,
         users=None,
         orgs=None,
         gists=None,
@@ -41,6 +42,7 @@ class GithubArchive:
         self.view = view
         self.clone = clone
         self.pull = pull
+        self.forks = forks
         self.users = users.lower().split(',') if users else ''
         self.orgs = orgs.lower().split(',') if orgs else ''
         self.gists = gists.lower().split(',') if gists else ''
@@ -64,7 +66,7 @@ class GithubArchive:
         # Personal (includes personal authenticated items)
         if self.token and self.authenticated_user_in_users and self.users:
             LOGGER.info('# Making API call to GitHub for personal repos...\n')
-            personal_repos = self.get_personal_repos()
+            personal_repos = self.get_all_git_assets(PERSONAL_CONTEXT)
 
             if self.view:
                 LOGGER.info('# Viewing user repos...\n')
@@ -80,7 +82,7 @@ class GithubArchive:
             # repos for the `users` logic.
             self.users.remove(self.authenticated_username)
 
-        # Users (can include personal non-authenticated items)
+        # Users (can include personal non-authenticated items, excludes personal authenticated calls)
         if self.users and len(self.users) > 0:
             LOGGER.info('# Making API calls to GitHub for user repos...\n')
             user_repos = self.get_all_git_assets(USER_CONTEXT)
@@ -167,44 +169,41 @@ class GithubArchive:
     def authenticated_user_in_users(self):
         return self.authenticated_user.login.lower() in self.users
 
-    def get_personal_repos(self):
-        """Retrieve all repos of the authenticated user."""
-        repos = self.authenticated_user.get_repos()
-        LOGGER.debug('Personal repos retrieved!')
-
-        return repos
-
     def get_all_git_assets(self, context):
         """Retrieve a list of lists via API of git assets (repos, gists) of the
         specified owner(s) (users, orgs). Return a sorted, flat, sorted list of git assets.
         """
-        get_user_repos = lambda item: self.github_instance.get_user(item).get_repos()  # noqa
-        get_org_repos = lambda item: self.github_instance.get_organization(item).get_repos()  # noqa
-        get_user_gists = lambda item: self.github_instance.get_user(item).get_gists()  # noqa
-        get_starred_repos = lambda item: self.github_instance.get_user(item).get_starred()  # noqa
+        get_org_repos = lambda owner: self.github_instance.get_organization(owner).get_repos()  # noqa
+        get_personal_repos = lambda owner: self.authenticated_user.get_repos(affiliation='owner')  # noqa
+        get_starred_repos = lambda owner: self.github_instance.get_user(owner).get_starred()  # noqa
+        get_user_gists = lambda owner: self.github_instance.get_user(owner).get_gists()  # noqa
+        get_user_repos = lambda owner: self.github_instance.get_user(owner).get_repos()  # noqa
 
         context_manager = {
-            USER_CONTEXT: [self.users, get_user_repos, 'repos'],
-            ORG_CONTEXT: [self.orgs, get_org_repos, 'repos'],
             GIST_CONTEXT: [self.gists, get_user_gists, 'gists'],
+            ORG_CONTEXT: [self.orgs, get_org_repos, 'repos'],
+            PERSONAL_CONTEXT: [self.users, get_personal_repos, 'repos'],
             STAR_CONTEXT: [self.stars, get_starred_repos, 'starred repos'],
+            USER_CONTEXT: [self.users, get_user_repos, 'repos'],
         }
 
         all_git_assets = []
-        cli_list = context_manager[context][0]
+        owner_list = context_manager[context][0]
         git_asset_string = context_manager[context][2]
 
-        for item in cli_list:
-            formatted_item_name = item.strip()
-            git_assets = sorted(context_manager[context][1](item), key=lambda item: item.name)
-            all_git_assets.append(git_assets)
-            LOGGER.debug(f'{formatted_item_name} {git_asset_string} retrieved!')
+        for owner in owner_list:
+            formatted_owner_name = owner.strip()
+            git_assets = context_manager[context][1](owner)
+            LOGGER.debug(f'{formatted_owner_name} {git_asset_string} retrieved!')
 
-        flattened_git_asset_list = [
-            git_asset for sublist_git_assets in all_git_assets for git_asset in sublist_git_assets
-        ]
+            for item in git_assets:
+                if self.forks or (self.forks is False and item.fork is False):
+                    all_git_assets.append(item)
+                else:
+                    # Do not include this forked asset
+                    pass
 
-        final_sorted_list = sorted(flattened_git_asset_list, key=lambda item: item.owner.login)
+        final_sorted_list = sorted(all_git_assets, key=lambda item: item.owner.login)
 
         return final_sorted_list
 
@@ -215,29 +214,18 @@ class GithubArchive:
 
         for repo in repos:
             repo_owner_username = repo.owner.login.lower()
-            # We check the owner name here to ensure that we only iterate
-            # through the user's personal repos which will exclude orgs.
-            #
-            # This is important because PyGithub will include a user's org repos
-            # in the list of repos for an authenticated user out of the box.
-            #
-            # Instead, the user can pass the "--clone_orgs" or "--pull_orgs"
-            # flags to allow for more granular control over which repos they get.
-            if self.token and repo_owner_username != self.authenticated_username and context == PERSONAL_CONTEXT:
-                continue
-            else:
-                repo_path = os.path.join(self.location, 'repos', repo_owner_username, repo.name)
-                repo_thread = Thread(
-                    target=self.archive_repo,
-                    args=(
-                        thread_limiter,
-                        repo,
-                        repo_path,
-                        operation,
-                    ),
-                )
-                thread_list.append(repo_thread)
-                repo_thread.start()
+            repo_path = os.path.join(self.location, 'repos', repo_owner_username, repo.name)
+            repo_thread = Thread(
+                target=self.archive_repo,
+                args=(
+                    thread_limiter,
+                    repo,
+                    repo_path,
+                    operation,
+                ),
+            )
+            thread_list.append(repo_thread)
+            repo_thread.start()
         for thread in thread_list:
             thread.join()
 
diff --git a/github_archive/cli.py b/github_archive/cli.py
index 334cae4..503a10b 100644
--- a/github_archive/cli.py
+++ b/github_archive/cli.py
@@ -35,6 +35,14 @@ class GithubArchiveCli:
             default=False,
             help='Pass this flag to pull git assets.',
         )
+        parser.add_argument(
+            '-f',
+            '--forks',
+            action='store_true',
+            required=False,
+            default=False,
+            help='Pass this flag to include forked git assets.',
+        )
         parser.add_argument(
             '-u',
             '--users',
@@ -109,6 +117,7 @@ class GithubArchiveCli:
             view=self.view,
             clone=self.clone,
             pull=self.pull,
+            forks=self.forks,
             users=self.users,
             orgs=self.orgs,
             gists=self.gists,


Test patch:
diff --git a/test/unit/test_archive.py b/test/unit/test_archive.py
index 7d4cabd..f157af0 100644
--- a/test/unit/test_archive.py
+++ b/test/unit/test_archive.py
@@ -28,8 +28,8 @@ def mock_thread_limiter():
 @mock.patch('github_archive.archive.Github.get_user')
 @mock.patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True)
 @mock.patch('github_archive.archive.GithubArchive.view_repos')
-@mock.patch('github_archive.archive.GithubArchive.get_personal_repos')
-def test_run_token_view(mock_get_personal_repos, mock_view_repos, mock_authed_user_in_users, mock_get_user):
+@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets')
+def test_run_token_view(mock_get_all_git_assets, mock_view_repos, mock_authed_user_in_users, mock_get_user):
     github_archive = GithubArchive(
         token='123',
         users='justintime50',
@@ -39,7 +39,7 @@ def test_run_token_view(mock_get_personal_repos, mock_view_repos, mock_authed_us
     github_archive.authenticated_username = 'justintime50'
     github_archive.run()
 
-    mock_get_personal_repos.assert_called_once()
+    mock_get_all_git_assets.assert_called_once()
     mock_view_repos.assert_called_once()
     assert github_archive.users == []  # Assert the authed user gets removed from list
 
@@ -47,9 +47,9 @@ def test_run_token_view(mock_get_personal_repos, mock_view_repos, mock_authed_us
 @mock.patch('github_archive.archive.Github.get_user')
 @mock.patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True)
 @mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive')
-@mock.patch('github_archive.archive.GithubArchive.get_personal_repos')
+@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets')
 def test_run_token_clone(
-    mock_get_personal_repos, mock_iterate_repos_to_archive, mock_authed_user_in_users, mock_get_user
+    mock_get_all_git_assets, mock_iterate_repos_to_archive, mock_authed_user_in_users, mock_get_user
 ):
     github_archive = GithubArchive(
         token='123',
@@ -60,7 +60,7 @@ def test_run_token_clone(
     github_archive.authenticated_username = 'justintime50'
     github_archive.run()
 
-    mock_get_personal_repos.assert_called_once()
+    mock_get_all_git_assets.assert_called_once()
     mock_iterate_repos_to_archive.assert_called_once()
     assert github_archive.users == []  # Assert the authed user gets removed from list
 
@@ -68,9 +68,9 @@ def test_run_token_clone(
 @mock.patch('github_archive.archive.Github.get_user')
 @mock.patch('github_archive.archive.GithubArchive.authenticated_user_in_users', return_value=True)
 @mock.patch('github_archive.archive.GithubArchive.iterate_repos_to_archive')
-@mock.patch('github_archive.archive.GithubArchive.get_personal_repos')
+@mock.patch('github_archive.archive.GithubArchive.get_all_git_assets')
 def test_run_token_pull(
-    mock_get_personal_repos, mock_iterate_repos_to_archive, mock_authed_user_in_users, mock_get_user
+    mock_get_all_git_assets, mock_iterate_repos_to_archive, mock_authed_user_in_users, mock_get_user
 ):
     github_archive = GithubArchive(
         token='123',
@@ -81,7 +81,7 @@ def test_run_token_pull(
     github_archive.authenticated_username = 'justintime50'
     github_archive.run()
 
-    mock_get_personal_repos.assert_called_once()
+    mock_get_all_git_assets.assert_called_once()
     mock_iterate_repos_to_archive.assert_called_once()
     assert github_archive.users == []  # Assert the authed user gets removed from list
 
@@ -275,11 +275,10 @@ def test_authenticated_user_in_users(mock_get_user):
 
 
 @mock.patch('github_archive.archive.Github.get_user')
-def test_get_personal_repos(mock_get_user):
+def test_get_all_git_assets(mock_get_user):
     GithubArchive(
-        token='123',
         users='justintime50',
-    ).get_personal_repos()
+    ).get_all_git_assets(USER_CONTEXT)
 
     mock_get_user.assert_called_once()
 
@@ -334,7 +333,7 @@ def test_iterate_repos_not_matching_authed_username(mock_archive_repo, mock_gith
         users='mock_username',
     ).iterate_repos_to_archive(repos, PERSONAL_CONTEXT, CLONE_OPERATION)
 
-    mock_archive_repo.assert_called()
+    mock_archive_repo.assert_called_once()
 
 
 @mock.patch('github_archive.archive.Github')
@@ -343,10 +342,10 @@ def test_iterate_repos_matching_authed_username(mock_archive_repo, mock_github_i
     repos = [mock_git_asset]
     GithubArchive(
         token='123',
-        users='mock_username',
+        users='mock_username',  # matches the username of the git asset
     ).iterate_repos_to_archive(repos, PERSONAL_CONTEXT, CLONE_OPERATION)
 
-    mock_archive_repo.assert_not_called()
+    mock_archive_repo.assert_called_once()
 
 
 @mock.patch('github_archive.archive.Github')