Repository: Justintime50/github-archive

Issue:
Add `--exclude` and `--include` flags for repo filtering
Hi, 
It would be nice to be able to change the download path with an argument, and also have an argument for a limit of repository.

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 80dcc7c..b922fe4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## NEXT RELEASE
+
+* Adds an `--include` and `--exclude` CLI flag that accepts a comma-separated list of repo names to either include or exclude. If neither are passed, no filtering will occur (closes #43)
+
 ## v4.3.0 (2021-12-16)
 
 * Adds the ability to specify a custom base_url for GitHub (useful for enterprise GitHub users with a custom hostname, closes #41)
diff --git a/README.md b/README.md
index 8c52d73..506af5d 100644
--- a/README.md
+++ b/README.md
@@ -48,6 +48,10 @@ Options:
     -c, --clone           Pass this flag to clone git assets.
     -p, --pull            Pass this flag to pull git assets.
     -f, --forks           Pass this flag to include forked git assets.
+    -i INCLUDE, --include INCLUDE
+                            Pass a comma separated list of repos to include in the Archive.
+    -e EXCLUDE, --exclude EXCLUDE
+                            Pass a comma separated list of repos to exclude from the Archive.
     -l LOCATION, --location LOCATION
                             The location where you want your GitHub Archive to be stored.
     -ht, --https          Use HTTPS URLs instead of SSH.
diff --git a/github_archive/archive.py b/github_archive/archive.py
index df4faf3..8af491c 100644
--- a/github_archive/archive.py
+++ b/github_archive/archive.py
@@ -40,6 +40,8 @@ class GithubArchive:
         pull=False,
         forks=False,
         location=DEFAULT_LOCATION,
+        include=None,
+        exclude=None,
         use_https=False,
         timeout=DEFAULT_TIMEOUT,
         threads=DEFAULT_NUM_THREADS,
@@ -56,6 +58,8 @@ class GithubArchive:
         self.pull = pull
         self.forks = forks
         self.location = location
+        self.include = include.lower().split(',') if include else ''
+        self.exclude = exclude.lower().split(',') if exclude else ''
         self.use_https = use_https
         self.timeout = timeout
         self.threads = threads
@@ -195,6 +199,10 @@ class GithubArchive:
             message = 'At least one git operation and one list must be provided to run github-archive.'
             logger.critical(message)
             raise ValueError(message)
+        elif self.include and self.exclude:
+            message = 'The include and exclude flags are mutually exclusive. Only one can be used on each run.'
+            logger.critical(message)
+            raise ValueError(message)
 
     def authenticated_user_in_users(self) -> bool:
         return self.authenticated_user.login.lower() in self.users
@@ -243,24 +251,35 @@ class GithubArchive:
         return final_sorted_list
 
     def iterate_repos_to_archive(self, repos: List[Repository.Repository], operation: str):
-        """Iterate over each repository and start a thread if it can be archived."""
+        """Iterate over each repository and start a thread if it can be archived.
+
+        We ignore repos not in the include or in the exclude list if either are present.
+        """
+        logger = woodchips.get(LOGGER_NAME)
         thread_limiter = BoundedSemaphore(self.threads)
         thread_list = []
 
         for repo in repos:
-            repo_owner_username = repo.owner.login.lower()
-            repo_path = os.path.join(self.location, 'repos', repo_owner_username, repo.name)
-            repo_thread = Thread(
-                target=self.archive_repo,
-                args=(
-                    thread_limiter,
-                    repo,
-                    repo_path,
-                    operation,
-                ),
-            )
-            thread_list.append(repo_thread)
-            repo_thread.start()
+            if (
+                (not self.include and not self.exclude)
+                or (self.include and repo.name in self.include)
+                or (self.exclude and repo.name not in self.exclude)
+            ):
+                repo_owner_username = repo.owner.login.lower()
+                repo_path = os.path.join(self.location, 'repos', repo_owner_username, repo.name)
+                repo_thread = Thread(
+                    target=self.archive_repo,
+                    args=(
+                        thread_limiter,
+                        repo,
+                        repo_path,
+                        operation,
+                    ),
+                )
+                thread_list.append(repo_thread)
+                repo_thread.start()
+            else:
+                logger.debug(f'{repo.name} skipped due to include/exclude filtering')
 
         # Wait for the number of threads in thread_limiter to finish before moving on
         for thread in thread_list:
diff --git a/github_archive/cli.py b/github_archive/cli.py
index dad77fd..a33c6d2 100644
--- a/github_archive/cli.py
+++ b/github_archive/cli.py
@@ -91,6 +91,22 @@ class GithubArchiveCli:
             default=False,
             help='Pass this flag to include forked git assets.',
         )
+        parser.add_argument(
+            '-i',
+            '--include',
+            type=str,
+            required=False,
+            default=None,
+            help='Pass a comma separated list of repos to include in the Archive.',
+        )
+        parser.add_argument(
+            '-e',
+            '--exclude',
+            type=str,
+            required=False,
+            default=None,
+            help='Pass a comma separated list of repos to exclude from the Archive.',
+        )
         parser.add_argument(
             '-l',
             '--location',
@@ -144,6 +160,8 @@ class GithubArchiveCli:
             pull=self.pull,
             forks=self.forks,
             location=self.location,
+            include=self.include,
+            exclude=self.exclude,
             use_https=self.https,
             timeout=self.timeout,
             threads=self.threads,


Test patch:
diff --git a/test/unit/test_archive.py b/test/unit/test_archive.py
index 51024ce..edd4cea 100644
--- a/test/unit/test_archive.py
+++ b/test/unit/test_archive.py
@@ -291,6 +291,24 @@ def test_initialize_project_missing_all_cli_args(mock_logger):
     assert message == str(error.value)
 
 
+@patch('logging.Logger.critical')
+def test_initialize_project_include_exclude_together(mock_logger):
+    # TODO: Is it possible to test all variations easily in one test?
+    # Parametrize doesn't work great because we can't easily swap the param name being used
+    message = 'The include and exclude flags are mutually exclusive. Only one can be used on each run.'
+    with pytest.raises(ValueError) as error:
+        github_archive = GithubArchive(
+            users='justintime50',
+            clone=True,
+            include='mock-repo',
+            exclude='another-mock-repo',
+        )
+        github_archive.initialize_project()
+
+    mock_logger.assert_called_with(message)
+    assert message == str(error.value)
+
+
 @patch('github_archive.archive.Github.get_user')
 def test_authenticated_user_in_users(mock_get_user):
     authenticated_user_in_users = GithubArchive(
@@ -375,6 +393,32 @@ def test_iterate_repos_matching_authed_username(mock_archive_repo, mock_github_i
     mock_archive_repo.assert_called_once()
 
 
+@patch('github_archive.archive.Github')
+@patch('github_archive.archive.GithubArchive.archive_repo')
+def test_iterate_repos_include_list(mock_archive_repo, mock_github_instance, mock_git_asset):
+    """Tests that we iterate repos that are on the include list."""
+    repos = [mock_git_asset]
+    GithubArchive(
+        users='mock_username',
+        include='mock-asset-name',
+    ).iterate_repos_to_archive(repos, CLONE_OPERATION)
+
+    mock_archive_repo.assert_called_once()
+
+
+@patch('github_archive.archive.Github')
+@patch('github_archive.archive.GithubArchive.archive_repo')
+def test_iterate_repos_exclude_list(mock_archive_repo, mock_github_instance, mock_git_asset):
+    """Tests that we do not iterate repos that are on the exclude list."""
+    repos = [mock_git_asset]
+    GithubArchive(
+        users='mock_username',
+        exclude='mock-asset-name',
+    ).iterate_repos_to_archive(repos, CLONE_OPERATION)
+
+    mock_archive_repo.assert_not_called()
+
+
 @patch('github_archive.archive.Github')
 @patch('github_archive.archive.GithubArchive.archive_gist')
 def test_iterate_gists(mock_archive_gist, mock_github_instance, mock_git_asset):