Repository: societe-generale/github-crawler
Pull request #111 (2022-11-14 14:30:06)

possibility to configure an inclusion list, instead of an exclusion list


Patch:
diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawler.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawler.kt
index 59df26f..9397d41 100644
--- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawler.kt
+++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawler.kt
@@ -45,7 +45,7 @@ class GitHubCrawler(private val remoteSourceControl: RemoteSourceControl,
 
         }
 
-        var repositoriesFromOrga = remoteSourceControl.fetchRepositories(organizationName)
+        val repositoriesFromOrga = remoteSourceControl.fetchRepositories(organizationName)
 
         fetchAndParseRepoContent(repositoriesFromOrga)
 
@@ -75,10 +75,10 @@ class GitHubCrawler(private val remoteSourceControl: RemoteSourceControl,
 
         log.info("${repositoriesFromOrga.size} repositories to crawl...")
 
-        val repoStream = if (gitHubCrawlerProperties.crawlInParallel) repositoriesFromOrga.parallelStream() else repositoriesFromOrga.stream();
+        val repoStream = if (gitHubCrawlerProperties.crawlInParallel) repositoriesFromOrga.parallelStream() else repositoriesFromOrga.stream()
 
         repoStream.map { repo -> logRepoProcessing(repo) }
-                .map { repo -> repo.flagAsExcludedIfRequired(gitHubCrawlerProperties.repositoriesToExclude) }
+                .map { repo -> repo.flagAsExcludedIfRequired(gitHubCrawlerProperties) }
                 .filter { repo -> shouldKeepForFurtherProcessing(repo, gitHubCrawlerProperties) }
                 .map { repo -> repositoryEnricher.loadRepoSpecificConfigIfAny(repo) }
                 .map { repo -> repo.flagAsExcludedIfConfiguredAtRepoLevel() }
@@ -97,7 +97,7 @@ class GitHubCrawler(private val remoteSourceControl: RemoteSourceControl,
 
     private fun logRepoProcessing(repo: Repository): Repository {
         log.info("processing repo ${repo.name}")
-        return repo;
+        return repo
     }
 
 
diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawlerProperties.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawlerProperties.kt
index 4f389ec..5423734 100644
--- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawlerProperties.kt
+++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/GitHubCrawlerProperties.kt
@@ -10,13 +10,21 @@ import org.springframework.boot.context.properties.NestedConfigurationProperty
 @EnableConfigurationProperties
 class GitHubCrawlerProperties(@NestedConfigurationProperty
                               val sourceControl : SourceControlConfig=SourceControlConfig(),
-                              val indicatorsToFetchByFile: Map<com.societegenerale.githubcrawler.FileToParse, List<IndicatorDefinition>> = HashMap(),
+                              val indicatorsToFetchByFile: Map<FileToParse, List<IndicatorDefinition>> = HashMap(),
                               var repositoriesToExclude: List<String> = ArrayList(),
+                              var repositoriesToInclude: List<String> = ArrayList(),
                               var publishExcludedRepositories: Boolean = false,
                               var crawlAllBranches: Boolean = false,
                               var crawlInParallel: Boolean = true,
                               //even if we're not using it, declaring outputs here so that it shows in completion
                               val outputs: List<GitHubCrawlerOutput> = emptyList(),
                               val miscRepositoryTasks: List<TaskDefinition>  = ArrayList()
-                              )
+                              ){
+  init{
+    if(repositoriesToExclude.isNotEmpty() && repositoriesToInclude.isNotEmpty()){
+      throw IllegalStateException("we can't have both exclusion AND inclusion patterns configured : please pick only one")
+    }
+  }
+
+}
 
diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/Repository.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/Repository.kt
index 937ba7a..1734934 100644
--- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/Repository.kt
+++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/Repository.kt
@@ -2,6 +2,7 @@ package com.societegenerale.githubcrawler.model
 
 import com.fasterxml.jackson.annotation.JsonIgnore
 import com.fasterxml.jackson.annotation.JsonProperty
+import com.societegenerale.githubcrawler.GitHubCrawlerProperties
 import com.societegenerale.githubcrawler.RepositoryConfig
 import feign.FeignException
 import org.slf4j.LoggerFactory
@@ -46,28 +47,44 @@ data class Repository(val url: String,
 
     val log = LoggerFactory.getLogger(this.javaClass)
 
-    fun flagAsExcludedIfRequired(repositoriesToExclude: List<String>): Repository {
+    fun flagAsExcludedIfRequired(config : GitHubCrawlerProperties): Repository {
 
-        val matchedOnAnExclusionPattern = repositoriesToExclude.map { s -> Pattern.compile(s) }
-                //below map returns true if it matches, false otherwise
-                .map { p -> logIfMatches(p) }
-                //we return the first item matching predicate "it", ie being true. Else, return false
-                .firstOrNull { it } ?: false
+      val repositoriesToExclude = config.repositoriesToExclude
+      val repositoriesToInclude = config.repositoriesToInclude
 
+      if(repositoriesToExclude.isNotEmpty()) {
+        val matchedOnAnExclusionPattern = repoNameMatchesAnyPatternsFrom(repositoriesToExclude)
 
         if (matchedOnAnExclusionPattern) {
-            log.info("\texcluding repo ${name} because of server config ")
+          log.info("\texcluding repo ${name} because of server config ")
 
-            return this.copy(excluded = true, reason = "excluded from server config side")
+          return this.copy(excluded = true, reason = "excluded from server config side")
         }
+      }
+      else if(repositoriesToInclude.isNotEmpty()){
+        val matchedOnAnInclusionPattern = repoNameMatchesAnyPatternsFrom(repositoriesToInclude)
 
-        return this
+        if (!matchedOnAnInclusionPattern) {
+          log.info("\texcluding repo ${name} because of server config - not matching the inclusion pattern")
+
+          return this.copy(excluded = true, reason = "excluded from server config side")
+        }
+
+      }
+
+      return this
     }
 
-    private fun logIfMatches(p: Pattern): Boolean {
+  private fun repoNameMatchesAnyPatternsFrom(repositoriesToInclude: List<String>) = repositoriesToInclude.map { s -> Pattern.compile(s) }
+      //below map returns true if it matches, false otherwise
+      .map { p -> logIfMatches(p) }
+      //we return the first item matching predicate "it", ie being true. Else, return false
+      .firstOrNull { it } ?: false
+
+  private fun logIfMatches(p: Pattern): Boolean {
 
         if (p.matcher(name).find()) {
-            log.debug("repo {} matched on exclusion pattern {}", name, p.pattern())
+            log.debug("repo {} matched on exclusion/inclusion pattern {}", name, p.pattern())
             return true
         }
 


Test patch:
diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerPropertiesTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerPropertiesTest.kt
new file mode 100644
index 0000000..cc54740
--- /dev/null
+++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerPropertiesTest.kt
@@ -0,0 +1,20 @@
+package com.societegenerale.githubcrawler
+
+import org.assertj.core.api.Assertions.assertThatThrownBy
+import org.junit.jupiter.api.Test
+
+class GitHubCrawlerPropertiesTest {
+
+  @Test
+  fun cant_have_both_inclusion_and_exclusion_patterns_configured() {
+
+    assertThatThrownBy {
+      var invalidProperties = GitHubCrawlerProperties(
+          repositoriesToInclude = listOf("pattern1"),
+          repositoriesToExclude = listOf("pattern2")
+      )
+    }.isInstanceOf(IllegalStateException::class.java)
+
+  }
+
+}
\ No newline at end of file
diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/model/RepositoryTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/model/RepositoryTest.kt
new file mode 100644
index 0000000..628d29f
--- /dev/null
+++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/model/RepositoryTest.kt
@@ -0,0 +1,38 @@
+package com.societegenerale.githubcrawler.model
+
+import com.societegenerale.githubcrawler.GitHubCrawlerProperties
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import java.sql.Date
+
+class RepositoryTest{
+
+  @Test
+  fun repo_flagged_according_to_inclusion_config(){
+
+    val inclusionConfigured = GitHubCrawlerProperties(
+        repositoriesToInclude = listOf("prefix-.*-service")
+    )
+
+    val repoTemplate=Repository(name="SOME_NAME_TO_REPLACE",
+        url="someUrl",
+        defaultBranch = "main",
+        creationDate = Date.valueOf("2022-11-14"),
+        lastUpdateDate = Date.valueOf("2022-11-14"),
+        fullName = "someFullName")
+
+
+    val repoToInclude=repoTemplate.copy(name="prefix-should_keep-service")
+    assertThat(repoToInclude.flagAsExcludedIfRequired(inclusionConfigured).excluded)
+        .`as`("repo should NOT be excluded")
+        .isFalse
+
+
+    val repoToNotInclude=repoTemplate.copy(name="prefix-should_not_keep-ui")
+    assertThat(repoToNotInclude.flagAsExcludedIfRequired(inclusionConfigured).excluded)
+        .`as`("repo SHOULD be excluded")
+        .isTrue
+  }
+
+
+}
\ No newline at end of file