Repository: globocom/huskyCI
Pull request #528 (2021-03-17 15:26:45)

Implementing Language Exclusion for analysis
### Description

We recently had an issue with a project where HuskyCI would run NPMAudit and YarnAudit due to a subproject folder containing UI code. We wanted to exclude that validation and noticed there was no resource for disabling a specific language analysis from being executed.


### Proposed Changes

This PR introduces a new environment variable "HUSKYCI_LANGUAGE_EXCLUSIONS", which is a comma separated list of languages to be excluded from the analysis flow.

A map containing the values is sent by the client to the API, which is then used after the enryScan is executed, where it check if any of the identified languages on the project is included in the exclusion list, removing it from list of repo languages to be analyzed.

### Testing

By adding the env variable HUSKYCI_LANGUAGE_EXCLUSIONS and executing make run-client, one should be able to turn off language specific validations from being executed.

Example:
```
export HUSKYCI_LANGUAGE_EXCLUSIONS=JavaScript,Ruby
```


Patch:
diff --git a/api/analysis/analysis.go b/api/analysis/analysis.go
index a8142b1..e0c9e9a 100644
--- a/api/analysis/analysis.go
+++ b/api/analysis/analysis.go
@@ -38,7 +38,7 @@ func StartAnalysis(RID string, repository types.Repository) {
 		}
 	}()
 
-	if err := enryScan.New(RID, repository.URL, repository.Branch, enryScan.SecurityTestName); err != nil {
+	if err := enryScan.New(RID, repository.URL, repository.Branch, enryScan.SecurityTestName, repository.LanguageExclusions); err != nil {
 		log.Error(logActionStart, logInfoAnalysis, 2011, err)
 		return
 	}
diff --git a/api/db/postgres.go b/api/db/postgres.go
index 680a6aa..78b903b 100644
--- a/api/db/postgres.go
+++ b/api/db/postgres.go
@@ -132,7 +132,7 @@ func (pR *PostgresRequests) FindAllDBAnalysis(
 
 // InsertDBRepository inserts a new repository into repository table.
 func (pR *PostgresRequests) InsertDBRepository(repository types.Repository) error {
-	if (types.Repository{}) == repository {
+	if repository.URL == "" || time.Time.IsZero(repository.CreatedAt) {
 		return errors.New("Empty repository data")
 	}
 	repositoryMap := map[string]interface{}{
diff --git a/api/types/types.go b/api/types/types.go
index fcdfb4c..a18c87e 100644
--- a/api/types/types.go
+++ b/api/types/types.go
@@ -10,9 +10,10 @@ import (
 
 // Repository is the struct that stores all data from repository to be analyzed.
 type Repository struct {
-	URL       string    `bson:"repositoryURL" json:"repositoryURL"`
-	Branch    string    `json:"repositoryBranch"`
-	CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
+	URL                string          `bson:"repositoryURL" json:"repositoryURL"`
+	Branch             string          `json:"repositoryBranch"`
+	LanguageExclusions map[string]bool `json:"languageExclusions"`
+	CreatedAt          time.Time       `bson:"createdAt" json:"createdAt"`
 }
 
 // SecurityTest is the struct that stores all data from the security tests to be executed.
diff --git a/client/analysis/analysis.go b/client/analysis/analysis.go
index e18edcd..7e95fff 100644
--- a/client/analysis/analysis.go
+++ b/client/analysis/analysis.go
@@ -27,6 +27,7 @@ func StartAnalysis() (string, error) {
 	requestPayload := types.JSONPayload{
 		RepositoryURL:    config.RepositoryURL,
 		RepositoryBranch: config.RepositoryBranch,
+		LanguageExclusions: config.LanguageExclusions,
 	}
 
 	marshalPayload, err := json.Marshal(requestPayload)
diff --git a/client/config/config.go b/client/config/config.go
index d8789c5..3c9e11a 100644
--- a/client/config/config.go
+++ b/client/config/config.go
@@ -7,6 +7,7 @@ package config
 import (
 	"errors"
 	"os"
+	"strings"
 )
 
 // RepositoryURL stores the repository URL of the project to be analyzed.
@@ -21,6 +22,7 @@ var RepositoryBranch string
 // HuskyToken is the token used to scan a repository.
 var HuskyToken string
 
+var LanguageExclusions map[string]bool
 // HuskyUseTLS stores if huskyCI is to use an HTTPS connection.
 var HuskyUseTLS bool
 
@@ -29,6 +31,14 @@ func SetConfigs() {
 	RepositoryURL = os.Getenv(`HUSKYCI_CLIENT_REPO_URL`)
 	RepositoryBranch = os.Getenv(`HUSKYCI_CLIENT_REPO_BRANCH`)
 	HuskyAPI = os.Getenv(`HUSKYCI_CLIENT_API_ADDR`)
+	exclusionsEnv := os.Getenv(`HUSKYCI_LANGUAGE_EXCLUSIONS`)
+	if exclusionsEnv != "" {
+		languagesToExclude := strings.Split(exclusionsEnv, ",")
+		LanguageExclusions = make(map[string]bool)
+		for _, lang := range languagesToExclude {
+			LanguageExclusions[lang] = true
+		}
+	}
 	HuskyToken = os.Getenv(`HUSKYCI_CLIENT_TOKEN`)
 	HuskyUseTLS = getUseTLS()
 }
diff --git a/client/types/types.go b/client/types/types.go
index 1dfe50d..a2ec2b6 100644
--- a/client/types/types.go
+++ b/client/types/types.go
@@ -23,6 +23,7 @@ var IsJSONoutput bool
 type JSONPayload struct {
 	RepositoryURL    string `json:"repositoryURL"`
 	RepositoryBranch string `json:"repositoryBranch"`
+	LanguageExclusions map[string]bool `json:"languageExclusions"`
 }
 
 // Target is the struct that represents HuskyCI API target


Test patch:
diff --git a/api/log/log_test.go b/api/log/log_test.go
index e09628e..1734ecb 100644
--- a/api/log/log_test.go
+++ b/api/log/log_test.go
@@ -66,7 +66,7 @@ func TestLog(t *testing.T) {
 		},
 		{
 			name:         "Testing log.Error fail",
-			logger:      &stubLogger{err: errors.New("server is down!")},
+			logger:       &stubLogger{err: errors.New("server is down!")},
 			wantAction:   "action",
 			wantInfo:     "err",
 			wantMsgCode:  11,
@@ -87,7 +87,7 @@ func TestLog(t *testing.T) {
 		},
 		{
 			name:         "Testing log.Warning fail",
-			logger:      &stubLogger{err: errors.New("server is down!")},
+			logger:       &stubLogger{err: errors.New("server is down!")},
 			wantAction:   "action",
 			wantInfo:     "err",
 			wantMsgCode:  11,
diff --git a/api/securitytest/enry.go b/api/securitytest/enry.go
index 1947c51..b4a9b40 100644
--- a/api/securitytest/enry.go
+++ b/api/securitytest/enry.go
@@ -52,12 +52,16 @@ func (enryScan *SecTestScanInfo) prepareEnryOutput() error {
 				return errMsg
 			}
 		}
-		newLanguage := types.Code{
-			Language: name,
-			Files:    fs,
+
+		if !enryScan.LanguageExclusions[name] {
+			newLanguage := types.Code{
+				Language: name,
+				Files:    fs,
+			}
+			repositoryLanguages = append(repositoryLanguages, newLanguage)
 		}
-		repositoryLanguages = append(repositoryLanguages, newLanguage)
 	}
+
 	enryScan.Codes = repositoryLanguages
 	return nil
 }
diff --git a/api/securitytest/run.go b/api/securitytest/run.go
index abbe61e..da87452 100644
--- a/api/securitytest/run.go
+++ b/api/securitytest/run.go
@@ -103,7 +103,9 @@ func (results *RunAllInfo) runGenericScans(enryScan SecTestScanInfo) error {
 		go func(genericTest *types.SecurityTest) {
 			defer wg.Done()
 			newGenericScan := SecTestScanInfo{}
-			if err := newGenericScan.New(enryScan.RID, enryScan.URL, enryScan.Branch, genericTest.Name); err != nil {
+			// LanguageExclusions is only utilized on first scan (enryScan) therefore set as nil 
+			enryScan.LanguageExclusions = nil
+			if err := newGenericScan.New(enryScan.RID, enryScan.URL, enryScan.Branch, genericTest.Name, enryScan.LanguageExclusions); err != nil {
 				select {
 				case <-syncChan:
 					return
@@ -166,7 +168,9 @@ func (results *RunAllInfo) runLanguageScans(enryScan SecTestScanInfo) error {
 		go func(languageTest *types.SecurityTest) {
 			defer wg.Done()
 			newLanguageScan := SecTestScanInfo{}
-			if err := newLanguageScan.New(enryScan.RID, enryScan.URL, enryScan.Branch, languageTest.Name); err != nil {
+			// LanguageExclusions is only utilized on first scan (enryScan) therefore set as nil 
+			enryScan.LanguageExclusions = nil
+			if err := newLanguageScan.New(enryScan.RID, enryScan.URL, enryScan.Branch, languageTest.Name, enryScan.LanguageExclusions); err != nil {
 				select {
 				case <-syncChan:
 					return
diff --git a/api/securitytest/securitytest.go b/api/securitytest/securitytest.go
index 447f3e7..a159dc2 100644
--- a/api/securitytest/securitytest.go
+++ b/api/securitytest/securitytest.go
@@ -32,6 +32,7 @@ type SecTestScanInfo struct {
 	URL                   string
 	Branch                string
 	SecurityTestName      string
+	LanguageExclusions    map[string]bool
 	ErrorFound            error
 	ReqNotFound           bool
 	WarningFound          bool
@@ -49,10 +50,11 @@ type SecTestScanInfo struct {
 }
 
 // New creates a new huskyCI scan based given RID, URL, Branch and a securityTest name and returns an error.
-func (scanInfo *SecTestScanInfo) New(RID, URL, branch, securityTestName string) error {
+func (scanInfo *SecTestScanInfo) New(RID, URL, branch, securityTestName string, languageExclusions map[string]bool) error {
 	scanInfo.RID = RID
 	scanInfo.URL = URL
 	scanInfo.Branch = branch
+	scanInfo.LanguageExclusions = languageExclusions
 	scanInfo.SecurityTestName = securityTestName
 	return scanInfo.setSecurityTestContainer(securityTestName)
 }