Repository: globocom/huskyCI
Pull request #496 (2020-07-13 21:37:16)

Add logic to support cloning method change from HTTPS to SSH
### Motivation
We have received some reports that Golang repositories without a `vendor` folder and using internal dependencies are having some issues running huskyCI. We have opened [an issue](https://github.com/securego/gosec/issues/504) in gosec's project because we were facing the following error:

```
$ docker run -v ~/go/src/internalRemote/internalteam/internalproject/:/go/src/code -it huskyci/gosec:v2.3.0 sh
/go/src # cd code
/go/src/code # gosec ./...
[gosec] 2020/07/09 13:41:18 Including rules: default
[gosec] 2020/07/09 13:41:18 Excluding rules: default
[gosec] 2020/07/09 13:41:18 Import directory: /go/src/code/measures
The authenticity of host 'internalRemote (10.0.0.10)' can't be established.
ECDSA key fingerprint is SHA256:uanaXunASUnausaunANXuASuxnasu.
The authenticity of host 'internalRemote (10.0.0.10)' can't be established.
ECDSA key fingerprint is SHA256:uanaXunASUnausaunANXuASuxnasu.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
root@internalRemote's password:
```

After that, we realized that gosec scans rely on `go build` (thanks for the tips, @ccojocar) and we should handle this case inside our [huskyCI's gosec container commands](https://github.com/globocom/huskyCI/blob/master/api/config.yaml#L61). An internal discussion resulted in having huskyCI forcing the cloning method to be SSH, rather than HTTPS, to avoid `go build ` prompting the internalRemote password.

### Proposed Changes
This PR will add the possibility to add two environment variables in `huskyCI_API` to substitute the standard cloning URL from HTTPS to SSH.
This can be done by setting values such as:
```
HUSKYCI_API_GIT_SSH_URL=gitlab@gitlab.example.com
HUSKYCI_API_GIT_URL_TO_SUBSTITUTE=https://gitlab.example.com/
```

### Testing
Initially, tests should be working:
```
make test
```

After that, we need to run huskyCI with an internal remote's ssh key.
Since we already have an internal development environment with a ssh key set, the best and quickest way to properly test these changes would be through it.


### Note

It's important to mention that this PR addresses the issue of needing only 1 internal remote access. For information on how the support for multiple internal remotes is, be sure to check this [issue](#497).




rafaveira3: Great, @Krlier! Thanks a lot for working on this one. Let's put this branch on hold and test it inside our DEV environment!

Patch:
diff --git a/api/config.yaml b/api/config.yaml
index 5248897..a49fcdd 100644
--- a/api/config.yaml
+++ b/api/config.yaml
@@ -4,7 +4,7 @@ enry:
   imageTag: dev-b78a58c
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -26,7 +26,7 @@ gitauthors:
   imageTag: "2.18.4"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -60,10 +60,11 @@ gosec:
   imageTag: v2.3.0
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
-    echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
+    echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
+    git config --global url."%GIT_SSH_URL%:".insteadOf "%GIT_URL_TO_SUBSTITUTE%"
     cd src
     GIT_TERMINAL_PROMPT=0 git clone -b %GIT_BRANCH% --single-branch %GIT_REPO% code --quiet 2> /tmp/errorGitCloneGosec
     if [ $? -eq 0 ]; then
@@ -86,7 +87,7 @@ bandit:
   imageTag: "1.6.2"
   cmd: |+
      mkdir -p ~/.ssh &&
-     echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+     echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
      chmod 600 ~/.ssh/huskyci_id_rsa &&
      echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
      echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -112,7 +113,7 @@ brakeman:
   imageTag: "4.8.2"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -141,7 +142,7 @@ safety:
   imageTag: "1.9.0"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -187,7 +188,7 @@ npmaudit:
   imageTag: "6.14.5"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -217,7 +218,7 @@ yarnaudit:
   imageTag: "1.22.4"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -253,7 +254,7 @@ spotbugs:
   imageTag: "4.0.0-beta4"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -309,7 +310,7 @@ gitleaks:
   imageTag: "2.1.0"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
@@ -340,7 +341,7 @@ tfsec:
   imageTag: "v0.21.0"
   cmd: |+
     mkdir -p ~/.ssh &&
-    echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&
+    echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&
     chmod 600 ~/.ssh/huskyci_id_rsa &&
     echo "IdentityFile ~/.ssh/huskyci_id_rsa" >> /etc/ssh/ssh_config &&
     echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&
diff --git a/api/util/util.go b/api/util/util.go
index c53bc4a..3aa5ddc 100644
--- a/api/util/util.go
+++ b/api/util/util.go
@@ -31,7 +31,7 @@ const (
 const logInfoAnalysis = "ANALYSIS"
 const logActionReceiveRequest = "ReceiveRequest"
 
-// HandleCmd will extract %GIT_REPO%, %GIT_BRANCH% and %INTERNAL_DEP_URL% from cmd and replace it with the proper repository URL.
+// HandleCmd will extract %GIT_REPO%, %GIT_BRANCH% from cmd and replace it with the proper repository URL.
 func HandleCmd(repositoryURL, repositoryBranch, cmd string) string {
 	if repositoryURL != "" && repositoryBranch != "" && cmd != "" {
 		replace1 := strings.Replace(cmd, "%GIT_REPO%", repositoryURL, -1)
@@ -41,10 +41,25 @@ func HandleCmd(repositoryURL, repositoryBranch, cmd string) string {
 	return ""
 }
 
+// HandleGitURLSubstitution will extract GIT_SSH_URL and GIT_URL_TO_SUBSTITUTE from cmd and replace it with the SSH equivalent.
+func HandleGitURLSubstitution(rawString string) string {
+	gitSSHURL := os.Getenv("HUSKYCI_API_GIT_SSH_URL")
+	gitURLToSubstitute := os.Getenv("HUSKYCI_API_GIT_URL_TO_SUBSTITUTE")
+
+	if gitSSHURL == "" || gitURLToSubstitute == "" {
+		gitSSHURL = ""
+		gitURLToSubstitute = ""
+	}
+	cmdReplaced := strings.Replace(rawString, "%GIT_SSH_URL%", gitSSHURL, -1)
+	cmdReplaced = strings.Replace(cmdReplaced, "%GIT_URL_TO_SUBSTITUTE%", gitURLToSubstitute, -1)
+
+	return cmdReplaced
+}
+
 // HandlePrivateSSHKey will extract %GIT_PRIVATE_SSH_KEY% from cmd and replace it with the proper private SSH key.
 func HandlePrivateSSHKey(rawString string) string {
 	privKey := os.Getenv("HUSKYCI_API_GIT_PRIVATE_SSH_KEY")
-	cmdReplaced := strings.Replace(rawString, "GIT_PRIVATE_SSH_KEY", privKey, -1)
+	cmdReplaced := strings.Replace(rawString, "%GIT_PRIVATE_SSH_KEY%", privKey, -1)
 	return cmdReplaced
 }
 


Test patch:
diff --git a/api/securitytest/securitytest.go b/api/securitytest/securitytest.go
index 460add0..447f3e7 100644
--- a/api/securitytest/securitytest.go
+++ b/api/securitytest/securitytest.go
@@ -89,6 +89,7 @@ func (scanInfo *SecTestScanInfo) dockerRun(timeOutInSeconds int) error {
 	image := scanInfo.Container.SecurityTest.Image
 	imageTag := scanInfo.Container.SecurityTest.ImageTag
 	cmd := util.HandleCmd(scanInfo.URL, scanInfo.Branch, scanInfo.Container.SecurityTest.Cmd)
+	cmd = util.HandleGitURLSubstitution(cmd)
 	finalCMD := util.HandlePrivateSSHKey(cmd)
 	CID, cOutput, err := huskydocker.DockerRun(image, imageTag, finalCMD, timeOutInSeconds)
 	if err != nil {
diff --git a/api/util/util_test.go b/api/util/util_test.go
index 4573344..d97f205 100644
--- a/api/util/util_test.go
+++ b/api/util/util_test.go
@@ -49,9 +49,54 @@ var _ = Describe("Util", func() {
 		})
 	})
 
+	Describe("HandleGitURLSubstitution", func() {
+
+		rawString := "git config --global url.\"%GIT_SSH_URL%:\".insteadOf \"%GIT_URL_TO_SUBSTITUTE%\""
+		expectedURLToSubstituteNotEmpty := "git config --global url.\":\".insteadOf \"\""
+		expectedSSHURLNotEmpty := "git config --global url.\":\".insteadOf \"\""
+		expectedBothVarsEmpty := "git config --global url.\":\".insteadOf \"\""
+		expectedNotEmpty := "git config --global url.\"gitlab@gitlab.example.com:\".insteadOf \"https://gitlab.example.com/\""
+
+		Context("When rawString is not empty, HUSKYCI_API_GIT_SSH_URL is empty, but HUSKYCI_API_GIT_URL_TO_SUBSTITUTE is not empty", func() {
+			It("Should return a string based on these params", func() {
+				os.Setenv("HUSKYCI_API_GIT_SSH_URL", "")
+				os.Setenv("HUSKYCI_API_GIT_URL_TO_SUBSTITUTE", "https://gitlab.example.com/")
+				Expect(util.HandleGitURLSubstitution(rawString)).To(Equal(expectedURLToSubstituteNotEmpty))
+			})
+		})
+		Context("When rawString is not empty, HUSKYCI_API_GIT_SSH_URL is not empty and HUSKYCI_API_GIT_URL_TO_SUBSTITUTE is empty", func() {
+			It("Should return a string based on these params", func() {
+				os.Setenv("HUSKYCI_API_GIT_SSH_URL", "gitlab@gitlab.example.com")
+				os.Setenv("HUSKYCI_API_GIT_URL_TO_SUBSTITUTE", "")
+				Expect(util.HandleGitURLSubstitution(rawString)).To(Equal(expectedSSHURLNotEmpty))
+			})
+		})
+		Context("When rawString is not empty, HUSKYCI_API_GIT_SSH_URL is empty and HUSKYCI_API_GIT_URL_TO_SUBSTITUTE is empty", func() {
+			It("Should return a string based on these params", func() {
+				os.Setenv("HUSKYCI_API_GIT_SSH_URL", "")
+				os.Setenv("HUSKYCI_API_GIT_URL_TO_SUBSTITUTE", "")
+				Expect(util.HandleGitURLSubstitution(rawString)).To(Equal(expectedBothVarsEmpty))
+			})
+		})
+		Context("When rawString is not empty, HUSKYCI_API_GIT_SSH_URL is not empty and HUSKYCI_API_GIT_URL_TO_SUBSTITUTE is not empty", func() {
+			It("Should return a string based on these params", func() {
+				os.Setenv("HUSKYCI_API_GIT_SSH_URL", "gitlab@gitlab.example.com")
+				os.Setenv("HUSKYCI_API_GIT_URL_TO_SUBSTITUTE", "https://gitlab.example.com/")
+				Expect(util.HandleGitURLSubstitution(rawString)).To(Equal(expectedNotEmpty))
+			})
+		})
+		Context("When rawString is empty", func() {
+			It("Should return an empty string", func() {
+				os.Setenv("HUSKYCI_API_GIT_SSH_URL", "gitlab@gitlab.example.com")
+				os.Setenv("HUSKYCI_API_GIT_URL_TO_SUBSTITUTE", "https://gitlab.example.com/")
+				Expect(util.HandleGitURLSubstitution("")).To(Equal(""))
+			})
+		})
+	})
+
 	Describe("HandlePrivateSSHKey", func() {
 
-		rawString := "echo 'GIT_PRIVATE_SSH_KEY' > ~/.ssh/huskyci_id_rsa &&"
+		rawString := "echo '%GIT_PRIVATE_SSH_KEY%' > ~/.ssh/huskyci_id_rsa &&"
 		expectedNotEmpty := "echo 'PRIVKEYTEST' > ~/.ssh/huskyci_id_rsa &&"
 		expectedEmpty := "echo '' > ~/.ssh/huskyci_id_rsa &&"