Repository: chanzuckerberg/fogg
Pull request #663 (2022-06-13 18:27:27)

feat: allow fogg to authenticate with GH OAuth token
### Summary
This PR introduces a feature to fogg that allows the user to provide a Github OAuth token when doing module source downloading instead of SSH keys. This is handy for bot users and OAuth automation that use Fogg in the backend and don't have SSH keys. This should allow someone to do the following:

~~~
FOGG_GITHUBTOKEN=<token> fogg apply
~~~

Where the `FOGG_GITHUBTOKEN` is a personal access token or a token received through an OAuth flow.

### Test Plan
* Unit tests

### References
(Optional) Additional links to provide more context.



Patch:
diff --git a/apply/apply.go b/apply/apply.go
index 6698233a..f5fd0bb4 100644
--- a/apply/apply.go
+++ b/apply/apply.go
@@ -204,9 +204,13 @@ func applyEnvs(
 			}
 
 			if componentPlan.ModuleSource != nil {
-				e := applyModuleInvocation(fs, path, *componentPlan.ModuleSource, componentPlan.ModuleName, templates.Templates.ModuleInvocation, commonBox)
-				if e != nil {
-					return errs.WrapUser(e, "unable to apply module invocation")
+				downloader, err := util.MakeDownloader(*componentPlan.ModuleSource)
+				if err != nil {
+					return errs.WrapUser(err, "unable to make a downloader")
+				}
+				err = applyModuleInvocation(fs, path, *componentPlan.ModuleSource, componentPlan.ModuleName, templates.Templates.ModuleInvocation, commonBox, downloader)
+				if err != nil {
+					return errs.WrapUser(err, "unable to apply module invocation")
 				}
 			}
 		}
@@ -379,13 +383,15 @@ func applyModuleInvocation(
 	path, moduleAddress string,
 	inModuleName *string,
 	box fs.FS,
-	commonBox fs.FS) error {
+	commonBox fs.FS,
+	downloadFunc util.ModuleDownloader,
+) error {
 	e := fs.MkdirAll(path, 0755)
 	if e != nil {
 		return errs.WrapUserf(e, "couldn't create %s directory", path)
 	}
 
-	moduleConfig, e := util.DownloadAndParseModule(fs, moduleAddress)
+	moduleConfig, e := downloadFunc.DownloadAndParseModule(fs)
 	if e != nil {
 		return errs.WrapUser(e, "could not download or parse module")
 	}
diff --git a/cmd/apply.go b/cmd/apply.go
index 349729dd..5693acd8 100644
--- a/cmd/apply.go
+++ b/cmd/apply.go
@@ -8,7 +8,7 @@ import (
 
 func init() {
 	applyCmd.Flags().StringP("config", "c", "fogg.yml", "Use this to override the fogg config file.")
-	applyCmd.Flags().BoolP("upgrade", "u", false, "use this when running a new version of fogg")
+	applyCmd.Flags().BoolP("upgrade", "u", false, "Use this when running a new version of fogg")
 	rootCmd.AddCommand(applyCmd)
 }
 
diff --git a/go.mod b/go.mod
index 53a31534..417ddfa3 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
 	github.com/hashicorp/terraform v0.14.9
 	github.com/hashicorp/terraform-config-inspect v0.0.0-20200806211835-c481b8bfa41e
 	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/kr/pretty v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/peterbourgon/diskv v2.0.1+incompatible
diff --git a/go.sum b/go.sum
index 47a1396a..6f46bcae 100644
--- a/go.sum
+++ b/go.sum
@@ -515,6 +515,8 @@ github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/l
 github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
 github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
diff --git a/util/module_storage.go b/util/module_storage.go
index 091aaed1..db643894 100644
--- a/util/module_storage.go
+++ b/util/module_storage.go
@@ -2,13 +2,18 @@ package util
 
 import (
 	"crypto/sha256"
+	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/chanzuckerberg/fogg/errs"
 	getter "github.com/hashicorp/go-getter"
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
+	"github.com/kelseyhightower/envconfig"
 	homedir "github.com/mitchellh/go-homedir"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 )
 
@@ -65,12 +70,69 @@ func GetFoggCachePath() (string, error) {
 	return dir, nil
 }
 
-func DownloadAndParseModule(fs afero.Fs, mod string) (*tfconfig.Module, error) {
+type ModuleDownloader interface {
+	DownloadAndParseModule(fs afero.Fs) (*tfconfig.Module, error)
+}
+
+type Downloader struct {
+	Source string
+}
+
+// Changes a URL to use the git protocol over HTTPS instead of SSH.
+// If the URL was not a remote URL or an git/SSH protocol,
+// it will return the path that was passed in. If it does
+// convert it properly, it will add Github credentials to the path
+func convertSSHToGithubHTTPURL(sURL, token string) string {
+	// only detect the remote destinations
+	s, err := getter.Detect(sURL, token, []getter.Detector{
+		&getter.GitLabDetector{},
+		&getter.GitHubDetector{},
+		&getter.GitDetector{},
+		&getter.BitBucketDetector{},
+		&getter.S3Detector{},
+		&getter.GCSDetector{},
+	})
+	if err != nil {
+		logrus.Debug(err)
+		return sURL
+	}
+
+	splits := strings.Split(s, "git::ssh://git@")
+	if len(splits) != 2 {
+		return sURL
+	}
+	u, err := url.Parse(fmt.Sprintf("https://%s", splits[1]))
+	if err != nil {
+		logrus.Debug(err)
+		return sURL
+	}
+	u.User = url.User(token)
+
+	// we want to force the git protocol
+	return fmt.Sprintf("git::%s", u.String())
+}
+
+func MakeDownloader(src string) (*Downloader, error) {
+	type HTTPAuth struct {
+		GithubToken *string
+	}
+	var httpAuth HTTPAuth
+	err := envconfig.Process("fogg", &httpAuth)
+	if err != nil {
+		return nil, errs.WrapUser(err, "unable to get env FOGG_GITHUBTOKEN flag")
+	}
+	if httpAuth.GithubToken != nil {
+		src = convertSSHToGithubHTTPURL(src, *httpAuth.GithubToken)
+	}
+	return &Downloader{Source: src}, nil
+}
+
+func (dd *Downloader) DownloadAndParseModule(fs afero.Fs) (*tfconfig.Module, error) {
 	dir, err := GetFoggCachePath()
 	if err != nil {
 		return nil, err
 	}
-	d, err := DownloadModule(fs, dir, mod)
+	d, err := DownloadModule(fs, dir, dd.Source)
 	if err != nil {
 		return nil, errs.WrapUser(err, "unable to download module")
 	}


Test patch:
diff --git a/apply/apply_test.go b/apply/apply_test.go
index 8add5769..d0b1a153 100644
--- a/apply/apply_test.go
+++ b/apply/apply_test.go
@@ -257,8 +257,9 @@ func TestApplyModuleInvocation(t *testing.T) {
 	r.NoError(err)
 
 	fs := afero.NewCopyOnWriteFs(pwdFs, testFs)
-
-	e := applyModuleInvocation(fs, "mymodule", "test-module", nil, templates.Templates.ModuleInvocation, templates.Templates.Common)
+	downloader, err := util.MakeDownloader("test-module")
+	r.NoError(err)
+	e := applyModuleInvocation(fs, "mymodule", "test-module", nil, templates.Templates.ModuleInvocation, templates.Templates.Common, downloader)
 	r.NoError(e)
 
 	s, e := fs.Stat("mymodule")
@@ -291,7 +292,9 @@ func TestApplyModuleInvocationWithModuleName(t *testing.T) {
 	fs := afero.NewCopyOnWriteFs(pwdFs, testFs)
 
 	moduleName := "module-name"
-	e := applyModuleInvocation(fs, "mymodule", "test-module", &moduleName, templates.Templates.ModuleInvocation, templates.Templates.Common)
+	downloader, err := util.MakeDownloader("test-module")
+	r.NoError(err)
+	e := applyModuleInvocation(fs, "mymodule", "test-module", &moduleName, templates.Templates.ModuleInvocation, templates.Templates.Common, downloader)
 	r.NoError(e)
 
 	s, e := fs.Stat("mymodule")
diff --git a/util/module_storage_test.go b/util/module_storage_test.go
index ad17ddb8..ba1596a9 100644
--- a/util/module_storage_test.go
+++ b/util/module_storage_test.go
@@ -4,10 +4,12 @@
 package util
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
 
+	getter "github.com/hashicorp/go-getter"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
 )
@@ -34,8 +36,9 @@ func TestDownloadAndParseModule(t *testing.T) {
 	pwd, e := os.Getwd()
 	r.NoError(e)
 	fs := afero.NewBasePathFs(afero.NewOsFs(), pwd)
-
-	c, e := DownloadAndParseModule(fs, "github.com/chanzuckerberg/fogg-test-module")
+	downloader, err := MakeDownloader("github.com/chanzuckerberg/fogg-test-module")
+	r.NoError(err)
+	c, e := downloader.DownloadAndParseModule(fs)
 	r.Nil(e)
 	r.NotNil(c)
 	r.NotNil(c.Variables)
@@ -43,3 +46,70 @@ func TestDownloadAndParseModule(t *testing.T) {
 	r.Len(c.Variables, 2)
 	r.Len(c.Outputs, 2)
 }
+
+func TestMakeDownloader(t *testing.T) {
+	r := require.New(t)
+	creds := "REDACTED"
+	downloader, err := MakeDownloader("git@github.com:chanzuckerberg/test-repo//terraform/modules/eks-airflow?ref=v0.80.0")
+	r.NoError(err)
+	r.Equal("git@github.com:chanzuckerberg/test-repo//terraform/modules/eks-airflow?ref=v0.80.0", downloader.Source)
+
+	os.Setenv("FOGG_GITHUBTOKEN", creds)
+	defer os.Unsetenv("FOGG_GITHUBTOKEN")
+	downloader, err = MakeDownloader("git@github.com:chanzuckerberg/test-repo//terraform/modules/eks-airflow?ref=v0.80.0")
+	r.NoError(err)
+	r.Equal(fmt.Sprintf("git::https://%s@github.com/chanzuckerberg/test-repo//terraform/modules/eks-airflow?ref=v0.80.0", creds), downloader.Source)
+}
+
+func TestConvertSSHToHTTP(t *testing.T) {
+	r := require.New(t)
+
+	type test struct {
+		in  string
+		out string
+	}
+	creds := "REDACTED"
+	tests := func(token string) []test {
+		return []test{
+			{
+				in:  "git@github.com:chanzuckerberg/test-repo//terraform/modules/eks-airflow?ref=v0.80.0",
+				out: fmt.Sprintf("git::https://%s@github.com/chanzuckerberg/test-repo//terraform/modules/eks-airflow?ref=v0.80.0", token),
+			},
+		}
+	}(creds)
+	for _, test := range tests {
+		u := convertSSHToGithubHTTPURL(test.in, creds)
+		s, err := getter.Detect(u, creds, []getter.Detector{
+			&getter.GitHubDetector{},
+		})
+		r.NoError(err)
+		r.Equal(test.out, u)
+		r.Equal(test.out, s)
+	}
+}
+func TestConvertSSHToHTTPFail(t *testing.T) {
+	r := require.New(t)
+
+	type test struct {
+		in string
+	}
+
+	tests := func() []test {
+		return []test{
+			{
+				in: "github.com/scholzj/terraform-aws-vpc",
+			},
+			{
+				in: "terraform/modules/test",
+			},
+			{
+				in: "github.com/hashicorp/go-getter?ref=abcd12",
+			},
+		}
+	}()
+	for _, test := range tests {
+		s := convertSSHToGithubHTTPURL(test.in, "")
+		// make sure nothing changed in a failure case
+		r.Equal(test.in, s)
+	}
+}