Repository: gruntwork-io/terragrunt (language: go)

Issue:
Optimize local download code behaviour
Hi!

Currently we are increasing our usage of terragrunt for a project and experiencing increasing execution times and also not easy optimizable CI/CD in terms of speed.
We use many local modules and therefor have references like `source = ../something`.

While investigating your code i found that local source code is always going to redownloaded and reinitialized.
https://github.com/gruntwork-io/terragrunt/blob/master/cli/download_source.go#L113

How about if we would hash the contents of the source directory with a function like: https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash#HashDir
and save the hash value as the "terragrunt version" and use it to compare if somethings needs to be downloaded or not?

Maybe something like this?

```go
func alreadyHaveLatestCode(terraformSource *tfsource.TerraformSource, terragruntOptions *options.TerragruntOptions) (bool, error) {
	fmt.Println(terraformSource)

	if !util.FileExists(terraformSource.DownloadDir) ||
		!util.FileExists(terraformSource.WorkingDir) ||
		!util.FileExists(terraformSource.VersionFile) {
		return false, nil
	}

	tfFiles, err := filepath.Glob(fmt.Sprintf("%s/*.tf", terraformSource.WorkingDir))
	if err != nil {
		return false, errors.WithStackTrace(err)
	}

	if len(tfFiles) == 0 {
		terragruntOptions.Logger.Debugf("Working dir %s exists but contains no Terraform files, so assuming code needs to be downloaded again.", terraformSource.WorkingDir)
		return false, nil
	}

	currentVersion := terraformSource.EncodeSourceVersion()
	previousVersion, err := readVersionFile(terraformSource)


	fmt.Println(currentVersion)
	fmt.Println(previousVersion)
	if err != nil {
		return false, err
	}

	if tfsource.IsLocalSource(terraformSource.CanonicalSourceURL) {
		sourceHash, err := dirhash.HashDir(terraformSource.WorkingDir, "hi", dirhash.DefaultHash)
		fmt.Println(sourceHash)

		if err != nil {
			return false, err
		}

		return previousVersion == sourceHash, nil
	}
	return previousVersion == currentVersion, nil
}
```


Discussion / hints:
This sounds reasonable! We're a bit buried with other initiatives to implement this ourselves, but if anyone from the community wants to open a PR with these changes, that would be greatly appreciated!

Fix patch:
diff --git a/cli/download_source.go b/cli/download_source.go
index d80d5ea9ae..209307b86f 100644
--- a/cli/download_source.go
+++ b/cli/download_source.go
@@ -126,14 +126,11 @@ func downloadTerraformSourceIfNecessary(terraformSource *tfsource.TerraformSourc
 
 // Returns true if the specified TerraformSource, of the exact same version, has already been downloaded into the
 // DownloadFolder. This helps avoid downloading the same code multiple times. Note that if the TerraformSource points
-// to a local file path, we assume the user is doing local development and always return false to ensure the latest
-// code is downloaded (or rather, copied) every single time. See the ProcessTerraformSource method for more info.
+// to a local file path, a hash will be generated from the contents of the source dir. See the ProcessTerraformSource method for more info.
 func alreadyHaveLatestCode(terraformSource *tfsource.TerraformSource, terragruntOptions *options.TerragruntOptions) (bool, error) {
-	if tfsource.IsLocalSource(terraformSource.CanonicalSourceURL) ||
-		!util.FileExists(terraformSource.DownloadDir) ||
+	if !util.FileExists(terraformSource.DownloadDir) ||
 		!util.FileExists(terraformSource.WorkingDir) ||
 		!util.FileExists(terraformSource.VersionFile) {
-
 		return false, nil
 	}
 
diff --git a/cli/tfsource/types.go b/cli/tfsource/types.go
index 745738f558..868d97035c 100644
--- a/cli/tfsource/types.go
+++ b/cli/tfsource/types.go
@@ -13,6 +13,8 @@ import (
 
 	"github.com/gruntwork-io/terragrunt/errors"
 	"github.com/gruntwork-io/terragrunt/util"
+
+	"golang.org/x/mod/sumdb/dirhash"
 )
 
 var forcedRegexp = regexp.MustCompile(`^([A-Za-z0-9]+)::(.+)$`)
@@ -40,9 +42,17 @@ func (src *TerraformSource) String() string {
 // string of the source URL, calculate its sha1, and base 64 encode it. For remote URLs (e.g. Git URLs), this is
 // based on the assumption that the scheme/host/path of the URL (e.g. git::github.com/foo/bar) identifies the module
 // name and the query string (e.g. ?ref=v0.0.3) identifies the version. For local file paths, there is no query string,
-// so the same file path (/foo/bar) is always considered the same version. See also the encodeSourceName and
-// ProcessTerraformSource methods.
+// so the same file path (/foo/bar) is always considered the same version. To detect changes the file path will be hashed
+// and returned as version. In case of hash error the default encoded source version will be returned.
+// See also the encodeSourceName and ProcessTerraformSource methods.
 func (terraformSource TerraformSource) EncodeSourceVersion() string {
+	if IsLocalSource(terraformSource.CanonicalSourceURL) {
+		hash, err := dirhash.HashDir(terraformSource.CanonicalSourceURL.Path, "prefix", dirhash.DefaultHash)
+		if err == nil {
+			return hash
+		}
+		// In case of error return default source version strategy
+	}
 	return util.EncodeBase64Sha1(terraformSource.CanonicalSourceURL.Query().Encode())
 }
 
diff --git a/go.mod b/go.mod
index 5db41ef93c..254f778e93 100644
--- a/go.mod
+++ b/go.mod
@@ -54,6 +54,7 @@ require (
 	github.com/zclconf/go-cty v1.8.3
 	go.mozilla.org/sops/v3 v3.7.2
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
+	golang.org/x/mod v0.5.1
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
diff --git a/go.sum b/go.sum
index 1df98bae65..924b8cb2e1 100644
--- a/go.sum
+++ b/go.sum
@@ -979,6 +979,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=


Test patch:
diff --git a/cli/download_source_test.go b/cli/download_source_test.go
index 4f620375bf..a6aae4d1df 100644
--- a/cli/download_source_test.go
+++ b/cli/download_source_test.go
@@ -21,6 +21,38 @@ import (
 	"github.com/gruntwork-io/terragrunt/util"
 )
 
+func TestAlreadyHaveLatestCodeLocalFilePathWithHashNoChanges(t *testing.T) {
+	t.Parallel()
+
+	canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world-local-hash"))
+	downloadDir := tmpDir(t)
+	defer os.Remove(downloadDir)
+
+	copyFolder(t, "../test/fixture-download-source/download-dir-version-file-local-hash", downloadDir)
+	testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, true)
+}
+
+func TestAlreadyHaveLatestCodeLocalFilePathWithHashChanged(t *testing.T) {
+	t.Parallel()
+
+	canonicalUrl := fmt.Sprintf("file://%s", absPath(t, "../test/fixture-download-source/hello-world-local-hash"))
+	downloadDir := tmpDir(t)
+	defer os.Remove(downloadDir)
+
+	copyFolder(t, "../test/fixture-download-source/download-dir-version-file-local-hash", downloadDir)
+
+	f, err := os.OpenFile(downloadDir+"/version-file.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	// Modify content of file to simulate change
+	fmt.Fprintln(f, "CHANGED")
+
+	testAlreadyHaveLatestCode(t, canonicalUrl, downloadDir, false)
+}
+
 func TestAlreadyHaveLatestCodeLocalFilePath(t *testing.T) {
 	t.Parallel()
 
diff --git a/test/fixture-download-source/download-dir-version-file-local-hash/main.tf b/test/fixture-download-source/download-dir-version-file-local-hash/main.tf
new file mode 100644
index 0000000000..e0ace37af6
--- /dev/null
+++ b/test/fixture-download-source/download-dir-version-file-local-hash/main.tf
@@ -0,0 +1,1 @@
+# Local file hash test
\ No newline at end of file
diff --git a/test/fixture-download-source/download-dir-version-file-local-hash/version-file.txt b/test/fixture-download-source/download-dir-version-file-local-hash/version-file.txt
new file mode 100644
index 0000000000..864fa82539
--- /dev/null
+++ b/test/fixture-download-source/download-dir-version-file-local-hash/version-file.txt
@@ -0,0 +1,1 @@
+h1:iZ9U+r4EPyqBU2l8Ih0StqixGyMnZTNoNqSGeEMT0QI=
\ No newline at end of file
diff --git a/test/fixture-download-source/hello-world-local-hash/main.tf b/test/fixture-download-source/hello-world-local-hash/main.tf
new file mode 100644
index 0000000000..e0ace37af6
--- /dev/null
+++ b/test/fixture-download-source/hello-world-local-hash/main.tf
@@ -0,0 +1,1 @@
+# Local file hash test
\ No newline at end of file