Repository: magefile/mage
Pull request #258 (2019-07-18 16:41:56)

add MAGEFILE_HASHFAST to avoid rerunning go build
This is a feature that is mostly aimed at developers running on Windows. For some reason, the `go build` code that determines whether it needs to rebuild is slow on windows.  For people running mage, that means that even if no code has changed, it can take a full second or longer for mage to figure out it can just reuse the binary it already built.  

To solve that problem, this PR adds an environment variable you can set to tell mage to just use the hash of the magefiles themselves, and not run `go build`.  This is the way mage used to work before moving to rely on go build's cache.  

The one drawback of this method is that if a dependency of the magefile has changed (like your magefile imports a library that gets updated), mage won't detect the change and will just rerun the binary it has cached.  You can force the rebuild by using the already-existing -f flag.


Patch:
diff --git a/mage/main.go b/mage/main.go
index a8adfb2..f24bb5a 100644
--- a/mage/main.go
+++ b/mage/main.go
@@ -110,6 +110,7 @@ type Invocation struct {
 	Args       []string      // args to pass to the compiled binary
 	GoCmd      string        // the go binary command to run
 	CacheDir   string        // the directory where we should store compiled binaries
+	HashFast   bool          // don't rely on GOCACHE, just hash the magefiles
 }
 
 // ParseAndRun parses the command line, and then compiles and runs the mage
@@ -282,7 +283,7 @@ Options:
 	if len(inv.Args) > 0 && cmd != None {
 		return inv, cmd, fmt.Errorf("unexpected arguments to command: %q", inv.Args)
 	}
-
+	inv.HashFast = mg.HashFast()
 	return inv, cmd, err
 }
 
@@ -321,12 +322,16 @@ func Invoke(inv Invocation) int {
 	debug.Println("output exe is ", exePath)
 
 	useCache := false
-	if s, err := internal.OutputDebug(inv.GoCmd, "env", "GOCACHE"); err == nil {
-		// if GOCACHE exists, always rebuild, so we catch transitive
-		// dependencies that have changed.
-		if s != "" {
-			debug.Println("build cache exists, will ignore any compiled binary")
-			useCache = true
+	if inv.HashFast {
+		debug.Println("user has set MAGEFILE_HASHFAST, so we'll ignore GOCACHE")
+	} else {
+		if s, err := internal.OutputDebug(inv.GoCmd, "env", "GOCACHE"); err == nil {
+			// if GOCACHE exists, always rebuild, so we catch transitive
+			// dependencies that have changed.
+			if s != "" {
+				debug.Println("go build cache exists, will ignore any compiled binary")
+				useCache = true
+			}
 		}
 	}
 
diff --git a/mg/runtime.go b/mg/runtime.go
index 087caf9..4dbe0b1 100644
--- a/mg/runtime.go
+++ b/mg/runtime.go
@@ -27,6 +27,13 @@ const GoCmdEnv = "MAGEFILE_GOCMD"
 // to ignore the default target specified in the magefile.
 const IgnoreDefaultEnv = "MAGEFILE_IGNOREDEFAULT"
 
+// HashFastEnv is the environment variable that indicates the user requested to
+// use a quick hash of magefiles to determine whether or not the magefile binary
+// needs to be rebuilt. This results in faster runtimes, but means that mage
+// will fail to rebuild if a dependency has changed. To force a rebuild, run
+// mage with the -f flag.
+const HashFastEnv = "MAGEFILE_HASHFAST"
+
 // Verbose reports whether a magefile was run with the verbose flag.
 func Verbose() bool {
 	b, _ := strconv.ParseBool(os.Getenv(VerboseEnv))
@@ -48,6 +55,13 @@ func GoCmd() string {
 	return "go"
 }
 
+// HashFast reports whether the user has requested to use the fast hashing
+// mechanism rather than rely on go's rebuilding mechanism.
+func HashFast() bool {
+	b, _ := strconv.ParseBool(os.Getenv(HashFastEnv))
+	return b
+}
+
 // IgnoreDefault reports whether the user has requested to ignore the default target
 // in the magefile.
 func IgnoreDefault() bool {
diff --git a/site/content/environment/_index.en.md b/site/content/environment/_index.en.md
index 0e1ea9f..f58829e 100644
--- a/site/content/environment/_index.en.md
+++ b/site/content/environment/_index.en.md
@@ -22,5 +22,13 @@ Sets the binary that mage will use to compile with (default is "go").
 
 ## MAGEFILE_IGNOREDEFAULT
 
-If set to 1 or true, will tell the compiled magefile to ignore the default
-target and print the list of targets when you run `mage`.
\ No newline at end of file
+If set to "1" or "true", tells the compiled magefile to ignore the default
+target and print the list of targets when you run `mage`.
+
+## MAGEFILE_HASHFAST
+
+If set to "1" or "true", tells mage to use a quick hash of magefiles to
+determine whether or not the magefile binary needs to be rebuilt. This results
+in faster run times (especially on Windows), but means that mage will fail to
+rebuild if a dependency has changed. To force a rebuild when you know or suspect
+a dependency has changed, run mage with the -f flag.
\ No newline at end of file


Test patch:
diff --git a/mage/main_test.go b/mage/main_test.go
index e4ac8b2..0b177f2 100644
--- a/mage/main_test.go
+++ b/mage/main_test.go
@@ -61,7 +61,7 @@ func testmain(m *testing.M) int {
 }
 
 func TestTransitiveDepCache(t *testing.T) {
-	cache, err := internal.OutputDebug("go", "env", "gocache")
+	cache, err := internal.OutputDebug("go", "env", "GOCACHE")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -108,6 +108,61 @@ func TestTransitiveDepCache(t *testing.T) {
 	}
 }
 
+func TestTransitiveHashFast(t *testing.T) {
+	cache, err := internal.OutputDebug("go", "env", "GOCACHE")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cache == "" {
+		t.Skip("skipping hashfast tests on go version without cache")
+	}
+
+	// Test that if we change a transitive dep, that we don't recompile.
+	// We intentionally run the first time without hashfast to ensure that
+	// we recompile the binary with the current code.
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	inv := Invocation{
+		Stderr: stderr,
+		Stdout: stdout,
+		Dir:    "testdata/transitiveDeps",
+		Args:   []string{"Run"},
+	}
+	code := Invoke(inv)
+	if code != 0 {
+		t.Fatalf("got code %v, err: %s", code, stderr)
+	}
+	expected := "woof\n"
+	if actual := stdout.String(); actual != expected {
+		t.Fatalf("expected %q but got %q", expected, actual)
+	}
+
+	// ok, so baseline, the generated and cached binary should do "woof"
+	// now change out the transitive dependency that does the output
+	// so that it produces different output.
+	if err := os.Rename("testdata/transitiveDeps/dep/dog.go", "testdata/transitiveDeps/dep/dog.notgo"); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Rename("testdata/transitiveDeps/dep/dog.notgo", "testdata/transitiveDeps/dep/dog.go")
+	if err := os.Rename("testdata/transitiveDeps/dep/cat.notgo", "testdata/transitiveDeps/dep/cat.go"); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Rename("testdata/transitiveDeps/dep/cat.go", "testdata/transitiveDeps/dep/cat.notgo")
+	stderr.Reset()
+	stdout.Reset()
+	inv.HashFast = true
+	code = Invoke(inv)
+	if code != 0 {
+		t.Fatalf("got code %v, err: %s", code, stderr)
+	}
+	// we should still get woof, even though the dependency was changed to
+	// return "meow", because we're only hashing the top level magefiles, not
+	// dependencies.
+	if actual := stdout.String(); actual != expected {
+		t.Fatalf("expected %q but got %q", expected, actual)
+	}
+}
+
 func TestListMagefilesMain(t *testing.T) {
 	buf := &bytes.Buffer{}
 	files, err := Magefiles("testdata/mixed_main_files", "", "", "go", buf, false)
diff --git a/mage/testdata/transitiveDeps/magefile.go b/mage/testdata/transitiveDeps/magefile.go
index 60ebb48..379d6ea 100644
--- a/mage/testdata/transitiveDeps/magefile.go
+++ b/mage/testdata/transitiveDeps/magefile.go
@@ -2,7 +2,7 @@
 
 package main
 
-import "./dep"
+import "github.com/magefile/mage/mage/testdata/transitiveDeps/dep"
 
 func Run() {
 	dep.Speak()