Repository: go-task/task
Pull request #656 (2022-01-15 03:46:23)

Add support for multi-level includes
This PR is in response to #390 -- This adds support for multi-level includes by rewiring `read.Taskfile` to be a recursive function that merges Taskfiles in depth first order. It also uses the new `read.ReaderNode` to keep track of included Taskfile parent nodes and checks for include cycles that could cause overflow issues.

I've also added some basic tests to confirm the new functionality. Happy to continue to refine this if any interest is shown in the feature. This would absolutely be a game changer for a lot of the projects I'm involved in.


Patch:
diff --git a/docs/usage.md b/docs/usage.md
index 780b1f27..50c39e2a 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -150,10 +150,6 @@ includes:
 > The included Taskfiles must be using the same schema version the main
 > Taskfile uses.
 
-> Also, for now included Taskfiles can't include other Taskfiles.
-> This was a deliberate decision to keep use and implementation simple.
-> If you disagree, open an GitHub issue and explain your use case. =)
-
 ### Optional includes
 
 Includes marked as optional will allow Task to continue execution as normal if
diff --git a/task.go b/task.go
index 17e5be10..21abcf79 100644
--- a/task.go
+++ b/task.go
@@ -106,7 +106,12 @@ func (e *Executor) Run(ctx context.Context, calls ...taskfile.Call) error {
 // Setup setups Executor's internal state
 func (e *Executor) Setup() error {
 	var err error
-	e.Taskfile, err = read.Taskfile(e.Dir, e.Entrypoint)
+	e.Taskfile, err = read.Taskfile(&read.ReaderNode{
+		Dir:        e.Dir,
+		Entrypoint: e.Entrypoint,
+		Parent:     nil,
+		Optional:   false,
+	})
 	if err != nil {
 		return err
 	}
diff --git a/taskfile/read/taskfile.go b/taskfile/read/taskfile.go
index ec7a3ab9..c0546741 100644
--- a/taskfile/read/taskfile.go
+++ b/taskfile/read/taskfile.go
@@ -15,29 +15,35 @@ import (
 )
 
 var (
-	// ErrIncludedTaskfilesCantHaveIncludes is returned when a included Taskfile contains includes
-	ErrIncludedTaskfilesCantHaveIncludes = errors.New("task: Included Taskfiles can't have includes. Please, move the include to the main Taskfile")
 	// ErrIncludedTaskfilesCantHaveDotenvs is returned when a included Taskfile contains dotenvs
 	ErrIncludedTaskfilesCantHaveDotenvs = errors.New("task: Included Taskfiles can't have dotenv declarations. Please, move the dotenv declaration to the main Taskfile")
 
 	defaultTaskfiles = []string{"Taskfile.yml", "Taskfile.yaml"}
 )
 
+type ReaderNode struct {
+	Dir        string
+	Entrypoint string
+	Optional   bool
+	Parent     *ReaderNode
+}
+
 // Taskfile reads a Taskfile for a given directory
 // Uses current dir when dir is left empty. Uses Taskfile.yml
 // or Taskfile.yaml when entrypoint is left empty
-func Taskfile(dir string, entrypoint string) (*taskfile.Taskfile, error) {
-	if dir == "" {
+func Taskfile(readerNode *ReaderNode) (*taskfile.Taskfile, error) {
+	if readerNode.Dir == "" {
 		d, err := os.Getwd()
 		if err != nil {
 			return nil, err
 		}
-		dir = d
+		readerNode.Dir = d
 	}
-	path, err := exists(filepath.Join(dir, entrypoint))
+	path, err := exists(filepath.Join(readerNode.Dir, readerNode.Entrypoint))
 	if err != nil {
 		return nil, err
 	}
+	readerNode.Entrypoint = filepath.Base(path)
 
 	t, err := readTaskfile(path)
 	if err != nil {
@@ -68,9 +74,8 @@ func Taskfile(dir string, entrypoint string) (*taskfile.Taskfile, error) {
 			return err
 		}
 		if !filepath.IsAbs(path) {
-			path = filepath.Join(dir, path)
+			path = filepath.Join(readerNode.Dir, path)
 		}
-
 		path, err = exists(path)
 		if err != nil {
 			if includedTask.Optional {
@@ -79,12 +84,23 @@ func Taskfile(dir string, entrypoint string) (*taskfile.Taskfile, error) {
 			return err
 		}
 
-		includedTaskfile, err := readTaskfile(path)
-		if err != nil {
+		includeReaderNode := &ReaderNode{
+			Dir:        filepath.Dir(path),
+			Entrypoint: filepath.Base(path),
+			Parent:     readerNode,
+			Optional:   includedTask.Optional,
+		}
+
+		if err := checkCircularIncludes(includeReaderNode); err != nil {
 			return err
 		}
-		if includedTaskfile.Includes.Len() > 0 {
-			return ErrIncludedTaskfilesCantHaveIncludes
+
+		includedTaskfile, err := Taskfile(includeReaderNode)
+		if err != nil {
+			if includedTask.Optional {
+				return nil
+			}
+			return err
 		}
 
 		if v >= 3.0 && len(includedTaskfile.Dotenv) > 0 {
@@ -94,12 +110,12 @@ func Taskfile(dir string, entrypoint string) (*taskfile.Taskfile, error) {
 		if includedTask.AdvancedImport {
 			for k, v := range includedTaskfile.Vars.Mapping {
 				o := v
-				o.Dir = filepath.Join(dir, includedTask.Dir)
+				o.Dir = filepath.Join(readerNode.Dir, includedTask.Dir)
 				includedTaskfile.Vars.Mapping[k] = o
 			}
 			for k, v := range includedTaskfile.Env.Mapping {
 				o := v
-				o.Dir = filepath.Join(dir, includedTask.Dir)
+				o.Dir = filepath.Join(readerNode.Dir, includedTask.Dir)
 				includedTaskfile.Env.Mapping[k] = o
 			}
 
@@ -120,7 +136,7 @@ func Taskfile(dir string, entrypoint string) (*taskfile.Taskfile, error) {
 	}
 
 	if v < 3.0 {
-		path = filepath.Join(dir, fmt.Sprintf("Taskfile_%s.yml", runtime.GOOS))
+		path = filepath.Join(readerNode.Dir, fmt.Sprintf("Taskfile_%s.yml", runtime.GOOS))
 		if _, err = os.Stat(path); err == nil {
 			osTaskfile, err := readTaskfile(path)
 			if err != nil {
@@ -170,3 +186,25 @@ func exists(path string) (string, error) {
 
 	return "", fmt.Errorf(`task: No Taskfile found in "%s". Use "task --init" to create a new one`, path)
 }
+
+func checkCircularIncludes(node *ReaderNode) error {
+	if node == nil {
+		return errors.New("task: failed to check for include cycle: node was nil")
+	}
+	if node.Parent == nil {
+		return errors.New("task: failed to check for include cycle: node.Parent was nil")
+	}
+	var curNode = node
+	var basePath = filepath.Join(node.Dir, node.Entrypoint)
+	for curNode.Parent != nil {
+		curNode = curNode.Parent
+		curPath := filepath.Join(curNode.Dir, curNode.Entrypoint)
+		if curPath == basePath {
+			return fmt.Errorf("task: include cycle detected between %s <--> %s",
+				curPath,
+				filepath.Join(node.Parent.Dir, node.Parent.Entrypoint),
+			)
+		}
+	}
+	return nil
+}


Test patch:
diff --git a/task_test.go b/task_test.go
index cb75bb00..2d3cc361 100644
--- a/task_test.go
+++ b/task_test.go
@@ -753,6 +753,35 @@ func TestIncludes(t *testing.T) {
 	tt.Run(t)
 }
 
+func TestIncludesMultiLevel(t *testing.T) {
+	tt := fileContentTest{
+		Dir:       "testdata/includes_multi_level",
+		Target:    "default",
+		TrimSpace: true,
+		Files: map[string]string{
+			"called_one.txt":   "one",
+			"called_two.txt":   "two",
+			"called_three.txt": "three",
+		},
+	}
+	tt.Run(t)
+}
+
+func TestIncludeCycle(t *testing.T) {
+	const dir = "testdata/includes_cycle"
+	expectedError := "task: include cycle detected between testdata/includes_cycle/Taskfile.yml <--> testdata/includes_cycle/one/two/Taskfile.yml"
+
+	var buff bytes.Buffer
+	e := task.Executor{
+		Dir:    dir,
+		Stdout: &buff,
+		Stderr: &buff,
+		Silent: true,
+	}
+
+	assert.EqualError(t, e.Setup(), expectedError)
+}
+
 func TestIncorrectVersionIncludes(t *testing.T) {
 	const dir = "testdata/incorrect_includes"
 	expectedError := "task: Import with additional parameters is only available starting on Taskfile version v3"
diff --git a/testdata/includes_cycle/Taskfile.yml b/testdata/includes_cycle/Taskfile.yml
new file mode 100644
index 00000000..2eee81b4
--- /dev/null
+++ b/testdata/includes_cycle/Taskfile.yml
@@ -0,0 +1,12 @@
+version: '3'
+
+includes:
+  'one': ./one/Taskfile.yml
+
+tasks:
+  default:
+    cmds:
+      - echo "called_dep" > called_dep.txt
+  level1:
+    cmds:
+      - echo "hello level 1"
diff --git a/testdata/includes_cycle/one/Taskfile.yml b/testdata/includes_cycle/one/Taskfile.yml
new file mode 100644
index 00000000..a8be0fd1
--- /dev/null
+++ b/testdata/includes_cycle/one/Taskfile.yml
@@ -0,0 +1,9 @@
+version: '3'
+
+includes:
+  'two': ./two/Taskfile.yml
+
+tasks:
+  level2:
+    cmds:
+      - echo "hello level 2"
diff --git a/testdata/includes_cycle/one/two/Taskfile.yml b/testdata/includes_cycle/one/two/Taskfile.yml
new file mode 100644
index 00000000..d849448a
--- /dev/null
+++ b/testdata/includes_cycle/one/two/Taskfile.yml
@@ -0,0 +1,9 @@
+version: '3'
+
+includes:
+  bad: "../../Taskfile.yml"
+
+tasks:
+  level3:
+    cmds:
+      - echo "hello level 3"
diff --git a/testdata/includes_multi_level/Taskfile.yml b/testdata/includes_multi_level/Taskfile.yml
new file mode 100644
index 00000000..3734d069
--- /dev/null
+++ b/testdata/includes_multi_level/Taskfile.yml
@@ -0,0 +1,11 @@
+version: '3'
+
+includes:
+  'one': ./one/
+
+tasks:
+  default:
+    cmds:
+      - task: one:default
+      - task: one:two:default
+      - task: one:two:three:default
diff --git a/testdata/includes_multi_level/called_one.txt b/testdata/includes_multi_level/called_one.txt
new file mode 100644
index 00000000..5626abf0
--- /dev/null
+++ b/testdata/includes_multi_level/called_one.txt
@@ -0,0 +1,1 @@
+one
diff --git a/testdata/includes_multi_level/called_three.txt b/testdata/includes_multi_level/called_three.txt
new file mode 100644
index 00000000..2bdf67ab
--- /dev/null
+++ b/testdata/includes_multi_level/called_three.txt
@@ -0,0 +1,1 @@
+three
diff --git a/testdata/includes_multi_level/called_two.txt b/testdata/includes_multi_level/called_two.txt
new file mode 100644
index 00000000..f719efd4
--- /dev/null
+++ b/testdata/includes_multi_level/called_two.txt
@@ -0,0 +1,1 @@
+two
diff --git a/testdata/includes_multi_level/one/Taskfile.yml b/testdata/includes_multi_level/one/Taskfile.yml
new file mode 100644
index 00000000..11ee8f74
--- /dev/null
+++ b/testdata/includes_multi_level/one/Taskfile.yml
@@ -0,0 +1,8 @@
+version: '3'
+
+includes:
+  'two': ./two/
+
+tasks:
+  default: echo one > called_one.txt
+  
\ No newline at end of file
diff --git a/testdata/includes_multi_level/one/two/Taskfile.yml b/testdata/includes_multi_level/one/two/Taskfile.yml
new file mode 100644
index 00000000..ed393064
--- /dev/null
+++ b/testdata/includes_multi_level/one/two/Taskfile.yml
@@ -0,0 +1,7 @@
+version: '3'
+
+includes:
+  'three': ./three/Taskfile.yml
+
+tasks:
+  default: echo two > called_two.txt
diff --git a/testdata/includes_multi_level/one/two/three/Taskfile.yml b/testdata/includes_multi_level/one/two/three/Taskfile.yml
new file mode 100644
index 00000000..b674fbc6
--- /dev/null
+++ b/testdata/includes_multi_level/one/two/three/Taskfile.yml
@@ -0,0 +1,5 @@
+version: '3'
+
+tasks:
+  default: echo three > called_three.txt
+  
\ No newline at end of file