Repository: go-task/task
Pull request #818 (2022-07-22 03:06:15)

Internal tasks
## Problem

Building on the fantastic work introduced by #656, I have been refactoring some of my projects with a `taskfiles` subdirectory. This helps me manage my extemely long files much more effectively and also allows me to take full advantage of the namespacing features (`a:b:c`).

However, I've noticed that in my more complex projects, the `--list-all` output is now _extremely_ long, as some of my "utility" taskfiles are being included in multiple other taskfiles. I've provided a simplified example below:

```yaml
# Taskfile.yml
version: '3'

includes:
  install: ./taskfiles/Install.yml
  gen: ./taskfiles/Gen.yml
  # etc...
```

```yaml
# ./taskfiles/Install.yml
version: '3'

tasks:
  
  proto-tools:
    # ...
  mock-tools:
    # ...
  # etc...
```

```yaml
# ./taskfiles/Gen.yml
version: '3'

includes:
  install: ./Install.yml

tasks:
  
  protos:
    deps:
      - install:proto-tools
  mocks:
    deps:
      - install:mock-tools
  # etc...
```

```ini
# --list-all output
* install:proto-tools
* install:mock-tools
* gen:protos
* gen:mocks
* gen:install:proto-tools
* gen:install:mock-tools
# etc...
```

I'd like to be able to call `install:*` from the command line, which is fine. However, I _really_ don't need to be able to call `gen:install:*` directly and I definitely don't want it to appear in my nicely documented list of tasks when running `--list-all`. Imagine this scenario, but with an much more complex setup, with several layers of includes and the list of tasks can very quickly become 100s of lines long and lag out my terminal autocompletion 😞 

## Solution

This PR adds a boolean `internal` parameter to tasks which stops them from being callable on the command-line. It also stops the task from being output using `--list|--list-all`.

In addition to this, it solves the problem illustrated above by adding the same parameter to `includes`. This will mark all the included tasks as `internal`, thus removing them from the task list. Internal tasks can still be called by other tasks in the usual way.

My example would now looks something like this:

```yaml
# ./taskfiles/Gen.yml
version: '3'

includes:
  install:
    taskfile: ./Install.yml
    internal: true

tasks:
  
  protos:
    deps:
      - install:proto-tools
  mocks:
    deps:
      - install:mock-tools
  # etc...
```

```ini
# --list-all output
* install:proto-tools
* install:mock-tools
* gen:protos
* gen:mocks
# etc...
```


Patch:
diff --git a/docs/docs/usage.md b/docs/docs/usage.md
index e6dbe7fa..ed2b2285 100644
--- a/docs/docs/usage.md
+++ b/docs/docs/usage.md
@@ -194,6 +194,21 @@ tasks:
       - echo "This command can still be successfully executed if ./tests/Taskfile.yml does not exist"
 ```
 
+### Internal includes
+
+Includes marked as internal will set all the tasks of the included file to be
+internal as well (See the [Internal Tasks](#internal-tasks) section below). This is useful when including utility tasks that are not
+intended to be used directly by the user.
+
+```yaml
+version: '3'
+
+includes:
+  tests:
+    taskfile: ./taskfiles/Utils.yml
+    internal: true
+```
+
 ### Vars of included Taskfiles
 
 You can also specify variables when including a Taskfile. This may be useful
@@ -223,6 +238,30 @@ use the [default function](https://go-task.github.io/slim-sprig/defaults.html):
 
 :::
 
+## Internal Tasks
+
+Internal tasks are tasks that cannot be called directly by the user. They will
+not appear in the output when running `task --list|--list-all`. Other tasks may
+call internal tasks in the usual way. This is useful for creating reusable,
+function-like tasks that have no useful purpose on the command line.
+
+```yaml
+version: '3'
+
+tasks:
+
+  build-image-1:
+    cmds:
+      - task: build-image
+        vars:
+          DOCKER_IMAGE: image-1
+
+  build-image:
+    internal: true
+    cmds:
+      - docker build -t {{.DOCKER_IMAGE}} .
+```
+
 ## Task directory
 
 By default, tasks will be executed in the directory where the Taskfile is
diff --git a/errors.go b/errors.go
index e13ad9b3..14f3faee 100644
--- a/errors.go
+++ b/errors.go
@@ -20,6 +20,14 @@ func (err *taskNotFoundError) Error() string {
 	return fmt.Sprintf(`task: Task "%s" not found`, err.taskName)
 }
 
+type taskInternalError struct {
+	taskName string
+}
+
+func (err *taskInternalError) Error() string {
+	return fmt.Sprintf(`task: Task "%s" is internal`, err.taskName)
+}
+
 type TaskRunError struct {
 	taskName string
 	err      error
diff --git a/help.go b/help.go
index 0dc99718..85ce0ce7 100644
--- a/help.go
+++ b/help.go
@@ -52,7 +52,9 @@ func (e *Executor) printTasks(listAll bool) {
 func (e *Executor) allTaskNames() (tasks []*taskfile.Task) {
 	tasks = make([]*taskfile.Task, 0, len(e.Taskfile.Tasks))
 	for _, task := range e.Taskfile.Tasks {
-		tasks = append(tasks, task)
+		if !task.Internal {
+			tasks = append(tasks, task)
+		}
 	}
 	sort.Slice(tasks, func(i, j int) bool { return tasks[i].Task < tasks[j].Task })
 	return
@@ -61,7 +63,7 @@ func (e *Executor) allTaskNames() (tasks []*taskfile.Task) {
 func (e *Executor) tasksWithDesc() (tasks []*taskfile.Task) {
 	tasks = make([]*taskfile.Task, 0, len(e.Taskfile.Tasks))
 	for _, task := range e.Taskfile.Tasks {
-		if task.Desc != "" {
+		if !task.Internal && task.Desc != "" {
 			compiledTask, err := e.FastCompiledTask(taskfile.Call{Task: task.Task})
 			if err == nil {
 				task = compiledTask
@@ -92,7 +94,7 @@ func (e *Executor) ListTaskNames(allTasks bool) {
 	// create a string slice from all map values (*taskfile.Task)
 	s := make([]string, 0, len(e.Taskfile.Tasks))
 	for _, t := range e.Taskfile.Tasks {
-		if allTasks || t.Desc != "" {
+		if (allTasks || t.Desc != "") && !t.Internal {
 			s = append(s, strings.TrimRight(t.Task, ":"))
 		}
 	}
diff --git a/task.go b/task.go
index 90991c30..108cc775 100644
--- a/task.go
+++ b/task.go
@@ -64,11 +64,16 @@ type Executor struct {
 func (e *Executor) Run(ctx context.Context, calls ...taskfile.Call) error {
 	// check if given tasks exist
 	for _, c := range calls {
-		if _, ok := e.Taskfile.Tasks[c.Task]; !ok {
+		t, ok := e.Taskfile.Tasks[c.Task]
+		if !ok {
 			// FIXME: move to the main package
 			e.ListTasksWithDesc()
 			return &taskNotFoundError{taskName: c.Task}
 		}
+		if t.Internal {
+			e.ListTasksWithDesc()
+			return &taskInternalError{taskName: c.Task}
+		}
 	}
 
 	if e.Summary {
diff --git a/taskfile/included_taskfile.go b/taskfile/included_taskfile.go
index ec4ade08..fe83bd7d 100644
--- a/taskfile/included_taskfile.go
+++ b/taskfile/included_taskfile.go
@@ -16,6 +16,7 @@ type IncludedTaskfile struct {
 	Taskfile       string
 	Dir            string
 	Optional       bool
+	Internal       bool
 	AdvancedImport bool
 	Vars           *Vars
 	BaseDir        string // The directory from which the including taskfile was loaded; used to resolve relative paths
@@ -101,6 +102,7 @@ func (it *IncludedTaskfile) UnmarshalYAML(unmarshal func(interface{}) error) err
 		Taskfile string
 		Dir      string
 		Optional bool
+		Internal bool
 		Vars     *Vars
 	}
 	if err := unmarshal(&includedTaskfile); err != nil {
@@ -109,6 +111,7 @@ func (it *IncludedTaskfile) UnmarshalYAML(unmarshal func(interface{}) error) err
 	it.Taskfile = includedTaskfile.Taskfile
 	it.Dir = includedTaskfile.Dir
 	it.Optional = includedTaskfile.Optional
+	it.Internal = includedTaskfile.Internal
 	it.AdvancedImport = true
 	it.Vars = includedTaskfile.Vars
 	return nil
diff --git a/taskfile/merge.go b/taskfile/merge.go
index a5731c71..baebad0e 100644
--- a/taskfile/merge.go
+++ b/taskfile/merge.go
@@ -9,7 +9,7 @@ import (
 const NamespaceSeparator = ":"
 
 // Merge merges the second Taskfile into the first
-func Merge(t1, t2 *Taskfile, namespaces ...string) error {
+func Merge(t1, t2 *Taskfile, internal bool, namespaces ...string) error {
 	if t1.Version != t2.Version {
 		return fmt.Errorf(`task: Taskfiles versions should match. First is "%s" but second is "%s"`, t1.Version, t2.Version)
 	}
@@ -43,6 +43,8 @@ func Merge(t1, t2 *Taskfile, namespaces ...string) error {
 		// have serious side-effects in the future, since we're editing
 		// the original references instead of deep copying them.
 
+		v.Internal = v.Internal || internal
+
 		t1.Tasks[taskNameWithNamespace(k, namespaces...)] = v
 
 		for _, dep := range v.Deps {
diff --git a/taskfile/read/taskfile.go b/taskfile/read/taskfile.go
index 285df4b7..59d70cc1 100644
--- a/taskfile/read/taskfile.go
+++ b/taskfile/read/taskfile.go
@@ -78,6 +78,7 @@ func Taskfile(readerNode *ReaderNode) (*taskfile.Taskfile, error) {
 				Taskfile:       tr.Replace(includedTask.Taskfile),
 				Dir:            tr.Replace(includedTask.Dir),
 				Optional:       includedTask.Optional,
+				Internal:       includedTask.Internal,
 				AdvancedImport: includedTask.AdvancedImport,
 				Vars:           includedTask.Vars,
 				BaseDir:        includedTask.BaseDir,
@@ -147,7 +148,7 @@ func Taskfile(readerNode *ReaderNode) (*taskfile.Taskfile, error) {
 			}
 		}
 
-		if err = taskfile.Merge(t, includedTaskfile, namespace); err != nil {
+		if err = taskfile.Merge(t, includedTaskfile, includedTask.Internal, namespace); err != nil {
 			return err
 		}
 		return nil
@@ -163,7 +164,7 @@ func Taskfile(readerNode *ReaderNode) (*taskfile.Taskfile, error) {
 			if err != nil {
 				return nil, err
 			}
-			if err = taskfile.Merge(t, osTaskfile); err != nil {
+			if err = taskfile.Merge(t, osTaskfile, false); err != nil {
 				return nil, err
 			}
 		}
diff --git a/taskfile/task.go b/taskfile/task.go
index 12484f0b..e13b8169 100644
--- a/taskfile/task.go
+++ b/taskfile/task.go
@@ -20,6 +20,7 @@ type Task struct {
 	Env                  *Vars
 	Silent               bool
 	Interactive          bool
+	Internal             bool
 	Method               string
 	Prefix               string
 	IgnoreError          bool
@@ -63,6 +64,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		Env           *Vars
 		Silent        bool
 		Interactive   bool
+		Internal      bool
 		Method        string
 		Prefix        string
 		IgnoreError   bool `yaml:"ignore_error"`
@@ -85,6 +87,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	t.Env = task.Env
 	t.Silent = task.Silent
 	t.Interactive = task.Interactive
+	t.Internal = task.Internal
 	t.Method = task.Method
 	t.Prefix = task.Prefix
 	t.IgnoreError = task.IgnoreError
diff --git a/variables.go b/variables.go
index 80a232a3..fd2124f0 100644
--- a/variables.go
+++ b/variables.go
@@ -57,6 +57,7 @@ func (e *Executor) compiledTask(call taskfile.Call, evaluateShVars bool) (*taskf
 		Env:                  nil,
 		Silent:               origTask.Silent,
 		Interactive:          origTask.Interactive,
+		Internal:             origTask.Internal,
 		Method:               r.Replace(origTask.Method),
 		Prefix:               r.Replace(origTask.Prefix),
 		IgnoreError:          origTask.IgnoreError,


Test patch:
diff --git a/task_test.go b/task_test.go
index 216e3968..9b933dbb 100644
--- a/task_test.go
+++ b/task_test.go
@@ -929,6 +929,86 @@ func TestIncludesRelativePath(t *testing.T) {
 	assert.Contains(t, buff.String(), "testdata/includes_rel_path/common")
 }
 
+func TestIncludesInternal(t *testing.T) {
+	const dir = "testdata/internal_task"
+	tests := []struct {
+		name           string
+		task           string
+		expectedErr    bool
+		expectedOutput string
+	}{
+		{"included internal task via task", "task-1", false, "Hello, World!\n"},
+		{"included internal task via dep", "task-2", false, "Hello, World!\n"},
+		{
+			"included internal direct",
+			"included:task-3",
+			true,
+			"task: No tasks with description available. Try --list-all to list all tasks\n",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var buff bytes.Buffer
+			e := task.Executor{
+				Dir:    dir,
+				Stdout: &buff,
+				Stderr: &buff,
+				Silent: true,
+			}
+			assert.NoError(t, e.Setup())
+
+			err := e.Run(context.Background(), taskfile.Call{Task: test.task})
+			if test.expectedErr {
+				assert.Error(t, err, test.expectedErr)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, test.expectedOutput, buff.String())
+		})
+	}
+}
+
+func TestInternalTask(t *testing.T) {
+	const dir = "testdata/internal_task"
+	tests := []struct {
+		name           string
+		task           string
+		expectedErr    bool
+		expectedOutput string
+	}{
+		{"internal task via task", "task-1", false, "Hello, World!\n"},
+		{"internal task via dep", "task-2", false, "Hello, World!\n"},
+		{
+			"internal direct",
+			"task-3",
+			true,
+			"task: No tasks with description available. Try --list-all to list all tasks\n",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var buff bytes.Buffer
+			e := task.Executor{
+				Dir:    dir,
+				Stdout: &buff,
+				Stderr: &buff,
+				Silent: true,
+			}
+			assert.NoError(t, e.Setup())
+
+			err := e.Run(context.Background(), taskfile.Call{Task: test.task})
+			if test.expectedErr {
+				assert.Error(t, err, test.expectedErr)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, test.expectedOutput, buff.String())
+		})
+	}
+}
+
 func TestSupportedFileNames(t *testing.T) {
 	fileNames := []string{
 		"Taskfile.yml",
diff --git a/testdata/includes_internal/Taskfile.yml b/testdata/includes_internal/Taskfile.yml
new file mode 100644
index 00000000..64121323
--- /dev/null
+++ b/testdata/includes_internal/Taskfile.yml
@@ -0,0 +1,16 @@
+version: '3'
+
+includes:
+  included:
+    taskfile: Taskfile2.yml
+    internal: true
+
+tasks:
+
+  task-1:
+    cmds:
+      - task: included:default
+
+  task-2:
+    deps:
+      - included:default
diff --git a/testdata/includes_internal/Taskfile2.yml b/testdata/includes_internal/Taskfile2.yml
new file mode 100644
index 00000000..dce136f0
--- /dev/null
+++ b/testdata/includes_internal/Taskfile2.yml
@@ -0,0 +1,7 @@
+version: '3'
+
+tasks:
+
+  task-3:
+    cmds:
+      - echo "Hello, World!"
diff --git a/testdata/internal_task/Taskfile.yml b/testdata/internal_task/Taskfile.yml
new file mode 100644
index 00000000..8317cdaa
--- /dev/null
+++ b/testdata/internal_task/Taskfile.yml
@@ -0,0 +1,16 @@
+version: '3'
+
+tasks:
+
+  task-1:
+    cmds:
+      - task: task-3
+
+  task-2:
+    deps:
+      - task-3
+
+  task-3:
+    internal: true
+    cmds:
+      - echo "Hello, World!"