Repository: pydoit/doit

Issue:
specify target of a DelayedTask on command line
Since not all tasks are created before execution starts, it is required some special handling for target names of targets that have not been created yet.

See discussion on https://github.com/getnikola/nikola/issues/1562#issuecomment-70836094

General idea is that if a target is not found, before raising an error to the user try to load DelayedTasks (as done by command list) to look for the given `target` name.

Some considerations:

 1) as of now a DelayedTask creates an implicit `task_dep` for the task given in it's `executed` param. But this `task_dep` is not preserved when the DelayedTask is re-created. It should not only be preserved but all created tasks should include this `task_dep` because the guarantee that the dependent task was already executed wont exist anymore!

2) if the selected tasks for execution includes know tasks and targets they should be executed **before**, any DelayedTask is loaded to look for an unknown `target`. This will ensure that same command line will work nicely even if it is on its first execution.

<bountysource-plugin>
---
Want to back this issue? **[Post a bounty on it!](https://www.bountysource.com/issues/7952756-specify-target-of-a-delayedtask-on-command-line?utm_campaign=plugin&utm_content=tracker%2F1885485&utm_medium=issues&utm_source=github)** We accept bounties via [Bountysource](https://www.bountysource.com/?utm_campaign=plugin&utm_content=tracker%2F1885485&utm_medium=issues&utm_source=github).
</bountysource-plugin>

Discussion:

Patch:
diff --git a/doit/cmd_run.py b/doit/cmd_run.py
index 8a3693a..50e34b9 100644
--- a/doit/cmd_run.py
+++ b/doit/cmd_run.py
@@ -116,6 +116,18 @@ opt_pdb = {
     }
 
 
+# use ".*" as default regex for delayed tasks without explicitly specified regex
+opt_auto_delayed_regex = {
+    'name': 'auto_delayed_regex',
+    'short': '',
+    'long': 'auto-delayed-regex',
+    'type': bool,
+    'default': False,
+    'help':
+"""Uses the default regex ".*" for every delayed task loader for which no regex was explicitly defined"""
+}
+
+
 class Run(DoitCmdBase):
     doc_purpose = "run tasks"
     doc_usage = "[TASK/TARGET...]"
@@ -124,7 +136,8 @@ class Run(DoitCmdBase):
 
     cmd_options = (opt_always, opt_continue, opt_verbosity,
                    opt_reporter, opt_outfile, opt_num_process,
-                   opt_parallel_type, opt_pdb, opt_single)
+                   opt_parallel_type, opt_pdb, opt_single,
+                   opt_auto_delayed_regex)
 
 
     def __init__(self, **kwargs):
@@ -162,7 +175,7 @@ class Run(DoitCmdBase):
     def _execute(self, outfile,
                  verbosity=None, always=False, continue_=False,
                  reporter='console', num_process=0, par_type='process',
-                 single=False):
+                 single=False, auto_delayed_regex=False):
         """
         @param reporter:
                (str) one of provided reporters or ...
@@ -172,7 +185,7 @@ class Run(DoitCmdBase):
         """
         # get tasks to be executed
         # self.control is saved on instance to be used by 'auto' command
-        self.control = TaskControl(self.task_list)
+        self.control = TaskControl(self.task_list, auto_delayed_regex=auto_delayed_regex)
         self.control.process(self.sel_tasks)
 
         if single:
diff --git a/doit/control.py b/doit/control.py
index 6350b99..c60d688 100644
--- a/doit/control.py
+++ b/doit/control.py
@@ -26,9 +26,10 @@ class TaskControl(object):
                           Value: task_name
     """
 
-    def __init__(self, task_list):
+    def __init__(self, task_list, auto_delayed_regex=False):
         self.tasks = {}
         self.targets = {}
+        self.auto_delayed_regex = auto_delayed_regex
 
         # name of task in order to be executed
         # this the order as in the dodo file. the real execution
@@ -174,22 +175,51 @@ class TaskControl(object):
             # by task name
             if filter_ in self.tasks:
                 selected_task.append(filter_)
+                continue
+
             # by target
-            elif filter_ in self.targets:
+            if filter_ in self.targets:
                 selected_task.append(self.targets[filter_])
-            else:
-                # if can not find name check if it is a sub-task of a delayed
-                basename = filter_.split(':', 1)[0]
-                if basename in self.tasks:
-                    loader = self.tasks[basename].loader
-                    loader.basename = basename
-                    self.tasks[filter_] = Task(filter_, None, loader=loader)
-                    selected_task.append(filter_)
+                continue
+
+            # if can not find name check if it is a sub-task of a delayed
+            basename = filter_.split(':', 1)[0]
+            if basename in self.tasks:
+                loader = self.tasks[basename].loader
+                loader.basename = basename
+                self.tasks[filter_] = Task(filter_, None, loader=loader)
+                selected_task.append(filter_)
+                continue
+
+            # check if target matches any regex
+            import re
+            tasks = []
+            for task in list(self.tasks.values()):
+                if task.loader and (task.loader.target_regex or self.auto_delayed_regex):
+                    if re.match(task.loader.target_regex if task.loader.target_regex else '.*', filter_):
+                        tasks.append(task)
+            if len(tasks) > 0:
+                if len(tasks) == 1:
+                    task = tasks[0]
+                    loader = task.loader
+                    loader.basename = task.name
+                    name = '_regex_target_' + filter_
+                    self.tasks[name] = Task(name, None,
+                                            loader=loader,
+                                            file_dep=[filter_])
+                    selected_task.append(name)
                 else:
-                    msg = ('cmd `run` invalid parameter: "%s".' +
-                           ' Must be a task, or a target.\n' +
-                           'Type "doit list" to see available tasks')
-                    raise InvalidCommand(msg % filter_)
+                    name = '_regex_target_' + filter_
+                    self.tasks[name] = Task(name, None,
+                                            task_dep=[task.name for task in tasks],
+                                            file_dep=[filter_])
+                    selected_task.append(name)
+            else:
+                # not found
+                msg = ('cmd `run` invalid parameter: "%s".' +
+                       ' Must be a task, or a target.\n' +
+                       'Type "doit list" to see available tasks')
+                raise InvalidCommand(msg % filter_)
         return selected_task
 
 
@@ -416,6 +446,9 @@ class TaskDispatcher(object):
             basename = this_task.loader.basename or this_task.name
             new_tasks = generate_tasks(basename, ref(), ref.__doc__)
             TaskControl.set_implicit_deps(self.targets, new_tasks)
+            # check itself for implicit dep (used by regex_target)
+            TaskControl.add_implicit_task_dep(
+                self.targets, this_task, this_task.file_dep)
             for nt in new_tasks:
                 if not nt.loader:
                     nt.loader = DelayedLoaded
diff --git a/doit/loader.py b/doit/loader.py
index 1f3bcd4..123d0f4 100644
--- a/doit/loader.py
+++ b/doit/loader.py
@@ -98,10 +98,11 @@ def get_module(dodo_file, cwd=None, seek_parent=False):
 
 
 
-def create_after(executed=None):
+def create_after(executed=None, target_regex=None):
     """Annotate a task-creator function with delayed loader info"""
     def decorated(func):
-        func.doit_create_after = DelayedLoader(func, executed=executed)
+        func.doit_create_after = DelayedLoader(func, executed=executed,
+                                               target_regex=target_regex)
         return func
     return decorated
 
diff --git a/doit/reporter.py b/doit/reporter.py
index 4f5cbe3..18df356 100644
--- a/doit/reporter.py
+++ b/doit/reporter.py
@@ -53,7 +53,8 @@ class ConsoleReporter(object):
 
     def skip_uptodate(self, task):
         """skipped up-to-date task"""
-        self.write("-- %s\n" % task.title())
+        if task.actions and (task.name[0] != '_'):
+            self.write("-- %s\n" % task.title())
 
     def skip_ignore(self, task):
         """skipped ignored task"""
diff --git a/doit/task.py b/doit/task.py
index 7595877..3440123 100644
--- a/doit/task.py
+++ b/doit/task.py
@@ -31,9 +31,10 @@ class DelayedLoader(object):
                     the loader call the creator function
     :ivar basename: (str) basename used when creating tasks
     """
-    def __init__(self, creator, executed=None):
+    def __init__(self, creator, executed=None, target_regex=None):
         self.creator = creator
         self.task_dep = executed
+        self.target_regex = target_regex
         self.basename = None
         self.created = False
 


Test patch:
diff --git a/tests/test_control.py b/tests/test_control.py
index 4237acb..734b449 100644
--- a/tests/test_control.py
+++ b/tests/test_control.py
@@ -111,6 +111,43 @@ class TestTaskControlCmdOptions(object):
         assert control.tasks['taskY:foo'].loader.basename == 'taskY'
         assert control.tasks['taskY:foo'].loader is t2.loader
 
+    def test_filter_delayed_regex_single(self):
+        t1 = Task("taskX", None)
+        t2 = Task("taskY", None, loader=DelayedLoader(lambda: None, target_regex='a.*'))
+        t3 = Task("taskZ", None, loader=DelayedLoader(lambda: None, target_regex='b.*'))
+        t4 = Task("taskW", None, loader=DelayedLoader(lambda: None))
+        control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False)
+        l = control._filter_tasks(['abc'])
+        assert isinstance(t2.loader, DelayedLoader)
+        assert len(l) == 1
+        assert l[0] == '_regex_target_abc'
+        assert control.tasks[l[0]].file_dep == {'abc'}
+        assert control.tasks[l[0]].loader.basename == 'taskY'
+        assert control.tasks[l[0]].loader is t2.loader
+
+    def test_filter_delayed_regex_multiple(self):
+        t1 = Task("taskX", None)
+        t2 = Task("taskY", None, loader=DelayedLoader(lambda: None, target_regex='a.*'))
+        t3 = Task("taskZ", None, loader=DelayedLoader(lambda: None, target_regex='ab.'))
+        t4 = Task("taskW", None, loader=DelayedLoader(lambda: None))
+        control = TaskControl([t1, t2, t3, t4], auto_delayed_regex=False)
+        l = control._filter_tasks(['abc'])
+        assert len(l) == 1
+        assert l[0] == '_regex_target_abc'
+        assert control.tasks[l[0]].file_dep == {'abc'}
+        assert set(control.tasks[l[0]].task_dep) == {t2.name, t3.name}
+
+    def test_filter_delayed_regex_auto(self):
+        t1 = Task("taskX", None)
+        t2 = Task("taskY", None, loader=DelayedLoader(lambda: None, target_regex='a.*'))
+        t3 = Task("taskZ", None, loader=DelayedLoader(lambda: None))
+        control = TaskControl([t1, t2, t3], auto_delayed_regex=True)
+        l = control._filter_tasks(['abc'])
+        assert len(l) == 1
+        assert l[0] == '_regex_target_abc'
+        assert control.tasks[l[0]].file_dep == {'abc'}
+        assert set(control.tasks[l[0]].task_dep) == {t2.name, t3.name}
+
     # filter a non-existent task raises an error
     def testFilterWrongName(self):
         tc =  TaskControl(TASKS_SAMPLE)