Repository: scikit-build/scikit-ci

Issue:
Introduce expclit step ordering and dependency
In the current implementation all steps have to be explicitly executed. There are no concept of dependencies and/or order.

Considering the implicit order of all `step(n)` is the following:

* `before_install` -> `step(1)`
* `install` -> `step(2)`
* `before_build` -> `step(3)`
* `build` -> `step(4)`
* `test` -> `step(5)`
* `after_test` -> `step(6)`


The proposal is to ensure that executing any `step(n)` using `ci step(n)` ensures that `step(n-1)` has been executed before.

Doing so will allow to update configuration like this one:

```yml
dependencies:
  override:
    - ci before_install
    - ci install

test:
  override:
    - ci before_build
    - ci build
    - ci test

deployment:
  master:
    branch: master
    commands:
      - ci after_test
```

into 

```yml
dependencies:
  override:
    - ci install

test:
  override:
    - ci test

deployment:
  master:
    branch: master
    commands:
      - ci after_test
```



Comments:

Patch:
diff --git a/appveyor.yml b/appveyor.yml
index 55b5f23..9d2d11c 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -31,13 +31,13 @@ install:
 
 build_script:
   - python -m ci before_build
-  - python -m ci build"
+  - python -m ci build
 
 test_script:
-  - python -m ci test"
+  - python -m ci test
 
 after_test:
-  - python -m ci after_test"
+  - python -m ci after_test
 # scikit-ci-yml.rst: end
 
 on_finish:
diff --git a/ci/__main__.py b/ci/__main__.py
index fc661ea..236b426 100644
--- a/ci/__main__.py
+++ b/ci/__main__.py
@@ -9,29 +9,25 @@ import ci
 import os
 
 
-class _OptionalChoices(argparse.Action):
-    """Custom action making a positional argument with choices optional.
-
-    Possible choices should be set with:
-    - ``default`` attribute set as a list
-    - ``nargs`` attribute set to ``'*'``
+class _OptionalStep(argparse.Action):
+    """Custom action making the ``step`` positional argument with choices
+    optional.
 
     Setting the ``choices`` attribute will fail with an *invalid choice* error.
 
     Adapted from http://stackoverflow.com/questions/8526675/python-argparse-optional-append-argument-with-choices/8527629#8527629
     """  # noqa: E501
-    def __call__(self, parser, namespace, values, option_string=None):
-        if values:
-            for value in values:
-                if value not in self.default:
-                    message = ("invalid choice: {0!r} (choose from {1})"
-                               .format(value,
-                                       ', '.join([repr(action)
-                                                  for action in
-                                                  self.default])))
+    def __call__(self, parser, namespace, value, option_string=None):
+        if value:
+            if value not in ci.STEPS:
+                message = ("invalid choice: {0!r} (choose from {1})"
+                           .format(value,
+                                   ', '.join([repr(action)
+                                              for action in
+                                              ci.STEPS])))
 
-                    raise argparse.ArgumentError(self, message)
-            setattr(namespace, self.dest, values)
+                raise argparse.ArgumentError(self, message)
+            setattr(namespace, self.dest, value)
 
 
 def main():
@@ -45,21 +41,29 @@ def main():
 
     parser = argparse.ArgumentParser(description=ci.__doc__)
     parser.add_argument(
-        "steps", type=str, nargs='*', default=ci.STEPS,
-        action=_OptionalChoices, metavar='STEP',
-        help="name of the steps to execute. "
+        "step", type=str, nargs='?', default=ci.STEPS[-1],
+        action=_OptionalStep, metavar='STEP',
+        help="name of the step to execute. "
              "Choose from: {}. "
-             "If not steps are specified, all are executed.".format(", ".join(
+             "If no step is specified, all are executed.".format(", ".join(
                 [repr(action) for action in ci.STEPS]))
     )
+    parser.add_argument(
+        "--force", action="store_true",
+        help="always execute the steps"
+    )
+    parser.add_argument(
+        "--without-deps", action="store_false",
+        help="do not execute dependent steps", dest='with_dependencies'
+    )
     parser.add_argument(
         "--version", action="version",
         version=version_str,
         help="display scikit-ci version and import information.")
     args = parser.parse_args()
 
-    for step in args.steps:
-        ci.execute_step(step)
+    ci.execute_step(
+        args.step, force=args.force, with_dependencies=args.with_dependencies)
 
 
 if __name__ == '__main__':
diff --git a/ci/driver.py b/ci/driver.py
index 5ce0f01..6713d1a 100644
--- a/ci/driver.py
+++ b/ci/driver.py
@@ -28,7 +28,7 @@ class DriverContext(object):
 
     def __exit__(self, exc_type, exc_value, traceback):
         if exc_type is None and exc_value is None and traceback is None:
-            self.driver.save_env()
+            self.driver.save_env(self.driver.env, self.env_file)
 
         self.driver.unload_env()
 
@@ -43,6 +43,13 @@ class Driver(object):
         print(" ".join(s))
         sys.stdout.flush()
 
+    @staticmethod
+    def read_env(env_file="env.json"):
+        if not os.path.exists(env_file):
+            return {}
+        with open(env_file) as _file:
+            return json.load(_file)
+
     def load_env(self, env_file="env.json"):
         if self.env is not None:
             self.unload_env()
@@ -52,16 +59,14 @@ class Driver(object):
         self._env_file = env_file
 
         if os.path.exists(self._env_file):
-            self.env.update(json.load(open(self._env_file)))
+            self.env.update(self.read_env(self._env_file))
 
         self.env = {str(k): str(v) for k, v in self.env.items()}
 
-    def save_env(self, env_file=None):
-        if env_file is None:
-            env_file = self._env_file
-
-        with open(env_file, "w") as env:
-            json.dump(self.env, env, indent=4)
+    @staticmethod
+    def save_env(env, env_file="env.json"):
+        with open(env_file, "w") as _file:
+            json.dump(env, _file, indent=4)
 
     def unload_env(self):
         self.env = None
@@ -266,14 +271,45 @@ class Driver(object):
             self.check_call(cmd.replace("\\\\", "\\\\\\\\"), env=self.env)
 
 
-def execute_step(step):
+def dependent_steps(step):
+    if step not in STEPS:  # pragma: no cover
+        raise KeyError("invalid step: {}".format(step))
+    step_index = STEPS.index(step)
+    if step_index == 0:
+        return []
+    return STEPS[0:step_index]
+
+
+def execute_step(step, force=False, with_dependencies=True):
 
     if not os.path.exists(SCIKIT_CI_CONFIG):  # pragma: no cover
         raise OSError(errno.ENOENT, "Couldn't find %s" % SCIKIT_CI_CONFIG)
 
     if step not in STEPS:  # pragma: no cover
-        raise KeyError("invalid stage: {}".format(step))
+        raise KeyError("invalid step: {}".format(step))
+
+    depends = dependent_steps(step)
+
+    # If forcing execution, remove SCIKIT_CI_<step> env. variables
+    if force:
+        env = Driver.read_env()
+        steps = [step]
+        if with_dependencies:
+            steps += depends
+        for _step in steps:
+            if 'SCIKIT_CI_%s' % _step.upper() in env:
+                del env['SCIKIT_CI_%s' % _step.upper()]
+        Driver.save_env(env)
+
+    # Skip step if it has already been executed
+    if 'SCIKIT_CI_%s' % step.upper() in Driver.read_env():
+        return
+
+    # Recursively execute dependent steps
+    if with_dependencies and depends:
+        execute_step(depends[-1], with_dependencies=with_dependencies)
 
     d = Driver()
     with d.env_context():
         d.execute_commands(step)
+        d.env['SCIKIT_CI_%s' % step.upper()] = '1'


Test patch:
diff --git a/tests/test_scikit_ci.py b/tests/test_scikit_ci.py
index 4b0c02f..bf957ea 100644
--- a/tests/test_scikit_ci.py
+++ b/tests/test_scikit_ci.py
@@ -9,8 +9,8 @@ import textwrap
 from ruamel.yaml.compat import ordereddict
 
 from . import captured_lines, display_captured_text, push_dir, push_env
-from ci.constants import SERVICES, SERVICES_ENV_VAR
-from ci.driver import Driver, execute_step
+from ci.constants import SERVICES, SERVICES_ENV_VAR, STEPS
+from ci.driver import Driver, dependent_steps, execute_step
 from ci.utils import current_service, current_operating_system
 
 """Indicate if the system has a Windows command line interpreter"""
@@ -55,6 +55,11 @@ def scikit_steps(tmpdir, service):
     ``(step, system, environment)`` for all supported steps.
     """
 
+    # Remove environment variables of the form 'SCIKIT_CI_<STEP>`
+    for step in STEPS:
+        if 'SCIKIT_CI_%s' % step.lower() in os.environ:
+            del os.environ['SCIKIT_CI_%s' % step.lower()]
+
     # By default, a service is associated with only one "implicit" operating
     # system.
     # Service supporting multiple operating system (e.g travis) should be
@@ -80,13 +85,7 @@ def scikit_steps(tmpdir, service):
         environment = os.environ
         enable_service(service, environment, system)
 
-        for step in [
-                'before_install',
-                'install',
-                'before_build',
-                'build',
-                'test',
-                'after_test']:
+        for step in STEPS:
 
             yield step, system, environment
 
@@ -153,13 +152,7 @@ def _generate_scikit_yml_content(service):
                 [textwrap.dedent(template_step).format(
                     what=step,
                     command_0=commands[0],
-                    command_1=commands[1]) for step in
-                 ['before_install',
-                  'install',
-                  'before_build',
-                  'build',
-                  'test',
-                  'after_test']
+                    command_1=commands[1]) for step in STEPS
                  ]
             )
     )
@@ -212,6 +205,16 @@ def test_driver(service, tmpdir, capfd):
                     raise
 
 
+def test_dependent_steps():
+    step = "before_install"
+    expected = []
+    assert dependent_steps(step) == expected
+
+    step = "build"
+    expected = ['before_install', 'install', 'before_build']
+    assert dependent_steps(step) == expected
+
+
 def test_shell_command(tmpdir, capfd):
 
     if platform.system().lower() == "windows":
@@ -356,7 +359,7 @@ def test_cli(tmpdir):
     assert tmpdir.join("install-done").exists()
 
 
-def test_cli_multiple_steps(tmpdir):
+def test_cli_force_and_without_deps(tmpdir):
     tmpdir.join('scikit-ci.yml').write(textwrap.dedent(
         r"""
         schema_version: "{version}"
@@ -376,17 +379,90 @@ def test_cli_multiple_steps(tmpdir):
     root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
     environment['PYTHONPATH'] = root
 
+    #
+    # Execute without --force
+    #
     subprocess.check_call(
-        "python -m ci before_install install",
+        "python -m ci install",
         shell=True,
         env=environment,
         stderr=subprocess.STDOUT,
         cwd=str(tmpdir)
     )
 
+    # Check that steps were executed
     assert tmpdir.join("before_install-done").exists()
     assert tmpdir.join("install-done").exists()
 
+    tmpdir.join("before_install-done").remove()
+    tmpdir.join("install-done").remove()
+
+    #
+    # Execute without --force
+    #
+    subprocess.check_call(
+        "python -m ci install",
+        shell=True,
+        env=environment,
+        stderr=subprocess.STDOUT,
+        cwd=str(tmpdir)
+    )
+
+    # Check that steps were NOT re-executed
+    assert not tmpdir.join("before_install-done").exists()
+    assert not tmpdir.join("install-done").exists()
+
+    #
+    # Execute with --force
+    #
+    subprocess.check_call(
+        "python -m ci install --force",
+        shell=True,
+        env=environment,
+        stderr=subprocess.STDOUT,
+        cwd=str(tmpdir)
+    )
+
+    # Check that steps were re-executed
+    assert tmpdir.join("before_install-done").exists()
+    assert tmpdir.join("install-done").exists()
+
+    tmpdir.join("before_install-done").remove()
+    tmpdir.join("install-done").remove()
+
+    #
+    # Execute with --force and --without-deps
+    #
+    subprocess.check_call(
+        "python -m ci install --force --without-deps",
+        shell=True,
+        env=environment,
+        stderr=subprocess.STDOUT,
+        cwd=str(tmpdir)
+    )
+
+    # Check that only specified step was re-executed
+    assert not tmpdir.join("before_install-done").exists()
+    assert tmpdir.join("install-done").exists()
+
+    tmpdir.join("install-done").remove()
+    tmpdir.join("env.json").remove()
+
+    #
+    # Execute with --without-deps
+    #
+    subprocess.check_call(
+        "python -m ci install --without-deps",
+        shell=True,
+        env=environment,
+        stderr=subprocess.STDOUT,
+        cwd=str(tmpdir)
+    )
+
+    # Check that only specified step was executed
+    assert not tmpdir.join("before_install-done").exists()
+    assert tmpdir.join("install-done").exists()
+
 
 def test_cli_execute_all_steps(tmpdir):
     tmpdir.join('scikit-ci.yml').write(textwrap.dedent(
@@ -699,3 +775,143 @@ def test_ci_name_reserved_environment_variable(tmpdir):
         failed = True
 
     assert failed
+
+
+def test_step_ordering_and_dependency(tmpdir):
+    quote = "" if HAS_COMSPEC else "\""
+    tmpdir.join('scikit-ci.yml').write(textwrap.dedent(
+        r"""
+        schema_version: "{version}"
+        before_install:
+          commands:
+            - "python -c \"with open('before_install', 'w') as file: file.write('')\""
+        install:
+          commands:
+            - "python -c \"with open('install', 'w') as file: file.write('')\""
+        before_build:
+          commands:
+            - "python -c \"with open('before_build', 'w') as file: file.write('')\""
+        build:
+          commands:
+            - "python -c \"with open('build', 'w') as file: file.write('')\""
+        test:
+          commands:
+            - "python -c \"exit(1)\""
+        after_test:
+          commands:
+            - "python -c \"with open('after_test', 'w') as file: file.write('')\""
+        """  # noqa: E501
+    ).format(quote=quote, version=SCHEMA_VERSION))
+    service = 'circle'
+
+    environment = dict(os.environ)
+    enable_service(service, environment)
+
+    with push_dir(str(tmpdir)), push_env(**environment):
+
+        execute_step("install")
+
+        #
+        # Check that steps `before_install` and `install` were executed
+        #
+        env = Driver.read_env()
+        assert env['SCIKIT_CI_BEFORE_INSTALL'] == '1'
+        assert env['SCIKIT_CI_INSTALL'] == '1'
+        assert tmpdir.join('before_install').exists()
+        assert tmpdir.join('install').exists()
+
+        # Remove files - This will to make sure the steps are not re-executed
+        tmpdir.join('before_install').remove()
+        tmpdir.join('install').remove()
+
+        # Check files have been removed
+        assert not tmpdir.join('before_install').exists()
+        assert not tmpdir.join('install').exists()
+
+        execute_step("build")
+
+        #
+        # Check that only `before_build` and `build` steps were executed
+        #
+        env = Driver.read_env()
+        assert env['SCIKIT_CI_BEFORE_INSTALL'] == '1'
+        assert env['SCIKIT_CI_INSTALL'] == '1'
+        assert env['SCIKIT_CI_BEFORE_BUILD'] == '1'
+        assert env['SCIKIT_CI_BUILD'] == '1'
+
+        assert not tmpdir.join('before_install').exists()
+        assert not tmpdir.join('install').exists()
+        assert tmpdir.join('before_build').exists()
+        assert tmpdir.join('build').exists()
+
+        # Remove files - This will to make sure the steps are not re-executed
+        tmpdir.join('before_build').remove()
+        tmpdir.join('build').remove()
+
+        failed = False
+        try:
+            execute_step("after_test")
+        except subprocess.CalledProcessError as e:
+            failed = "exit(1)" in e.cmd
+
+        #
+        # Check that `after_test` step was NOT executed. It should not be
+        # execute because `test` step is failing.
+        #
+        assert failed
+        env = Driver.read_env()
+        assert env['SCIKIT_CI_BEFORE_INSTALL'] == '1'
+        assert env['SCIKIT_CI_INSTALL'] == '1'
+        assert env['SCIKIT_CI_BEFORE_BUILD'] == '1'
+        assert env['SCIKIT_CI_BUILD'] == '1'
+        assert 'SCIKIT_CI_TEST' not in env
+        assert 'SCIKIT_CI_AFTER_TEST' not in env
+
+        assert not tmpdir.join('before_install').exists()
+        assert not tmpdir.join('install').exists()
+        assert not tmpdir.join('before_install').exists()
+        assert not tmpdir.join('install').exists()
+        assert not tmpdir.join('test').exists()
+        assert not tmpdir.join('after_test').exists()
+
+        #
+        # Check `force=True` works as expected
+        #
+        execute_step("install", force=True)
+
+        # Check that steps `before_install` and `install` were re-executed
+        env = Driver.read_env()
+        assert env['SCIKIT_CI_BEFORE_INSTALL'] == '1'
+        assert env['SCIKIT_CI_INSTALL'] == '1'
+        assert tmpdir.join('before_install').exists()
+        assert tmpdir.join('install').exists()
+
+        tmpdir.join('before_install').remove()
+        tmpdir.join('install').remove()
+
+        #
+        # Check `force=True` and `with_dependencies=True` work as expected
+        #
+        execute_step("install", force=True, with_dependencies=False)
+
+        # Check that only step `install` was re-executed
+        env = Driver.read_env()
+        assert env['SCIKIT_CI_BEFORE_INSTALL'] == '1'
+        assert env['SCIKIT_CI_INSTALL'] == '1'
+        assert not tmpdir.join('before_install').exists()
+        assert tmpdir.join('install').exists()
+
+        tmpdir.join('install').remove()
+
+        #
+        # Check `force=False` and `with_dependencies=True` work as expected
+        #
+        tmpdir.join('env.json').remove()
+        execute_step("install", with_dependencies=False)
+
+        # Check that only step `install` was executed
+        env = Driver.read_env()
+        assert 'SCIKIT_CI_BEFORE_INSTALL' not in env
+        assert env['SCIKIT_CI_INSTALL'] == '1'
+        assert not tmpdir.join('before_install').exists()
+        assert tmpdir.join('install').exists()