Repository: ansible/ansible-runner

Issue:
implement project directory execution isolation
The idea behind this is that when Runner starts execution it will copy the project directory and execute the runner invocation in that.

The issue that leads to this arises when multiple invocations of Runner are sharing the project directory... if they are managing the directory (such as checking out galaxy roles, etc) then they can end up stepping on each other's toes

Discussion:
softwarefactory-project-zuul[bot]: Build succeeded.

- [tox ](https://ansible.softwarefactory-project.io/logs/10/210/3896998b55242525109457b6b9bfcdc1bd2f2b5b/check/tox/694aa60/) : SUCCESS in 5m 35s

softwarefactory-project-zuul[bot]: Build succeeded.

- [tox ](https://ansible.softwarefactory-project.io/logs/10/210/d7cbadf3d67be73f87f09ae6391a4ca756b9b49f/check/tox/40d0cc1/) : SUCCESS in 5m 35s

chrismeyersfsu: I can't seem to get the feature to work.

```
tree /tmp/my/                                                             /tmp/my/
├── isolated_dir
├── private_data_dir
└── project_dir
    └── main.yml
```

`ansible-runner --directory-isolation-base-path /tmp/my/isolated_dir --project-dir /tmp/my/project_dir -p main.yml run /tmp/my/private_data_dir`



Patch:
diff --git a/ansible_runner/__main__.py b/ansible_runner/__main__.py
index 8e61dc3..200b91e 100644
--- a/ansible_runner/__main__.py
+++ b/ansible_runner/__main__.py
@@ -68,11 +68,16 @@ def role_manager(args):
         kwargs.update(private_data_dir=args.private_data_dir,
                       json_mode=args.json,
                       ignore_logging=False,
+                      project_dir=args.project_dir,
                       rotate_artifacts=args.rotate_artifacts)
         if args.artifact_dir:
             kwargs.artifact_dir = args.artifact_dir
 
-        project_path = os.path.join(args.private_data_dir, 'project')
+        if args.project_dir:
+            project_path = kwargs.project_dir = args.project_dir
+        else:
+            project_path = os.path.join(args.private_data_dir, 'project')
+
         project_exists = os.path.exists(project_path)
 
         env_path = os.path.join(args.private_data_dir, 'env')
@@ -191,9 +196,13 @@ def main(sys_args=None):
 
     parser.add_argument("--role-skip-facts", action="store_true", default=False,
                         help="Disable fact collection when executing a role directly")
+
     parser.add_argument("--artifact-dir",
                         help="Optional Path for the artifact root directory, by default it is located inside the private data dir")
 
+    parser.add_argument("--project-dir",
+                        help="Optional Path for the location of the playbook content directory, by default this is 'project' inside the private data dir")
+
     parser.add_argument("--inventory",
                         help="Override the default inventory location in private_data_dir")
 
@@ -235,6 +244,9 @@ def main(sys_args=None):
     parser.add_argument("--process-isolation-ro-paths", dest='process_isolation_ro_paths',
                         help="List of paths on the system that should be exposed to the playbook run as read-only")
 
+    parser.add_argument("--directory-isolation-base-path", dest='directory_isolation_base_path',
+                        help="Copies the project directory to a location in this directory to prevent multiple simultaneous executions from conflicting")
+
     args = parser.parse_args(sys_args)
 
     output.configure()
@@ -294,13 +306,15 @@ def main(sys_args=None):
                                    ignore_logging=False,
                                    json_mode=args.json,
                                    inventory=args.inventory,
+                                   project_dir=args.project_dir,
                                    roles_path=[args.roles_path] if args.roles_path else None,
                                    process_isolation=args.process_isolation,
                                    process_isolation_executable=args.process_isolation_executable,
                                    process_isolation_path=args.process_isolation_path,
                                    process_isolation_hide_paths=args.process_isolation_hide_paths,
                                    process_isolation_show_paths=args.process_isolation_show_paths,
-                                   process_isolation_ro_paths=args.process_isolation_ro_paths)
+                                   process_isolation_ro_paths=args.process_isolation_ro_paths,
+                                   directory_isolation_base_path=args.directory_isolation_base_path)
                 if args.cmdline:
                     run_options['cmdline'] = args.cmdline
 
diff --git a/ansible_runner/interface.py b/ansible_runner/interface.py
index 032533d..dfe81f4 100644
--- a/ansible_runner/interface.py
+++ b/ansible_runner/interface.py
@@ -105,7 +105,8 @@ def run(**kwargs):
     :param limit: Matches ansible's ``--limit`` parameter to further constrain the inventory to be used
     :param verbosity: Control how verbose the output of ansible-playbook is
     :param quiet: Disable all output
-    :param artifact_dir: The path to the directory where artifacts should live
+    :param artifact_dir: The path to the directory where artifacts should live, this defaults to 'artifacts' under the private data dir
+    :param project_dir: The path to the playbook content, this defaults to 'project' within the private data dir
     :param rotate_artifacts: Keep at most n artifact directories, disable with a value of 0 which is the default
     :param event_handler: An optional callback that will be invoked any time an event is received by Runner itself
     :param cancel_callback: An optional callback that can inform runner to cancel (returning True) or not (returning False)
@@ -117,6 +118,8 @@ def run(**kwargs):
     :param process_isolation_hide_paths: A path or list of paths on the system that should be hidden from the playbook run.
     :param process_isolation_show_paths: A path or list of paths on the system that should be exposed to the playbook run.
     :param process_isolation_ro_paths: A path or list of paths on the system that should be exposed to the playbook run as read-only.
+    :param directory_isolation_base_path: An optional path will be used as the base path to create a temp directory, the project contents will be
+                                          copied to this location which will then be used as the working directory during playbook execution.
     :param fact_cache: A string that will be used as the name for the subdirectory of the fact cache in artifacts directory.
                        This is only used for 'jsonfile' type fact caches.
     :param fact_cache_type: A string of the type of fact cache to use.  Defaults to 'jsonfile'.
@@ -131,6 +134,7 @@ def run(**kwargs):
     :type settings: dict
     :type ssh_key: str
     :type artifact_dir: str
+    :type project_dir: str
     :type rotate_artifacts: int
     :type cmdline: str
     :type quiet: bool
@@ -145,6 +149,7 @@ def run(**kwargs):
     :type process_isolation_hide_paths: str or list
     :type process_isolation_show_paths: str or list
     :type process_isolation_ro_paths: str or list
+    :type directory_isolation_base_path: str
     :type fact_cache: str
     :type fact_cache_type: str
 
diff --git a/ansible_runner/runner.py b/ansible_runner/runner.py
index 619cdcb..2304ab9 100644
--- a/ansible_runner/runner.py
+++ b/ansible_runner/runner.py
@@ -5,6 +5,7 @@ import time
 import json
 import errno
 import signal
+import shutil
 import codecs
 import collections
 
@@ -182,6 +183,8 @@ class Runner(object):
                 os.close(os.open(artifact_path, os.O_CREAT, stat.S_IRUSR | stat.S_IWUSR))
             with open(artifact_path, 'w') as f:
                 f.write(str(data))
+        if self.config.directory_isolation_path and self.config.directory_isolation_cleanup:
+            shutil.rmtree(self.config.directory_isolation_path)
         if self.finished_callback is not None:
             try:
                 self.finished_callback(self)
diff --git a/ansible_runner/runner_config.py b/ansible_runner/runner_config.py
index ac37f67..3fbd904 100644
--- a/ansible_runner/runner_config.py
+++ b/ansible_runner/runner_config.py
@@ -30,6 +30,8 @@ try:
 except ImportError:
     from collections import Mapping
 
+from distutils.dir_util import copy_tree
+
 from six import iteritems, string_types
 
 from ansible_runner import output
@@ -68,7 +70,8 @@ class RunnerConfig(object):
                  rotate_artifacts=0, host_pattern=None, binary=None, extravars=None, suppress_ansible_output=False,
                  process_isolation=False, process_isolation_executable=None, process_isolation_path=None,
                  process_isolation_hide_paths=None, process_isolation_show_paths=None, process_isolation_ro_paths=None,
-                 tags=None, skip_tags=None, fact_cache_type='jsonfile', fact_cache=None):
+                 tags=None, skip_tags=None, fact_cache_type='jsonfile', fact_cache=None, project_dir=None,
+                 directory_isolation_base_path=None):
         self.private_data_dir = os.path.abspath(private_data_dir)
         self.ident = ident
         self.json_mode = json_mode
@@ -94,6 +97,11 @@ class RunnerConfig(object):
         self.process_isolation_hide_paths = process_isolation_hide_paths
         self.process_isolation_show_paths = process_isolation_show_paths
         self.process_isolation_ro_paths = process_isolation_ro_paths
+        self.directory_isolation_path = directory_isolation_base_path
+        if not project_dir:
+            self.project_dir = os.path.join(self.private_data_dir, 'project')
+        else:
+            self.project_dir = project_dir
         self.verbosity = verbosity
         self.quiet = quiet
         self.suppress_ansible_output = suppress_ansible_output
@@ -124,6 +132,12 @@ class RunnerConfig(object):
             raise ConfigurationError("Only one of playbook and module options are allowed")
         if not os.path.exists(self.artifact_dir):
             os.makedirs(self.artifact_dir)
+        if self.directory_isolation_path is not None:
+            self.directory_isolation_path = tempfile.mkdtemp(prefix='runner_di_', dir=self.directory_isolation_path)
+            if os.path.exists(self.project_dir):
+                output.debug("Copying directory tree from {} to {} for working directory isolation".format(self.project_dir,
+                                                                                                           self.directory_isolation_path))
+                copy_tree(self.project_dir, self.directory_isolation_path, preserve_symlinks=True)
 
         self.prepare_inventory()
         self.prepare_env()
@@ -219,11 +233,15 @@ class RunnerConfig(object):
 
         self.pexpect_use_poll = self.settings.get('pexpect_use_poll', True)
         self.suppress_ansible_output = self.settings.get('suppress_ansible_output', self.quiet)
+        self.directory_isolation_cleanup = bool(self.settings.get('directory_isolation_cleanup', True))
 
-        if 'AD_HOC_COMMAND_ID' in self.env or not os.path.exists(os.path.join(self.private_data_dir, 'project')):
+        if 'AD_HOC_COMMAND_ID' in self.env or not os.path.exists(self.project_dir):
             self.cwd = self.private_data_dir
         else:
-            self.cwd = os.path.join(self.private_data_dir, 'project')
+            if self.directory_isolation_path is not None:
+                self.cwd = self.directory_isolation_path
+            else:
+                self.cwd = self.project_dir
 
         if 'fact_cache' in self.settings:
             if 'fact_cache_type' in self.settings:
@@ -369,7 +387,10 @@ class RunnerConfig(object):
 
         if 'ansible-playbook' in args[0]:
             # playbook runs should cwd to the SCM checkout dir
-            new_args.extend(['--chdir', os.path.join(self.private_data_dir, 'project')])
+            if self.directory_isolation_path is not None:
+                new_args.extend(['--chdir', self.directory_isolation_path])
+            else:
+                new_args.extend(['--chdir', self.project_dir])
         else:
             # ad-hoc runs should cwd to the root of the private data dir
             new_args.extend(['--chdir', self.private_data_dir])
diff --git a/docs/standalone.rst b/docs/standalone.rst
index cb1868a..4716b25 100644
--- a/docs/standalone.rst
+++ b/docs/standalone.rst
@@ -85,6 +85,17 @@ list of arguments accepted by ``ansible-runner``::
 
   $ ansible-runner --help
 
+Running with Directory Isolation
+--------------------------------
+
+If you need to be able to execute multiple tasks in parallel that might conflict with each other or if you want to make sure a single invocation of
+Ansible/Runner doesn't pollute or overwrite the playbook content you can give a base path::
+
+  $ ansible-runner --directory-isoltaion-base-path /tmp/runner
+
+**Runner** will copy the project directory to a temporary directory created under that path, set it as the working directory, and execute from that location.
+After running that temp directory will be cleaned up and removed.
+
 Outputting json (raw event data) to the console instead of normal output
 ------------------------------------------------------------------------
 


Test patch:
diff --git a/test/unit/test_runner_config.py b/test/unit/test_runner_config.py
index 6e6bb0c..0ad562d 100644
--- a/test/unit/test_runner_config.py
+++ b/test/unit/test_runner_config.py
@@ -65,6 +65,13 @@ def test_runner_config_init_with_ident():
     assert isinstance(rc.loader, ArtifactLoader)
 
 
+def test_runner_config_project_dir():
+    rc = RunnerConfig('/', project_dir='/another/path')
+    assert rc.project_dir == '/another/path'
+    rc = RunnerConfig('/')
+    assert rc.project_dir == '/project'
+
+
 def test_prepare_environment_vars_only_strings():
     rc = RunnerConfig(private_data_dir="/")
 
@@ -169,6 +176,15 @@ def test_prepare_env_defaults():
         assert rc.cwd == '/project'
 
 
+def test_prepare_env_directory_isolation():
+    with patch('os.path.exists') as path_exists:
+        path_exists.return_value=True
+        rc = RunnerConfig('/')
+        rc.directory_isolation_path = '/tmp/foo'
+        rc.prepare_env()
+        assert rc.cwd == '/tmp/foo'
+
+
 def test_prepare_inventory():
     rc = RunnerConfig(private_data_dir='/')
     rc.prepare_inventory()
@@ -388,6 +404,30 @@ def test_process_isolation_defaults():
     ]
 
 
+@patch('os.makedirs', return_value=True)
+@patch('shutil.copytree', return_value=True)
+@patch('tempfile.mkdtemp', return_value="/tmp/dirisolation/foo")
+@patch('os.chmod', return_value=True)
+def test_process_isolation_and_directory_isolation(mock_makedirs, mock_copytree, mock_mkdtemp, mock_chmod):
+    rc = RunnerConfig('/')
+    rc.artifact_dir = '/tmp/artifacts'
+    rc.directory_isolation_path = '/tmp/dirisolation'
+    rc.playbook = 'main.yaml'
+    rc.command = 'ansible-playbook'
+    rc.process_isolation = True
+    rc.prepare()
+
+    assert rc.command == [
+        'bwrap',
+        '--unshare-pid',
+        '--dev-bind', '/', '/',
+        '--proc', '/proc',
+        '--bind', '/', '/',
+        '--chdir', '/tmp/dirisolation/foo',
+        'ansible-playbook', '-i', '/inventory', 'main.yaml',
+    ]
+
+
 def test_process_isolation_settings():
     rc = RunnerConfig('/')
     rc.artifact_dir = '/tmp/artifacts'
@@ -401,8 +441,8 @@ def test_process_isolation_settings():
     rc.process_isolation_path = '/tmp'
 
     with patch('os.path.exists') as path_exists:
-        rc.prepare()
         path_exists.return_value=True
+        rc.prepare()
 
     assert rc.command[0:7] == [
         'not_bwrap',