Repository: iterative/dvc

Issue:
experiments: test_new_simple is flaky
Need to investigate flaky test, probably hitting some gitpython related race condition again

context:
https://travis-ci.com/github/iterative/dvc/jobs/372602915
https://travis-ci.com/github/iterative/dvc/jobs/372071149


Discussion:
Another failure on a [different test](https://travis-ci.com/github/iterative/dvc/jobs/373237624):
```
=================================== FAILURES ===================================
_____________________________ test_show_experiment _____________________________
[gw2] linux -- Python 3.7.1 /home/travis/virtualenv/python3.7.1/bin/python
tmp_dir = PosixTmpDir('/tmp/pytest-of-travis/pytest-0/popen-gw2/test_show_experiment0')
scm = Git: '/tmp/pytest-of-travis/pytest-0/popen-gw2/test_show_experiment0/.git'
dvc = Repo: '/tmp/pytest-of-travis/pytest-0/popen-gw2/test_show_experiment0'
    def test_show_experiment(tmp_dir, scm, dvc):
        tmp_dir.gen("copy.py", COPY_SCRIPT)
        tmp_dir.gen("params.yaml", "foo: 1")
        dvc.run(
            cmd="python copy.py params.yaml metrics.yaml",
            metrics_no_cache=["metrics.yaml"],
            params=["foo"],
            name="foo",
        )
        scm.add(["copy.py", "params.yaml", "metrics.yaml", "dvc.yaml", "dvc.lock"])
        scm.commit("baseline")
        baseline_rev = scm.get_rev()
        timestamp = datetime.fromtimestamp(
            scm.repo.rev_parse(baseline_rev).committed_date
        )
    
        dvc.reproduce(PIPELINE_FILE, experiment=True, params=["foo=2"])
        results = dvc.experiments.show()
    
        expected_baseline = {
            "metrics": {"metrics.yaml": {"foo": 1}},
            "params": {"params.yaml": {"foo": 1}},
            "queued": False,
            "timestamp": timestamp,
        }
        expected_params = {"foo": 2}
    
        assert set(results.keys()) == {"workspace", baseline_rev}
        experiments = results[baseline_rev]
        assert len(experiments) == 2
        for rev, exp in experiments.items():
            if rev == "baseline":
                assert exp == expected_baseline
            else:
                assert exp["metrics"]["metrics.yaml"] == expected_params
                assert exp["params"]["params.yaml"] == expected_params
>               assert exp["timestamp"] > timestamp
E               assert datetime.datetime(2020, 8, 18, 10, 1, 1) > datetime.datetime(2020, 8, 18, 10, 1, 1)
```

Could be a one-time failure though, but maybe this assumption about time should not even be here?
Happened again, but with a different message: https://travis-ci.com/github/iterative/dvc/jobs/375383416

```
self = BaselineMismatchError(<git.Commit "d31b10fcd1a8e1191e215399a2a978db245349d5">, '232a7cbd9b448ce97013eb3db486fc3e2a104b00')
rev = <git.Commit "d31b10fcd1a8e1191e215399a2a978db245349d5">
expected = '232a7cbd9b448ce97013eb3db486fc3e2a104b00'
    def __init__(self, rev, expected):
>       rev_str = f"{rev[:7]}" if rev is not None else "dangling commit"
E       TypeError: 'Commit' object is not subscriptable
```

I've narrowed down the issue to gitpython `Repo.is_dirty()` misbehaving, but still not sure of the actual cause. In the flaky runs, for some reason `is_dirty()` is incorrectly returning False.

Inserting this code into `experiments._commit`
```py
        if not self.scm.is_dirty():
            tree = self.exp_dvc.tree
            logger.debug("--- workspace params.yaml:")
            with tree.open("params.yaml") as fobj:
                data = fobj.read()
            logger.debug(f"{data}")

            rev = self.scm.get_rev()
            tree = self.scm.get_tree(rev)
            logger.debug("--- baseline params.yaml:")
            with tree.open(os.path.join(self.exp_dvc.root_dir, "params.yaml")) as fobj:
                data = fobj.read()
            logger.debug(f"{data}")

            logger.debug(f"non-dirty exp workspace @ '{rev}'")
            raise UnchangedExperimentError(self.scm.get_rev())
```

gives

```
2020-08-20 07:38:35,961 DEBUG: Commiting experiment '484f6348df10e5f532c32e264e5278730ff648f379791b54ecf134caf8582cac'
2020-08-20 07:38:35,996 DEBUG: --- workspace params.yaml:
2020-08-20 07:38:35,996 DEBUG: foo: 2
2020-08-20 07:38:35,998 DEBUG: --- baseline params.yaml:
2020-08-20 07:38:36,000 DEBUG: foo: 1
2020-08-20 07:38:36,001 DEBUG: non-dirty exp workspace @ '627722c61614e0af2aa6f0a4fdf7abd5b81961f5'
2020-08-20 07:38:36,003 DEBUG: Experiment 'f75f01e0ed462d7c07ffd8c16ae87632f19c6a49' identical to baseline '627722c61614e0af2aa6f0a4fdf7abd5b81961f5'
Reproduced experiment '627722c'.
```

so `params.yaml` should be dirty here


edit: the issue here was that `exp_dvc.tree.open("params.yaml")` opens `cwd / params.yaml` and not the experiments workspace `params.yaml`, `is_dirty()` was actually working properly
So the underlying issue is related to input collection when we populate the executor tmp dir. The gitpython index/tree doesn't always behave consistently for merge commits, so sometimes the uploaded tree did not contain the correct changes.

Abstracting the tmp dir input collection to use `git stash apply` and `upload`ing directly from the dirty experiments workspace (the same way we do output collection now) rather than relying on merge commit git trees.

Resolving patch:
diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py
--- a/dvc/repo/experiments/__init__.py
+++ b/dvc/repo/experiments/__init__.py
@@ -15,12 +15,13 @@
 
 from funcy import cached_property, first
 
-from dvc.exceptions import DownloadError, DvcException
+from dvc.exceptions import DownloadError, DvcException, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 from dvc.repo.experiments.executor import ExperimentExecutor, LocalExecutor
 from dvc.scm.git import Git
 from dvc.stage.serialize import to_lockfile
+from dvc.tree.repo import RepoTree
 from dvc.utils import dict_sha256, env2bool, relpath
 from dvc.utils.fs import remove
 
@@ -53,6 +54,8 @@ def __init__(self, rev):
 
 class BaselineMismatchError(DvcException):
     def __init__(self, rev, expected):
+        if hasattr(rev, "hexsha"):
+            rev = rev.hexsha
         rev_str = f"{rev[:7]}" if rev is not None else "dangling commit"
         super().__init__(
             f"Experiment derived from '{rev_str}', expected '{expected[:7]}'."
@@ -249,6 +252,7 @@ def _commit(self, exp_hash, check_exists=True, branch=True):
         """Commit stages as an experiment and return the commit SHA."""
         if not self.scm.is_dirty():
             raise UnchangedExperimentError(self.scm.get_rev())
+
         rev = self.scm.get_rev()
         exp_name = f"{rev[:7]}-{exp_hash}"
         if branch:
@@ -339,19 +343,26 @@ def reproduce(
                 for rev in revs
             }
 
-        # setup executors
+        logger.debug(
+            "Reproducing experiment revs '%s'",
+            ", ".join((rev[:7] for rev in to_run)),
+        )
+
+        # setup executors - unstash experiment, generate executor, upload
+        # contents of (unstashed) exp workspace to the executor tree
         executors = {}
         for rev, baseline_rev in to_run.items():
-            tree = self.scm.get_tree(rev)
-            repro_args, repro_kwargs = self._unpack_args(tree)
+            self._scm_checkout(baseline_rev)
+            self.scm.repo.git.stash("apply", rev)
+            repro_args, repro_kwargs = self._unpack_args()
             executor = LocalExecutor(
-                tree,
                 baseline_rev,
                 repro_args=repro_args,
                 repro_kwargs=repro_kwargs,
                 dvc_dir=self.dvc_dir,
                 cache_dir=self.repo.cache.local.cache_dir,
             )
+            self._collect_input(executor)
             executors[rev] = executor
 
         exec_results = self._reproduce(executors, **kwargs)
@@ -434,34 +445,56 @@ def _reproduce(self, executors: dict, jobs: Optional[int] = 1) -> dict:
 
         return result
 
+    def _collect_input(self, executor: ExperimentExecutor):
+        """Copy (upload) input from the experiments workspace to the executor
+        tree.
+        """
+        logger.debug("Collecting input for '%s'", executor.tmp_dir)
+        repo_tree = RepoTree(self.exp_dvc)
+        self._process(
+            executor.tree,
+            self.exp_dvc.tree,
+            executor.collect_files(self.exp_dvc.tree, repo_tree),
+        )
+
     def _collect_output(self, executor: ExperimentExecutor):
         """Copy (download) output from the executor tree into experiments
         workspace.
         """
-        from dvc.cache.local import _log_exceptions
-
         logger.debug("Collecting output from '%s'", executor.tmp_dir)
-        dest_tree = self.exp_dvc.tree
-        src_tree = executor.tree
+        self._process(
+            self.exp_dvc.tree,
+            executor.tree,
+            executor.collect_output(),
+            download=True,
+        )
+
+    @staticmethod
+    def _process(dest_tree, src_tree, collected_files, download=False):
+        from dvc.cache.local import _log_exceptions
 
         from_infos = []
         to_infos = []
         names = []
-        for from_info in executor.collect_output():
+        for from_info in collected_files:
             from_infos.append(from_info)
             fname = from_info.relative_to(src_tree.path_info)
             names.append(str(fname))
-            to_info = dest_tree.path_info / fname
             to_infos.append(dest_tree.path_info / fname)
-            logger.debug(f"from '{from_info}' to '{to_info}'")
         total = len(from_infos)
 
-        func = partial(
-            _log_exceptions(dest_tree.download, "download"),
-            dir_mode=dest_tree.dir_mode,
-            file_mode=dest_tree.file_mode,
-        )
-        with Tqdm(total=total, unit="file", desc="Downloading") as pbar:
+        if download:
+            func = partial(
+                _log_exceptions(src_tree.download, "download"),
+                dir_mode=dest_tree.dir_mode,
+                file_mode=dest_tree.file_mode,
+            )
+            desc = "Downloading"
+        else:
+            func = partial(_log_exceptions(dest_tree.upload, "upload"))
+            desc = "Uploading"
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
             func = pbar.wrap_fn(func)
             # TODO: parallelize this, currently --jobs for repro applies to
             # number of repro executors not download threads
@@ -469,7 +502,9 @@ def _collect_output(self, executor: ExperimentExecutor):
                 fails = sum(dl_executor.map(func, from_infos, to_infos, names))
 
         if fails:
-            raise DownloadError(fails)
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
 
     @scm_locked
     def checkout_exp(self, rev):
diff --git a/dvc/repo/experiments/executor.py b/dvc/repo/experiments/executor.py
--- a/dvc/repo/experiments/executor.py
+++ b/dvc/repo/experiments/executor.py
@@ -11,8 +11,6 @@
 from dvc.tree.base import BaseTree
 from dvc.tree.local import LocalTree
 from dvc.tree.repo import RepoTree
-from dvc.utils import relpath
-from dvc.utils.fs import copy_fobj_to_file, makedirs
 
 logger = logging.getLogger(__name__)
 
@@ -21,7 +19,6 @@ class ExperimentExecutor:
     """Base class for executing experiments in parallel.
 
     Args:
-        src_tree: source tree for this experiment.
         baseline_rev: baseline revision that this experiment is derived from.
 
     Optional keyword args:
@@ -29,8 +26,7 @@ class ExperimentExecutor:
         repro_kwargs: Keyword args to be passed into reproduce.
     """
 
-    def __init__(self, src_tree: BaseTree, baseline_rev: str, **kwargs):
-        self.src_tree = src_tree
+    def __init__(self, baseline_rev: str, **kwargs):
         self.baseline_rev = baseline_rev
         self.repro_args = kwargs.pop("repro_args", [])
         self.repro_kwargs = kwargs.pop("repro_kwargs", {})
@@ -73,27 +69,29 @@ def unpack_repro_args(path, tree=None):
 
 
 class LocalExecutor(ExperimentExecutor):
-    """Local machine exepriment executor."""
+    """Local machine experiment executor."""
+
+    def __init__(self, baseline_rev: str, **kwargs):
+        from dvc.repo import Repo
 
-    def __init__(self, src_tree: BaseTree, baseline_rev: str, **kwargs):
         dvc_dir = kwargs.pop("dvc_dir")
         cache_dir = kwargs.pop("cache_dir")
-        super().__init__(src_tree, baseline_rev, **kwargs)
+        super().__init__(baseline_rev, **kwargs)
         self.tmp_dir = TemporaryDirectory()
-        logger.debug("Init local executor in dir '%s'.", self.tmp_dir)
+
+        # init empty DVC repo (will be overwritten when input is uploaded)
+        Repo.init(root_dir=self.tmp_dir.name, no_scm=True)
+        logger.debug(
+            "Init local executor in dir '%s' with baseline '%s'.",
+            self.tmp_dir,
+            baseline_rev[:7],
+        )
         self.dvc_dir = os.path.join(self.tmp_dir.name, dvc_dir)
-        try:
-            for fname in src_tree.walk_files(src_tree.tree_root):
-                dest = self.path_info / relpath(fname, src_tree.tree_root)
-                if not os.path.exists(dest.parent):
-                    makedirs(dest.parent)
-                with src_tree.open(fname, "rb") as fobj:
-                    copy_fobj_to_file(fobj, dest)
-        except Exception:
-            self.tmp_dir.cleanup()
-            raise
         self._config(cache_dir)
         self._tree = LocalTree(self.dvc, {"url": self.dvc.root_dir})
+        # override default CACHE_MODE since files must be writable in order
+        # to run repro
+        self._tree.CACHE_MODE = 0o644
 
     def _config(self, cache_dir):
         local_config = os.path.join(self.dvc_dir, "config.local")
@@ -151,11 +149,13 @@ def filter_pipeline(stage):
 
     def collect_output(self) -> Iterable["PathInfo"]:
         repo_tree = RepoTree(self.dvc)
+        yield from self.collect_files(self.tree, repo_tree)
+
+    @staticmethod
+    def collect_files(tree: BaseTree, repo_tree: RepoTree):
         for fname in repo_tree.walk_files(repo_tree.root_dir, dvcfiles=True):
             if not repo_tree.isdvc(fname):
-                yield self.tree.path_info / fname.relative_to(
-                    repo_tree.root_dir
-                )
+                yield tree.path_info / fname.relative_to(repo_tree.root_dir)
 
     def cleanup(self):
         logger.debug("Removing tmpdir '%s'", self.tmp_dir)
diff --git a/dvc/tree/local.py b/dvc/tree/local.py
--- a/dvc/tree/local.py
+++ b/dvc/tree/local.py
@@ -326,7 +326,7 @@ def _upload(
         )
 
         self.protect(tmp_file)
-        os.rename(tmp_file, to_info)
+        os.replace(tmp_file, to_info)
 
     @staticmethod
     def _download(


Test patch:
diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py
--- a/tests/func/experiments/test_experiments.py
+++ b/tests/func/experiments/test_experiments.py
@@ -1,10 +1,6 @@
-import pytest
-
 from tests.func.test_repro_multistage import COPY_SCRIPT
 
 
-# https://github.com/iterative/dvc/issues/4401
-@pytest.mark.flaky(max_runs=3, min_passes=1)
 def test_new_simple(tmp_dir, scm, dvc, mocker):
     tmp_dir.gen("copy.py", COPY_SCRIPT)
     tmp_dir.gen("params.yaml", "foo: 1")