Repository: iterative/dvc

Issue:
checkpoints: create final checkpoint on job termination
This was marked as a research TODO in the current code, creating a final checkpoint should resolve:

> checkpoint (model file) is not preserved if I cancel a job
> no issues with the model If I train till the end - it seems like the `--persist` was just ignored


Discussion:

Resolving patch:
diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py
--- a/dvc/repo/__init__.py
+++ b/dvc/repo/__init__.py
@@ -121,8 +121,7 @@ def __init__(
             self.tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs)
 
         self.config = Config(self.dvc_dir, tree=self.tree)
-        no_scm = self.config["core"].get("no_scm", False)
-        self.scm = scm if scm else SCM(self.root_dir, no_scm=no_scm)
+        self._scm = scm
 
         # used by RepoTree to determine if it should traverse subrepos
         self.subrepos = subrepos
@@ -158,6 +157,13 @@ def __init__(
         self.plots = Plots(self)
         self.params = Params(self)
 
+    @cached_property
+    def scm(self):
+        from dvc.scm import SCM
+
+        no_scm = self.config["core"].get("no_scm", False)
+        return self._scm if self._scm else SCM(self.root_dir, no_scm=no_scm)
+
     @property
     def tree(self):
         return self._tree
diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py
--- a/dvc/repo/experiments/__init__.py
+++ b/dvc/repo/experiments/__init__.py
@@ -543,7 +543,7 @@ def _reproduce(self, executors: dict, jobs: Optional[int] = 1) -> dict:
                         result[rev] = {exp_rev: exp_hash}
                 else:
                     logger.exception(
-                        "Failed to reproduce experiment '%s'", rev
+                        "Failed to reproduce experiment '%s'", rev[:7]
                     )
                 executor.cleanup()
 
@@ -574,7 +574,8 @@ def _checkpoint_callback(rev, executor, unchanged, stages):
                     result[rev] = {exp_rev: exp_hash}
 
             checkpoint_func = partial(_checkpoint_callback, rev, executor)
-            executor.reproduce(
+
+            exp_hash = executor.reproduce(
                 executor.dvc_dir,
                 cwd=executor.dvc.root_dir,
                 checkpoint=True,
@@ -582,14 +583,19 @@ def _checkpoint_callback(rev, executor, unchanged, stages):
                 **executor.repro_kwargs,
             )
 
-            # TODO: determine whether or not we should create a final
-            # checkpoint commit after repro is killed, or if we should only do
-            # it on explicit make_checkpoint() signals.
+            # NOTE: GitPython Repo instances cannot be re-used after
+            # process has received SIGINT or SIGTERM, so we need this hack
+            # to re-instantiate git instances after checkpoint runs. See:
+            # https://github.com/gitpython-developers/GitPython/issues/427
+            del self.repo.scm
+            del self.scm
 
-        # NOTE: our cached GitPython Repo instance cannot be re-used if the
-        # checkpoint run was interrupted via SIGINT, so we need this hack
-        # to create a new git repo instance after checkpoint runs.
-        del self.scm
+            # Create final checkpoint commit if needed
+            exp_rev = self._collect_and_commit(
+                rev, executor, exp_hash, checkpoint=True
+            )
+            if exp_rev not in result[rev]:
+                result[rev] = {exp_rev: exp_hash}
 
         return result
 
@@ -610,13 +616,11 @@ def _collect_and_commit(self, rev, executor, exp_hash, **kwargs):
             exp_rev = self._commit(
                 exp_hash, create_branch=create_branch, **kwargs
             )
-        except UnchangedExperimentError:
+        except UnchangedExperimentError as exc:
             logger.debug(
-                "Experiment '%s' identical to baseline '%s'",
-                rev,
-                executor.baseline_rev,
+                "Experiment '%s' identical to '%s'", rev, exc.rev,
             )
-            exp_rev = executor.baseline_rev
+            exp_rev = exc.rev
         return exp_rev
 
     def _collect_input(self, executor: ExperimentExecutor):


Test patch: