Repository: Epistimio/orion

Issue:
EVC crashes when main script is not found on file system
If the path to main script in a previous experiment cannot be found, attempting to branch from it will make the EVC crash. The parent experiment should be loaded with a parser using option `allow_non_existing_files=True`
```
Traceback (most recent call last):
  File "/path/orion", line 8, in <module>
    sys.exit(main())
  File "/path//lib/python3.8/site-packages/orion/core/cli/__init__.py", line 39, in main
    return orion_parser.execute(argv)
  File "/path//lib/python3.8/site-packages/orion/core/cli/base.py", line 94, in execute
    returncode = function(args)
  File "/path/lib/python3.8/site-packages/orion/core/cli/hunt.py", line 83, in main
    experiment = experiment_builder.build_from_args(args)
  File "/path/lib/python3.8/site-packages/orion/core/io/experiment_builder.py", line 729, in build_from_args
    return build(**cmd_config)
  File "/path/lib/python3.8/site-packages/orion/core/io/experiment_builder.py", line 204, in build
    conflicts = _get_conflicts(experiment, branching)
  File "/path/lib/python3.8/site-packages/orion/core/io/experiment_builder.py", line 659, in _get_conflicts
    conflicts = detect_conflicts(
  File "/path/lib/python3.8/site-packages/orion/core/evc/conflicts.py", line 111, in detect_conflicts
    for conflict in conflict_class.detect(old_config, new_config, branching):
  File "/path/lib/python3.8/site-packages/orion/core/evc/conflicts.py", line 1350, in detect
    old_nameless_args = cls.get_nameless_args(old_config, **branching_config)
  File "/path/lib/python3.8/site-packages/orion/core/evc/conflicts.py", line 1326, in get_nameless_args
    parser.set_state_dict(config["metadata"]["parser"])
  File "/path/lib/python3.8/site-packages/orion/core/io/orion_cmdline_parser.py", line 136, in set_state_dict
    self.infer_user_script(self.parser.format(self.parser.arguments))
  File "/path/lib/python3.8/site-packages/orion/core/io/orion_cmdline_parser.py", line 179, in infer_user_script
    raise OSError(
FileNotFoundError: [Errno 2] The path specified for the script does not exist: '/path/main.py'
```
@mgermain

Discussion:

Patch:
diff --git a/src/orion/core/evc/conflicts.py b/src/orion/core/evc/conflicts.py
index 93772d98..90aa4361 100644
--- a/src/orion/core/evc/conflicts.py
+++ b/src/orion/core/evc/conflicts.py
@@ -1318,7 +1318,7 @@ class CommandLineConflict(Conflict):
         log.debug("User script config: %s", user_script_config)
         log.debug("Non monitored arguments: %s", non_monitored_arguments)
 
-        parser = OrionCmdlineParser(user_script_config)
+        parser = OrionCmdlineParser(user_script_config, allow_non_existing_files=True)
         parser.set_state_dict(config["metadata"]["parser"])
         priors = parser.priors_to_normal()
         nameless_keys = set(parser.parser.arguments.keys()) - set(priors.keys())
@@ -1478,7 +1478,7 @@ class ScriptConfigConflict(Conflict):
         if user_script_config is None:
             user_script_config = orion.core.config.worker.user_script_config
 
-        parser = OrionCmdlineParser(user_script_config)
+        parser = OrionCmdlineParser(user_script_config, allow_non_existing_files=True)
         parser.set_state_dict(config["metadata"]["parser"])
 
         nameless_config = dict(


Test patch:
diff --git a/tests/functional/branching/test_branching.py b/tests/functional/branching/test_branching.py
index fefa246c..88d0dcfb 100644
--- a/tests/functional/branching/test_branching.py
+++ b/tests/functional/branching/test_branching.py
@@ -5,6 +5,7 @@
 import os
 
 import pytest
+import yaml
 
 import orion.core.cli
 import orion.core.io.experiment_builder as experiment_builder
@@ -295,7 +296,7 @@ def init_full_x_new_algo(init_full_x):
 
 @pytest.fixture
 def init_full_x_new_cli(init_full_x):
-    """Remove z from full x full z and give a default value of 4"""
+    """Change commandline call"""
     name = "full_x"
     branch = "full_x_new_cli"
     orion.core.cli.main(
@@ -341,6 +342,37 @@ def init_full_x_ignore_cli(init_full_x):
     orion.core.cli.main("insert -n {name} script -x=-1.2".format(name=name).split(" "))
 
 
+@pytest.fixture
+def init_full_x_new_config(init_full_x, tmp_path):
+    """Add configuration script"""
+    name = "full_x"
+    branch = "full_x_new_config"
+
+    config_file = tmp_path / "config.yaml"
+    config_file.write_text(
+        yaml.dump(
+            {"new_arg": "some-value", "y": "orion~uniform(-10, 10, default_value=0)"}
+        )
+    )
+
+    orion.core.cli.main(
+        (
+            "hunt --init-only -n {branch} --branch-from {name} "
+            "--cli-change-type noeffect "
+            "--config-change-type unsure "
+            "./black_box_new.py -x~uniform(-10,10) --config {config_file}"
+        )
+        .format(name=name, branch=branch, config_file=config_file)
+        .split(" ")
+    )
+    orion.core.cli.main(
+        "insert -n {branch} script -x=1.2 -y=2".format(branch=branch).split(" ")
+    )
+    orion.core.cli.main(
+        "insert -n {branch} script -x=-1.2 -y=3".format(branch=branch).split(" ")
+    )
+
+
 @pytest.fixture
 def init_entire(
     init_half_x_full_y,  # 1.1.1
@@ -906,6 +938,139 @@ def test_no_cli_no_branching():
     assert error_code == 0
 
 
+def test_new_script(init_full_x, monkeypatch):
+    """Test that experiment can branch with new script path even if previous is not present"""
+
+    name = "full_x"
+    experiment = experiment_builder.load(name=name)
+
+    # Mess with DB to change script path
+    metadata = experiment.metadata
+    metadata["user_script"] = "oh_oh_idontexist.py"
+    metadata["user_args"][0] = "oh_oh_idontexist.py"
+    metadata["parser"]["parser"]["arguments"][0][1] = "oh_oh_idontexist.py"
+    get_storage().update_experiment(experiment, metadata=metadata)
+
+    orion.core.cli.main(
+        (
+            "hunt --init-only -n {name} --config orion_config.yaml ./black_box.py "
+            "-x~uniform(-10,10) --some-new args"
+        )
+        .format(name=name)
+        .split(" ")
+    )
+
+    new_experiment = experiment_builder.load(name=name)
+    assert new_experiment.version == experiment.version + 1
+
+    assert new_experiment.refers["adapter"].configuration == [
+        {"change_type": "break", "of_type": "commandlinechange"}
+    ]
+
+
+def test_new_config(init_full_x_new_config, monkeypatch):
+    """Test experiment branching with new config"""
+    experiment = experiment_builder.load(name="full_x_new_config")
+
+    assert experiment.refers["adapter"].configuration == [
+        {"change_type": "noeffect", "of_type": "commandlinechange"},
+        {
+            "of_type": "dimensionaddition",
+            "param": {"name": "/y", "type": "real", "value": 0},
+        },
+        {"change_type": "unsure", "of_type": "scriptconfigchange"},
+    ]
+
+    assert len(experiment.fetch_trials(with_evc_tree=True)) == 3
+    assert len(experiment.fetch_trials()) == 2
+
+
+def test_missing_config(init_full_x_new_config, monkeypatch):
+    """Test that experiment can branch with new config if previous is not present"""
+    name = "full_x_new_config"
+    experiment = experiment_builder.load(name=name)
+
+    # Mess with DB to change config path
+    metadata = experiment.metadata
+    bad_config_file = "ho_ho_idontexist.yaml"
+    config_file = metadata["parser"]["file_config_path"]
+    metadata["parser"]["file_config_path"] = bad_config_file
+    metadata["parser"]["parser"]["arguments"][2][1] = bad_config_file
+    metadata["user_args"][3] = bad_config_file
+    get_storage().update_experiment(experiment, metadata=metadata)
+
+    orion.core.cli.main(
+        (
+            "hunt --init-only -n {name} "
+            "--cli-change-type noeffect "
+            "--config-change-type unsure "
+            "./black_box_new.py -x~uniform(-10,10) --config {config_file}"
+        )
+        .format(name=name, config_file=config_file)
+        .split(" ")
+    )
+
+    new_experiment = experiment_builder.load(name=name)
+    assert new_experiment.version == experiment.version + 1
+
+    assert new_experiment.refers["adapter"].configuration == [
+        {"change_type": "noeffect", "of_type": "commandlinechange"}
+    ]
+
+
+def test_missing_and_new_config(init_full_x_new_config, monkeypatch):
+    """Test that experiment can branch with new config if previous is not present, with correct
+    diff.
+    """
+    name = "full_x_new_config"
+    experiment = experiment_builder.load(name=name)
+
+    # Mess with DB to change config path
+    metadata = experiment.metadata
+    bad_config_file = "ho_ho_idontexist.yaml"
+    config_file = metadata["parser"]["file_config_path"]
+    metadata["parser"]["file_config_path"] = bad_config_file
+    metadata["parser"]["parser"]["arguments"][2][1] = bad_config_file
+    metadata["user_args"][3] = bad_config_file
+
+    with open(config_file, "w") as f:
+        f.write(
+            yaml.dump(
+                {
+                    "new_arg": "some-new-value",
+                    "y": "orion~uniform(-10, 20, default_value=0)",
+                }
+            )
+        )
+
+    get_storage().update_experiment(experiment, metadata=metadata)
+
+    orion.core.cli.main(
+        (
+            "hunt --init-only -n {name} "
+            "--cli-change-type noeffect "
+            "--config-change-type unsure "
+            "./black_box_new.py -x~uniform(-10,10) --config {config_file}"
+        )
+        .format(name=name, config_file=config_file)
+        .split(" ")
+    )
+
+    new_experiment = experiment_builder.load(name=name)
+    assert new_experiment.version == experiment.version + 1
+
+    assert new_experiment.refers["adapter"].configuration == [
+        {
+            "name": "/y",
+            "new_prior": "uniform(-10, 20, default_value=0)",
+            "of_type": "dimensionpriorchange",
+            "old_prior": "uniform(-10, 10, default_value=0)",
+        },
+        {"change_type": "noeffect", "of_type": "commandlinechange"},
+        {"change_type": "unsure", "of_type": "scriptconfigchange"},
+    ]
+
+
 def test_auto_resolution_does_resolve(init_full_x_full_y, monkeypatch):
     """Test that auto-resolution does resolve all conflicts"""
     # Patch cmdloop to avoid autoresolution's prompt