Repository: seznam/flexp

Issue:
Flexp.setup doesn't clean the experiment directory.
When creating an experiment, I usually did clean_experiment_folder to get rid of previous results. 
Now if I do this:
```python
    flexp.setup(EXPS_DIR, DESC, with_date=not SHORT_RUN, default_rights=RWXRWXRWX)
    clean_experiment_folder(flexp.get_file_path(""))
```
it will remove the log.txt as well, because it is now created by flexp.setup and not flog. Before calling `flexp.setup`, I don't know what is the experiment dir so I can't clean it.

I suggest adding a switch `clean_experiment_folder`, defaults to True, which removes everything in the experiment folder and only after that creates the log file.

What do you think?

Discussion:
ofilip: OK for me.

Patch:
diff --git a/flexp/flexp/core.py b/flexp/flexp/core.py
index 0c4d474..6a14022 100644
--- a/flexp/flexp/core.py
+++ b/flexp/flexp/core.py
@@ -3,6 +3,7 @@ from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import division
 
+import shutil
 from datetime import datetime
 from shutil import copyfile, copystat
 import io
@@ -222,8 +223,38 @@ class ExperimentHandler(object):
         self._metadata[key] = value
 
 
+def clean_experiment_dir(override_dir):
+    """
+    Cleans experiment folder.
+    :param bool override_dir: If true, clean the folder, if false, check it's empty or doesn't exists
+    :return:
+    """
+    experiment_dir = get_file_path("")
+    # Check experiment_dir is empty or doesn't exists
+    if not override_dir:
+        if not os.path.exists(experiment_dir) or not os.listdir(experiment_dir):
+            return
+        else:
+            raise FileExistsError(
+                "Directory '{}' already exists and is not empty! "
+                "Use override_dir=True to clean the directory."
+                    .format(name())
+            )
+    else:
+        # Dir doesn't exists, so everything's fine.
+        if not os.path.exists(experiment_dir):
+            return
+        # Remove all files in the directory
+        for the_file in os.listdir(experiment_dir):
+            file_path = os.path.join(experiment_dir, the_file)
+            if os.path.isfile(file_path):
+                os.unlink(file_path)
+            elif os.path.isdir(file_path):
+                shutil.rmtree(file_path)
+
+
 def setup(root_dir, exp_name=None, with_date=False, backup=True, default_rights=RWXRWS, loglevel=logging.DEBUG,
-          log_filename='log.txt', disable_stderr=False):
+          log_filename='log.txt', disable_stderr=False, override_dir=False):
     """Set up Experiment handler.
 
     :param root_dir: Path to experiments root directory. New directories will be created in the folder.
@@ -234,11 +265,17 @@ def setup(root_dir, exp_name=None, with_date=False, backup=True, default_rights=
     :param loglevel: Loglevel, DEBUG by default, can be overriden by FLEXP_LOGLEVEL environment variable.
     :param log_filename: Filename of log in experiment dir. 'log.txt' by default. If None, logging to file is disabled.
     :param disable_stderr: Enable/disable logging on stderr. False by default.
+    :param bool override_dir: If True, remove all contents of the directory.
+                              If False and there is any file in the directory, raise an exception.
     """
     if "experiment" in _eh:
         raise Exception("flexp.setup() called twice")
     _eh["experiment"] = ExperimentHandler(default_rights=default_rights)
     _eh["experiment"].setup(root_dir, exp_name, with_date=with_date)
+
+    # Clean the directory or check it's empty
+    clean_experiment_dir(override_dir)
+
     if backup:
         _eh["experiment"].backup_files([sys.argv[0]])
     atexit.register(_eh["experiment"]._write_metadata)
@@ -327,4 +364,5 @@ def set_metadata(key, value):
 def disable():
     """Surpress all output and returns /dev/null as path to all files."""
     _eh["experiment"].disabled = True
+    _setup_logging(logging.FATAL, None, True)
 
diff --git a/flexp/flexp/logging.py b/flexp/flexp/logging.py
index aa57d50..b15c269 100644
--- a/flexp/flexp/logging.py
+++ b/flexp/flexp/logging.py
@@ -3,7 +3,6 @@ import warnings
 import os
 import six
 
-
 log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
 
@@ -39,6 +38,7 @@ def get_loglevel_from_env(default_level):
 
 
 def _setup_logging(level=logging.DEBUG, filename='log.txt', disable_stderr=False):
+    _close_file_handlers()
     level = loglevel_from_string(level)
     root_logger = logging.getLogger()
     root_logger.setLevel(level)
@@ -52,3 +52,9 @@ def _setup_logging(level=logging.DEBUG, filename='log.txt', disable_stderr=False
         root_logger.addHandler(stream_handler)
 
     warnings.simplefilter("once")
+
+
+def _close_file_handlers():
+    root_logger = logging.getLogger()
+    for file_handler in root_logger.handlers:
+        file_handler.close()


Test patch:
diff --git a/tests/test_flexp.py b/tests/test_flexp.py
index 574e942..e814871 100644
--- a/tests/test_flexp.py
+++ b/tests/test_flexp.py
@@ -30,6 +30,7 @@ def test_working():
     if not hasattr(atexit, "unregister") and path.exists(
             path.join(expdir, "flexp_info.txt")):
         os.unlink(path.join(expdir, "flexp_info.txt"))
+    flexp.disable()
     shutil.rmtree(expdir)
 
 
diff --git a/tests/test_flexp_override.py b/tests/test_flexp_override.py
new file mode 100644
index 0000000..90623d0
--- /dev/null
+++ b/tests/test_flexp_override.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+
+import atexit
+import io
+import os
+from os import path
+import shutil
+
+import pytest
+
+from flexp import flexp
+
+
+def test_override():
+    expdir = path.join("tests/data/", "exp01")
+
+    # Remove the experiment dir if it exists
+    if os.path.exists(expdir):
+        shutil.rmtree(expdir)
+
+    # We have to reset the _eh to make flexp stop complaining about calling setup twice.
+    flexp.core._eh = {}
+    flexp.setup("tests/data/", "exp01", False, override_dir=False)
+
+    assert path.isdir(expdir), "flexp didn't create experiment dir with override_dir=False"
+
+    # Test that it fails to create the directory, there should be logging file already.
+    with pytest.raises(FileExistsError):
+        flexp.core._eh = {}
+        flexp.setup("tests/data/", "exp01", False, override_dir=False)
+
+    # This should be ok
+    flexp.core._eh = {}
+    flexp.setup("tests/data/", "exp01", False, override_dir=True)
+
+    # Disable logging to be able to delete the experiment directory.
+    flexp.disable()
+
+    # Remove the experiment dir
+    if os.path.exists(expdir):
+        shutil.rmtree(expdir)