Repository: IDSIA/sacred
Pull request #384 (2018-11-14 18:13:48)

Ensure that FileStorageObserver's save_cout appends new console output
I got a feedback from a system administrator that my process was displaying a strange file system usage pattern. When I investigated this, I have seen that issue #383 was the culprit.

So I commited this change to solve this problem.


Patch:
diff --git a/docs/collected_information.rst b/docs/collected_information.rst
index 7c95311..0519580 100644
--- a/docs/collected_information.rst
+++ b/docs/collected_information.rst
@@ -97,7 +97,7 @@ also collected. The default host info includes:
     ENV              captured ENVIRONMENT variables (if set)
     ===============  ==========================================
 
-Host information is available from the :ref:`api_run` through ``run.host_info``.
+Host information is available from the :ref:api_run through ``run.host_info``.
 It is sent to the observers by the :ref:`started_event <event_started>`.
 
 The list of captured ENVIRONMENT variables (empty by default) can be extended
diff --git a/docs/command_line.rst b/docs/command_line.rst
index 1d5d92d..e6f8a1f 100644
--- a/docs/command_line.rst
+++ b/docs/command_line.rst
@@ -574,11 +574,11 @@ some distributed workers fetch and start the queued up runs.
 Priority
 --------
 
-+--------------------------+----------------------------------------+
-| ``-P PRIORITY``          |  The (numeric) priority for this run.  |
-+--------------------------+                                        |
-| ``--priority=PRIORITY``  |                                        |
-+--------------------------+----------------------------------------+
++---------------+-----------------------------------------+
+| ``-p``        |  Only queue this run, do not start it.  |
++---------------+                                         |
+| ``--queue``   |                                         |
++---------------+-----------------------------------------+
 
 
 
@@ -605,11 +605,11 @@ repository, i.e. with no uncommitted changes.
 
 Print Config
 ------------
-+------------------------+------------------------------------------+
-| ``-p``                 |  Always print the config first.          |
-+------------------------+                                          |
-| ``--print_config``     |                                          |
-+------------------------+------------------------------------------+
++-------------------------+---------------------------------------------------+
+| ``-P PRIORITY``         |  Always print the config first.                   |
++-------------------------+                                                   |
+| ``--priority=PRIORITY`` |                                                   |
++-------------------------+---------------------------------------------------+
 
 If this flag is set, sacred will always print the current configuration
 including modifications (like the :ref:`print_config` command) before running
diff --git a/docs/observers.rst b/docs/observers.rst
index 34e0540..d9fe6c6 100644
--- a/docs/observers.rst
+++ b/docs/observers.rst
@@ -672,12 +672,6 @@ or pickle file containing...
   * optionally: a boolean for ``silent_completion``. If set to true, regular experiment completions
     will use no or less intrusive notifications, depending on the receiving device's platform.
     Experiment starts will always be sent silently, interruptions and failures always with full notifications.
-  * optionally: a string for ``proxy_url``. Specify this field, if Telegram is blocked in the local network or
-    in the country, and you want to use proxy server.
-    Format: ``PROTOCOL://PROXY_HOST:[PROXY_PORT]/``. Socks5 and HTTP protocols are supported.
-    These settings also could be received from ``HTTPS_PROXY`` or ``https_proxy`` environment variable.
-  * optionally: ``username`` for proxy.
-  * optionally: ``password`` for proxy.
 
 The observer is then added to the experment like this:
 
diff --git a/requirements.txt b/requirements.txt
index ab6b317..f5bde5a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-colorama>=0.4.0
 docopt>=0.6.2
 jsonpickle>=0.9.3
 mock>=2.0.0
@@ -8,4 +7,4 @@ py>=1.4.32
 py-cpuinfo>=4.0
 pytest>=3.0.5
 six>=1.10.0
-wrapt>=1.10.8
+wrapt>=1.10.8
\ No newline at end of file
diff --git a/sacred/__about__.py b/sacred/__about__.py
index 62e4f49..6983d44 100644
--- a/sacred/__about__.py
+++ b/sacred/__about__.py
@@ -11,7 +11,7 @@ from __future__ import division, print_function, unicode_literals
 
 __all__ = ("__version__", "__author__", "__author_email__", "__url__")
 
-__version__ = "0.7.4"
+__version__ = "0.7.4-onurgu"
 
 __author__ = 'Klaus Greff'
 __author_email__ = 'klaus.greff@startmail.com'
diff --git a/sacred/commands.py b/sacred/commands.py
index 3cea92b..5cfc51b 100644
--- a/sacred/commands.py
+++ b/sacred/commands.py
@@ -6,11 +6,7 @@ from __future__ import division, print_function, unicode_literals
 import pprint
 import pydoc
 import re
-import os
 from collections import namedtuple, OrderedDict
-from pkg_resources import parse_version
-
-from colorama import Fore, Style
 
 from sacred.config import save_config_file
 from sacred.serializer import flatten
@@ -19,18 +15,16 @@ from sacred.utils import PATHCHANGE, iterate_flattened_separately
 __all__ = ('print_config', 'print_dependencies', 'save_config',
            'help_for_command', 'print_named_configs')
 
-COLOR_DIRTY = Fore.RED
-COLOR_TYPECHANGED = Fore.RED  # prepend Style.BRIGHT for bold
-COLOR_ADDED = Fore.GREEN
-COLOR_MODIFIED = Fore.BLUE
-COLOR_DOC = Style.DIM
-ENDC = Style.RESET_ALL  # '\033[0m'
+BLUE = '\033[94m'
+GREEN = '\033[92m'
+RED = '\033[91m'
+GREY = '\033[90m'
+ENDC = '\033[0m'
 
-LEGEND = \
-    '(' + COLOR_MODIFIED + 'modified' + ENDC +\
-    ', ' + COLOR_ADDED + 'added' + ENDC +\
-    ', ' + COLOR_TYPECHANGED + 'typechanged' + ENDC +\
-    ', ' + COLOR_DOC + 'doc' + ENDC + ')'
+LEGEND = '(' + BLUE + 'modified' + ENDC +\
+    ', ' + GREEN + 'added' + ENDC +\
+    ', ' + RED + 'typechanged' + ENDC +\
+    ', ' + GREY + 'doc' + ENDC + ')'
 
 ConfigEntry = namedtuple('ConfigEntry',
                          'key value added modified typechanged doc')
@@ -74,16 +68,16 @@ def _format_named_config(indent, path, named_config):
     if hasattr(named_config, '__doc__') and named_config.__doc__ is not None:
         doc_string = named_config.__doc__
         if doc_string.strip().count('\n') == 0:
-            assign += COLOR_DOC + '   # {}'.format(doc_string.strip()) + ENDC
+            assign += GREY + '   # {}'.format(doc_string.strip()) + ENDC
         else:
             doc_string = doc_string.replace('\n', '\n' + indent)
-            assign += COLOR_DOC + '\n{}"""{}"""'.format(indent + '  ',
-                                                        doc_string) + ENDC
+            assign += GREY + '\n{}"""{}"""'.format(indent + '  ',
+                                                   doc_string) + ENDC
     return indent + assign
 
 
 def _format_named_configs(named_configs, indent=2):
-    lines = ['Named Configurations (' + COLOR_DOC + 'doc' + ENDC + '):']
+    lines = ['Named Configurations (' + GREY + 'doc' + ENDC + '):']
     for path, named_config in named_configs.items():
         lines.append(_format_named_config(indent, path, named_config))
     if len(lines) < 2:
@@ -131,7 +125,7 @@ def print_dependencies(_run):
         repos = _run.experiment_info['repositories']
         print('\nVersion Control:')
         for repo in repos:
-            mod = COLOR_DIRTY + 'M' if repo['dirty'] else ' '
+            mod = RED + 'M' if repo['dirty'] else ' '
             print('{} {:<43}  {}'.format(mod, repo['url'], repo['commit']) +
                   ENDC)
     print('')
@@ -173,13 +167,13 @@ def _format_entry(indent, entry):
     color = ""
     indent = ' ' * indent
     if entry.typechanged:
-        color = COLOR_TYPECHANGED  # red
+        color = RED
     elif entry.added:
-        color = COLOR_ADDED  # green
+        color = GREEN
     elif entry.modified:
-        color = COLOR_MODIFIED  # blue
+        color = BLUE
     if entry.key == '__doc__':
-        color = COLOR_DOC  # grey
+        color = GREY
         doc_string = entry.value.replace('\n', '\n' + indent)
         assign = '{}"""{}"""'.format(indent, doc_string)
     elif isinstance(entry, ConfigEntry):
@@ -187,7 +181,7 @@ def _format_entry(indent, entry):
     else:  # isinstance(entry, PathEntry):
         assign = indent + entry.key + ":"
     if entry.doc:
-        doc_string = COLOR_DOC + '# ' + entry.doc + ENDC
+        doc_string = GREY + '# ' + entry.doc + ENDC
         if len(assign) <= 35:
             assign = "{:<35}  {}".format(assign, doc_string)
         else:
diff --git a/sacred/config/config_scope.py b/sacred/config/config_scope.py
index 79dcbe3..ece27af 100644
--- a/sacred/config/config_scope.py
+++ b/sacred/config/config_scope.py
@@ -133,7 +133,7 @@ def dedent_function_body(body):
         if is_empty_or_comment(line):
             continue
         else:
-            indent = re.match(r'^\s*', line).group()
+            indent = re.match('^\s*', line).group()
             break
 
     out_lines = [dedent_line(line, indent) for line in lines]
diff --git a/sacred/dependencies.py b/sacred/dependencies.py
index ccff14f..c257743 100644
--- a/sacred/dependencies.py
+++ b/sacred/dependencies.py
@@ -248,7 +248,7 @@ class PackageDependency(object):
                     for tln in toplevel_names:
                         cls.modname_to_dist[
                             tln] = dist.project_name, dist.version
-                except Exception:
+                except:
                     pass
 
         # version = PackageDependency.get_version_heuristic(mod)
diff --git a/sacred/host_info.py b/sacred/host_info.py
index 86fa893..8370867 100644
--- a/sacred/host_info.py
+++ b/sacred/host_info.py
@@ -149,7 +149,7 @@ def _get_cpu_by_sysctl():
 def _get_cpu_by_proc_cpuinfo():
     command = ["cat", "/proc/cpuinfo"]
     all_info = subprocess.check_output(command).decode()
-    model_pattern = re.compile(r"^\s*model name\s*:")
+    model_pattern = re.compile("^\s*model name\s*:")
     for line in all_info.split("\n"):
         if model_pattern.match(line):
             return model_pattern.sub("", line, 1).strip()
diff --git a/sacred/observers/file_storage.py b/sacred/observers/file_storage.py
index 7c1f793..1fa819a 100644
--- a/sacred/observers/file_storage.py
+++ b/sacred/observers/file_storage.py
@@ -11,7 +11,7 @@ from shutil import copyfile
 from sacred.commandline_options import CommandLineOption
 from sacred.dependencies import get_digest
 from sacred.observers.base import RunObserver
-from sacred.utils import FileNotFoundError, FileExistsError  # py2 compat.
+from sacred.utils import FileNotFoundError, FileExistsError  # For compatibility with py2
 from sacred import optional as opt
 from sacred.serializer import flatten
 
@@ -49,6 +49,7 @@ class FileStorageObserver(RunObserver):
         self.config = None
         self.info = None
         self.cout = ""
+        self.cout_write_cursor = 0
 
     def queued_event(self, ex_info, command, host_info, queue_time, config,
                      meta_info, _id):
@@ -127,6 +128,7 @@ class FileStorageObserver(RunObserver):
         self.config = config
         self.info = {}
         self.cout = ""
+        self.cout_write_cursor = 0
 
         self.save_json(self.run_entry, 'run.json')
         self.save_json(self.config, 'config.json')
@@ -154,8 +156,9 @@ class FileStorageObserver(RunObserver):
         copyfile(filename, os.path.join(self.dir, target_name))
 
     def save_cout(self):
-        with open(os.path.join(self.dir, 'cout.txt'), 'wb') as f:
-            f.write(self.cout.encode('utf-8'))
+        with open(os.path.join(self.dir, 'cout.txt'), 'a', encoding="utf-8") as f:
+            f.write(self.cout[self.cout_write_cursor:])
+            self.cout_write_cursor = len(self.cout)
 
     def render_template(self):
         if opt.has_mako and self.template:
diff --git a/sacred/observers/mongo.py b/sacred/observers/mongo.py
index faed914..c6ddcb5 100644
--- a/sacred/observers/mongo.py
+++ b/sacred/observers/mongo.py
@@ -339,7 +339,7 @@ class MongoDbOption(CommandLineOption):
     RUN_ID_PATTERN = r"(?P<overwrite>\d{1,12})"
     PORT1_PATTERN = r"(?P<port1>\d{1,5})"
     PORT2_PATTERN = r"(?P<port2>\d{1,5})"
-    PRIORITY_PATTERN = r"(?P<priority>-?\d+)?"
+    PRIORITY_PATTERN = "(?P<priority>-?\d+)?"
     DB_NAME_PATTERN = r"(?P<db_name>[_A-Za-z]" \
                       r"[0-9A-Za-z#%&'()+\-;=@\[\]^_{}]{0,63})"
     COLL_NAME_PATTERN = r"(?P<collection>[_A-Za-z]" \
diff --git a/sacred/observers/telegram_obs.py b/sacred/observers/telegram_obs.py
index a1cb558..96a97a8 100644
--- a/sacred/observers/telegram_obs.py
+++ b/sacred/observers/telegram_obs.py
@@ -54,37 +54,8 @@ class TelegramObserver(RunObserver):
         import telegram
         d = load_config_file(filename)
         obs = None
-
-        if 'proxy_url' in d:
-            from telegram.utils.request import Request
-
-            if d['proxy_url'].startswith('socks5'):
-                urllib3_proxy_kwargs = dict()
-                for key in ['username', 'password']:
-                    if key in d:
-                        urllib3_proxy_kwargs[key] = d[key]
-                request = Request(proxy_url=d['proxy_url'], urllib3_proxy_kwargs=urllib3_proxy_kwargs)
-            elif d['proxy_url'].startswith('http'):
-                cred_string = ''
-                if 'username' in d:
-                    cred_string += d['username']
-                if 'password' in d:
-                    cred_string += ':' + d['password']
-                if len(cred_string) > 0:
-                    domain = d['proxy_url'].split('/')[-1].split('@')[-1]
-                    cred_string += '@'
-                    proxy_url = 'http://{}{}'.format(cred_string, domain)
-                    request = Request(proxy_url=proxy_url)
-                else:
-                    request = Request(proxy_url=d['proxy_url'])
-            else:
-                raise Exception("Proxy URL should be in format PROTOCOL://PROXY_HOST[:PROXY_PORT].\n"
-                                "HTTP and Socks5 are supported.")
-        else:
-            request = None
-
         if 'token' in d and 'chat_id' in d:
-            bot = telegram.Bot(d['token'], request=request)
+            bot = telegram.Bot(d['token'])
             obs = cls(bot, **d)
         else:
             raise ValueError("Telegram configuration file must contain "
diff --git a/sacred/utils.py b/sacred/utils.py
index d41dd66..0429887 100755
--- a/sacred/utils.py
+++ b/sacred/utils.py
@@ -26,8 +26,8 @@ __all__ = ["NO_LOGGER", "PYTHON_IDENTIFIER", "CircularDependencyError",
            "convert_to_nested_dict", "convert_camel_case_to_snake_case",
            "print_filtered_stacktrace", "is_subdir",
            "optional_kwargs_decorator", "get_inheritors",
-           "apply_backspaces_and_linefeeds", "StringIO", "FileNotFoundError",
-           "FileExistsError", "rel_path", "IntervalTimer"]
+           "apply_backspaces_and_linefeeds", "StringIO", "FileNotFoundError", "FileExistsError",
+           "rel_path", "IntervalTimer"]
 
 # A PY2 compatible basestring, int_types and FileNotFoundError
 if sys.version_info[0] == 2:
diff --git a/setup.cfg b/setup.cfg
index b600334..2a18cd1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -7,4 +7,4 @@ pep8ignore =
     dist/* ALL
     sacred.egg-info/* ALL
 [flake8]
-ignore = D100,D101,D102,D103,D104,D105,D203,D401,F821,W504,E722
+ignore = D100,D101,D102,D103,D104,D105,D203,D401,F821,E722
diff --git a/setup.py b/setup.py
index dd9894d..30f3e64 100755
--- a/setup.py
+++ b/setup.py
@@ -43,8 +43,7 @@ setup(
         'jsonpickle>=0.7.2, <1.0',
         'munch>=2.0.2, <3.0',
         'wrapt>=1.0, <2.0',
-        'py-cpuinfo>=4.0',
-        'colorama>=0.4',
+        'py-cpuinfo>=4.0'
     ],
     tests_require=[
         'mock>=0.8, <3.0',
diff --git a/tox.ini b/tox.ini
index 4a8812e..3365d07 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, py35, py36, flake8
+envlist = py27, py35, py36, flake8, py37
 # py32 does not work because of the 'wrapt' dependency
 # py33 is not supported by newer versions of numpy
 
@@ -40,4 +40,4 @@ commands =
     py.test \
         --cov sacred \
         {posargs}
-    coveralls
+    coveralls
\ No newline at end of file


Test patch:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 686324c..3f3a8b7 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -8,8 +8,7 @@ from collections import OrderedDict
 
 import pytest
 from sacred import Ingredient, Experiment
-from sacred.commands import (COLOR_MODIFIED, ENDC, COLOR_DOC, COLOR_ADDED,
-                             COLOR_TYPECHANGED, ConfigEntry,
+from sacred.commands import (BLUE, ENDC, GREY, GREEN, RED, ConfigEntry,
                              PathEntry, _format_config, _format_entry,
                              help_for_command, _iterate_marked, _non_unicode_repr,
                              _format_named_configs, _format_named_config)
@@ -114,28 +113,28 @@ def test_iterate_marked_typechanged(cfg):
     (PathEntry('f', False, False, None, None), "f:"),
     # Docstring entry
     (ConfigEntry('__doc__', 'multiline\ndocstring', False, False, None, None),
-     COLOR_DOC + '"""multiline\ndocstring"""' + ENDC),
+     GREY + '"""multiline\ndocstring"""' + ENDC),
 ])
 def test_format_entry(entry, expected):
     assert _format_entry(0, entry) == expected
 
 
 @pytest.mark.parametrize("entry,color", [
-    (ConfigEntry('a', 1, True, False, None, None),         COLOR_ADDED),
-    (ConfigEntry('b', 2, False, True, None, None),         COLOR_MODIFIED),
-    (ConfigEntry('c', 3, False, False, (bool, int), None), COLOR_TYPECHANGED),
-    (ConfigEntry('d', 4, True, True, None, None),          COLOR_ADDED),
-    (ConfigEntry('e', 5, True, False, (bool, int), None),  COLOR_TYPECHANGED),
-    (ConfigEntry('f', 6, False, True, (bool, int), None),  COLOR_TYPECHANGED),
-    (ConfigEntry('g', 7, True, True, (bool, int), None),   COLOR_TYPECHANGED),
+    (ConfigEntry('a', 1, True, False, None, None),         GREEN),
+    (ConfigEntry('b', 2, False, True, None, None),         BLUE),
+    (ConfigEntry('c', 3, False, False, (bool, int), None), RED),
+    (ConfigEntry('d', 4, True, True, None, None),          GREEN),
+    (ConfigEntry('e', 5, True, False, (bool, int), None),  RED),
+    (ConfigEntry('f', 6, False, True, (bool, int), None),  RED),
+    (ConfigEntry('g', 7, True, True, (bool, int), None),   RED),
     # Path entries
-    (PathEntry('a', True, False, None, None),         COLOR_ADDED),
-    (PathEntry('b', False, True, None, None),         COLOR_MODIFIED),
-    (PathEntry('c', False, False, (bool, int), None), COLOR_TYPECHANGED),
-    (PathEntry('d', True, True, None, None),          COLOR_ADDED),
-    (PathEntry('e', True, False, (bool, int), None),  COLOR_TYPECHANGED),
-    (PathEntry('f', False, True, (bool, int), None),  COLOR_TYPECHANGED),
-    (PathEntry('g', True, True, (bool, int), None),   COLOR_TYPECHANGED),
+    (PathEntry('a', True, False, None, None),         GREEN),
+    (PathEntry('b', False, True, None, None),         BLUE),
+    (PathEntry('c', False, False, (bool, int), None), RED),
+    (PathEntry('d', True, True, None, None),          GREEN),
+    (PathEntry('e', True, False, (bool, int), None),  RED),
+    (PathEntry('f', False, True, (bool, int), None),  RED),
+    (PathEntry('g', True, True, (bool, int), None),   RED),
 ])
 def test_format_entry_colors(entry, color):
     s = _format_entry(0, entry)
@@ -184,10 +183,10 @@ def _config_scope_with_multiline_doc():
     (0, 'a', None, 'a'),
     (1, 'b', None, ' b'),
     (4, 'a.b.c', None, '    a.b.c'),
-    (0, 'c', ConfigScope(_config_scope_with_single_line_doc), 'c' + COLOR_DOC
+    (0, 'c', ConfigScope(_config_scope_with_single_line_doc), 'c' + GREY
      + '   # doc' + ENDC),
     (0, 'd', ConfigScope(_config_scope_with_multiline_doc),
-     'd' + COLOR_DOC + '\n  """Multiline\n    docstring!\n    """' + ENDC)
+     'd' + GREY + '\n  """Multiline\n    docstring!\n    """' + ENDC)
 ])
 def test_format_named_config(indent, path, named_config, expected):
     assert _format_named_config(indent, path, named_config) == expected
@@ -212,7 +211,7 @@ def test_format_named_configs():
     named_configs_text = _format_named_configs(OrderedDict(
         ex.gather_named_configs()))
     assert named_configs_text.startswith('Named Configurations (' +
-                                            COLOR_DOC + 'doc' + ENDC + '):')
+                                            GREY + 'doc' + ENDC + '):')
     assert 'named_config2' in named_configs_text
     assert '# named config with doc' in named_configs_text
     assert 'ingred.named_config1' in named_configs_text
diff --git a/tests/test_observers/test_file_storage_observer.py b/tests/test_observers/test_file_storage_observer.py
index d9ad0a1..7f3732f 100644
--- a/tests/test_observers/test_file_storage_observer.py
+++ b/tests/test_observers/test_file_storage_observer.py
@@ -159,6 +159,32 @@ def test_fs_observer_heartbeat_event_updates_run(dir_obs, sample_run):
     assert info == i
 
 
+def test_fs_observer_heartbeat_event_multiple_updates_run(dir_obs, sample_run):
+    basedir, obs = dir_obs
+    _id = obs.started_event(**sample_run)
+    run_dir = basedir.join(_id)
+    info = {'my_info': [1, 2, 3], 'nr': 7}
+
+    captured_outs = [ ("some output %d\n" % i) for i in range(10)]
+    beat_times = [(T2 + datetime.timedelta(seconds=i*10)) for i in range(10)]
+
+    for idx in range(len(beat_times)):
+        expected_captured_output = "\n".join([x.strip() for x in captured_outs[:(idx+1)]]) + "\n"
+        obs.heartbeat_event(info=info, captured_out=expected_captured_output,
+                                                              beat_time=beat_times[idx],
+                                                              result=17)
+
+        assert run_dir.join('cout.txt').read() == expected_captured_output
+        run = json.loads(run_dir.join('run.json').read())
+
+        assert run['heartbeat'] == beat_times[idx].isoformat()
+        assert run['result'] == 17
+
+        assert run_dir.join('info.json').exists()
+        i = json.loads(run_dir.join('info.json').read())
+        assert info == i
+
+
 def test_fs_observer_completed_event_updates_run(dir_obs, sample_run):
     basedir, obs = dir_obs
     _id = obs.started_event(**sample_run)