Repository: DataDog/dd-agent

Issue:
dogstream doesn't work with windows full paths
Since it parse the values by `:`, and windows paths look like C:\path\to\things, so it will get mistakenly split.



Discussion:
the module loading functions also assume unix paths https://github.com/DataDog/dd-agent/blob/master/modules.py#L44

Is there a known workaround for this issue? I've tried to use relative path with no luck.

On my system I just wrote  dogstreams: \test.log on the configuation and placed a test.log file on the C:\  drive 

Thanks @archie-sh for your workaround!

We are going to prioritize that issue for one of the upcoming releases!


Resolving patch:
diff --git a/checks/datadog.py b/checks/datadog.py
--- a/checks/datadog.py
+++ b/checks/datadog.py
@@ -1,5 +1,4 @@
-from checks.utils import TailFile
-import modules
+# stdlib
 import os
 import sys
 import traceback
@@ -8,7 +7,11 @@
 from datetime import datetime
 from itertools import groupby # >= python 2.4
 
+# project
+import modules
 from checks import LaconicFilter
+from checks.utils import TailFile
+from util import windows_friendly_colon_split
 
 if hasattr('some string', 'partition'):
     def partition(s, sep):
@@ -46,7 +49,7 @@ def init(cls, logger, config):
             for config_item in dogstreams_config.split(','):
                 try:
                     config_item = config_item.strip()
-                    parts = config_item.split(':')
+                    parts = windows_friendly_colon_split(config_item)
                     if len(parts) == 1:
                         dogstreams.append(Dogstream.init(logger, log_path=parts[0]))
                     elif len(parts) == 2:
@@ -58,8 +61,6 @@ def init(cls, logger, config):
                             parser_spec=':'.join(parts[1:3]),
                             parser_args=parts[3:],
                             config=config))
-                    elif len(parts) > 3:
-                        logger.warn("Invalid dogstream: %s" % ':'.join(parts))
                 except Exception:
                     logger.exception("Cannot build dogstream")
 
@@ -101,7 +102,7 @@ def init(cls, logger, log_path, parser_spec=None, parser_args=None, config=None)
 
         if parser_spec:
             try:
-                parse_func = modules.load(parser_spec, 'parser')
+                parse_func = modules.load(parser_spec)
                 if isinstance(parse_func, type):
                     logger.info('Instantiating class-based dogstream')
                     parse_func = parse_func(
@@ -389,4 +390,4 @@ def testddForwarder():
 
 
 if __name__ == '__main__':
-    testddForwarder()
\ No newline at end of file
+    testddForwarder()
diff --git a/modules.py b/modules.py
--- a/modules.py
+++ b/modules.py
@@ -1,10 +1,17 @@
 """ Tools for loading Python modules from arbitrary locations.
 """
 
+# stdlib
 import os
 import imp
+import re
 import sys
 
+# project
+from util import windows_friendly_colon_split
+
+WINDOWS_PATH = re.compile('[A-Z]:.*')
+
 def imp_type_for_filename(filename):
     """Given the name of a Python module, return a type description suitable to
     be passed to imp.load_module()"""
@@ -41,8 +48,16 @@ def module_name_for_filename(filename):
     all_segments = filename.split(os.sep)
     path_elements = all_segments[:-1]
     module_elements = [all_segments[-1].rsplit('.', 1)[0]]
-    while os.path.exists('/'.join(path_elements + ['__init__.py'])):
-        module_elements.insert(0, path_elements.pop())
+    while True:
+        init_path = os.path.join(*(path_elements +['__init__.py']))
+        if path_elements[0] is "":
+            # os.path.join will not put the leading '/'
+            # it will return a/b/c for os.path.join("","a","b","c")
+            init_path = '/' + init_path
+        if os.path.exists(init_path):
+            module_elements.insert(0, path_elements.pop())
+        else:
+            break
     modulename = '.'.join(module_elements)
     basename = '/'.join(path_elements)
     return (basename, modulename)
@@ -52,7 +67,7 @@ def get_module(name):
     and return a Python module.
     
     If the module is already loaded, takes no action."""
-    if name.startswith('/'):
+    if name.startswith('/') or WINDOWS_PATH.match(name):
         basename, modulename = module_name_for_filename(name)
         path = [basename]
     else:
@@ -64,9 +79,13 @@ def get_module(name):
 
 def load(config_string, default_name=None):
     """Given a module name and an object expected to be contained within,
-    return said object"""
-    (module_name, object_name) = \
-            (config_string.rsplit(':', 1) + [default_name])[:2]
+    return said object.
+    """
+    split = windows_friendly_colon_split(config_string)
+    if len(split)> 1:
+        module_name, object_name = ":".join(split[:-1]), split[-1]
+    else:
+        module_name, object_name = config_string, default_name
     module = get_module(module_name)
     if object_name:
         return getattr(module, object_name)
diff --git a/util.py b/util.py
--- a/util.py
+++ b/util.py
@@ -22,6 +22,7 @@
 
 VALID_HOSTNAME_RFC_1123_PATTERN = re.compile(r"^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$")
 MAX_HOSTNAME_LEN = 255
+COLON_NON_WIN_PATH = re.compile(':(?!\\\\)')
 
 log = logging.getLogger(__name__)
 
@@ -71,6 +72,16 @@ def headers(agentConfig):
         'Accept': 'text/html, */*',
     }
 
+def windows_friendly_colon_split(config_string):
+    '''
+    Perform a split by ':' on the config_string
+    without splitting on the start of windows path
+    '''
+    if Platform.is_win32():
+        # will split on path/to/module.py:blabla but not on C:\\path
+        return COLON_NON_WIN_PATH.split(config_string)
+    else:
+        return config_string.split(':')
 
 def getTopIndex():
     macV = None


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -5,7 +5,7 @@
 
 from config import get_config
 
-from util import PidFile, is_valid_hostname
+from util import PidFile, is_valid_hostname, Platform, windows_friendly_colon_split
 
 class TestConfig(unittest.TestCase):
     def testWhiteSpaceConfig(self):
@@ -77,6 +77,23 @@ def testHostname(self):
         for hostname in not_valid_hostnames:
             self.assertFalse(is_valid_hostname(hostname), hostname)
 
+    def testWindowsSplit(self):
+        # Make the function run as if it was on windows
+        func = Platform.is_win32
+        try:
+            Platform.is_win32 = staticmethod(lambda : True)
+
+            test_cases = [
+                ("C:\\Documents\\Users\\script.py:C:\\Documents\\otherscript.py", ["C:\\Documents\\Users\\script.py","C:\\Documents\\otherscript.py"]),
+                ("C:\\Documents\\Users\\script.py:parser.py", ["C:\\Documents\\Users\\script.py","parser.py"])
+                ]
+
+            for test_case, expected_result in test_cases:
+                self.assertEqual(windows_friendly_colon_split(test_case), expected_result)
+        finally:
+            # cleanup
+            Platform.is_win32 = staticmethod(func)
+
 if __name__ == '__main__':
     unittest.main()
 
diff --git a/tests/test_modules.py b/tests/test_modules.py
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -12,20 +12,25 @@
 has_been_mutated = False
 
 class TestModuleLoad(unittest.TestCase):
+
     def setUp(self):
         sys.modules[__name__].has_been_mutated = True
         if 'tests.target_module' in sys.modules:
             del sys.modules['tests.target_module']
+
     def tearDown(self):
         sys.modules[__name__].has_been_mutated = False
+
     def test_cached_module(self):
         """Modules already in the cache should be reused"""
         self.assertTrue(modules.load('%s:has_been_mutated' % __name__))
+
     def test_cache_population(self):
         """Python module cache should be populated"""
         self.assertTrue(not 'tests.target_module' in sys.modules)
         modules.load('tests.target_module')
         self.assertTrue('tests.target_module' in sys.modules)
+
     def test_modname_load_default(self):
         """When the specifier contains no module name, any provided default
         should be used"""
@@ -35,6 +40,7 @@ def test_modname_load_default(self):
                 'default_target'),
             'DEFAULT'
         )
+
     def test_modname_load_specified(self):
         """When the specifier contains a module name, any provided default
         should be overridden"""
@@ -44,8 +50,12 @@ def test_modname_load_specified(self):
                 'default_target'),
             'SPECIFIED'
         )
+
     def test_pathname_load_finds_package(self):
         """"Loading modules by absolute path should correctly set the name of
         the loaded module to include any package containing it."""
         m = modules.load(os.getcwd() + '/tests/target_module.py')
         self.assertEquals(m.__name__, 'tests.target_module')
+
+if __name__ == '__main__':
+    unittest.main()