Repository: mkdocs/mkdocs
Pull request #592 (2015-06-04 15:52:01)

YAML now loads all strings as unicode
- All strings are redirected to be loaded as unicode
- I was experiencing an issue on my system where I was getting errors
trying to access config file objects because they were not being closed.
In all cases, once the YAML content was loaded, we were done accessing
the file content.  YAML object will close file access in the objects
once processing the content is complete.


Patch:
diff --git a/docs/about/release-notes.md b/docs/about/release-notes.md
index 69478543..93754c65 100644
--- a/docs/about/release-notes.md
+++ b/docs/about/release-notes.md
@@ -15,6 +15,7 @@ You can determine your currently installed version using `mkdocs --version`:
 
 ## Version 0.14.0 (2015-??-??)
 
+* Improve Unicode handling by ensuring that all YAML strings are loaded as Unicode.
 * Remove dependancy on the six library. (#583)
 * Add `--quiet` and `--verbose` options to all subcommands.
 * Add short options (`-a`) to most command line options.
@@ -252,4 +253,3 @@ documentation.
 * Bugfix: Fix the mkdocs command creation under Windows. (#122)
 * Bugfix: Correctly handle external `extra_javascript` and `extra_css`. (#92)
 * Bugfix: Fixed favicon support. (#87)
-
diff --git a/mkdocs/config/base.py b/mkdocs/config/base.py
index 43c1897e..d8a0df82 100644
--- a/mkdocs/config/base.py
+++ b/mkdocs/config/base.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 import logging
 import os
-import yaml
 
 from mkdocs import exceptions
 from mkdocs import utils
@@ -94,7 +93,7 @@ class Config(utils.UserDict):
         self.data.update(patch)
 
     def load_file(self, config_file):
-        return self.load_dict(yaml.load(config_file))
+        return self.load_dict(utils.yaml_load(config_file))
 
 
 def _open_config_file(config_file):
diff --git a/mkdocs/utils.py b/mkdocs/utils.py
index 5f72cd96..951ef2e4 100644
--- a/mkdocs/utils.py
+++ b/mkdocs/utils.py
@@ -12,6 +12,7 @@ import os
 import sys
 import shutil
 import markdown
+import yaml
 
 from mkdocs import toc
 
@@ -35,6 +36,37 @@ else:                           # pragma: no cover
     text_type = unicode         # noqa
 
 
+def yaml_load(source, loader=yaml.Loader):
+    """
+    Wrap PyYaml's loader so we can extend it to suit our needs.
+
+    Load all strings as unicode.
+    http://stackoverflow.com/a/2967461/3609487
+    """
+
+    def construct_yaml_str(self, node):
+        """Override the default string handling function to always return unicode objects."""
+        return self.construct_scalar(node)
+
+    class Loader(loader):
+        """Define a custom loader derived from the global loader to leave the global loader unaltered."""
+
+    # Attach our unicode constructor to our custom loader ensuring all strings will be unicode on translation.
+    Loader.add_constructor('tag:yaml.org,2002:str', construct_yaml_str)
+
+    try:
+        return yaml.load(source, Loader)
+    finally:
+        # TODO: Remove this when external calls are properly cleaning up file objects.
+        # Some mkdocs internal calls, sometimes in test lib, will load configs
+        # with a file object but never close it.  On some systems, if a delete
+        # action is performed on that file without Python closing that object,
+        # there will be an access error. This will process the file and close it
+        # as there should be no more use for the file once we process the yaml content.
+        if hasattr(source, 'close'):
+            source.close()
+
+
 def reduce_list(data_set):
     """ Reduce duplicate items in a list and preserve order """
     seen = set()


Test patch:
diff --git a/mkdocs/tests/legacy_tests.py b/mkdocs/tests/legacy_tests.py
index 38e7d71b..a9e04fa1 100644
--- a/mkdocs/tests/legacy_tests.py
+++ b/mkdocs/tests/legacy_tests.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 import unittest
 
-import yaml
-
-from mkdocs import legacy
+from mkdocs import legacy, utils
 
 
 class TestCompatabilityShim(unittest.TestCase):
@@ -43,8 +41,8 @@ class TestCompatabilityShim(unittest.TestCase):
         - CLI Guide: cli.md
         """
         self.assertEqual(
-            legacy.pages_compat_shim(yaml.load(pages_yaml_old)['pages']),
-            yaml.load(pages_yaml_new)['pages'])
+            legacy.pages_compat_shim(utils.yaml_load(pages_yaml_old)['pages']),
+            utils.yaml_load(pages_yaml_new)['pages'])
 
     def test_convert_no_home(self):
 
@@ -62,5 +60,5 @@ class TestCompatabilityShim(unittest.TestCase):
         - About: about.md
         """
         self.assertEqual(
-            legacy.pages_compat_shim(yaml.load(pages_yaml_old)['pages']),
-            yaml.load(pages_yaml_new)['pages'])
+            legacy.pages_compat_shim(utils.yaml_load(pages_yaml_old)['pages']),
+            utils.yaml_load(pages_yaml_new)['pages'])
diff --git a/mkdocs/tests/utils_tests.py b/mkdocs/tests/utils_tests.py
index fcef2328..0407f9fd 100644
--- a/mkdocs/tests/utils_tests.py
+++ b/mkdocs/tests/utils_tests.py
@@ -6,6 +6,7 @@ import os
 import unittest
 
 from mkdocs import nav, utils
+from mkdocs.tests.base import dedent
 
 
 class UtilsTests(unittest.TestCase):
@@ -141,3 +142,17 @@ class UtilsTests(unittest.TestCase):
                     j('about', 'release-notes.md')]}
             ]
         )
+
+    def test_unicode_yaml(self):
+
+        yaml_src = dedent(
+            '''
+            key: value
+            key2:
+              - value
+            '''
+        )
+
+        config = utils.yaml_load(yaml_src)
+        self.assertTrue(isinstance(config['key'], utils.text_type))
+        self.assertTrue(isinstance(config['key2'][0], utils.text_type))