Repository: znicholls/CMIP6-json-data-citation-generator

Issue:
Check special character support 
check the support for 'utf-8' in the json template and 
the json result?
The authors or even their affiliations might be 'utf-8' (non-English). 
To give you a German example: One of our most common last names is: 
'Müller'.

Discussion:

Patch:
diff --git a/CMIP6_json_data_citation_generator/__init__.py b/CMIP6_json_data_citation_generator/__init__.py
index bb6ab69..dceaa2a 100644
--- a/CMIP6_json_data_citation_generator/__init__.py
+++ b/CMIP6_json_data_citation_generator/__init__.py
@@ -1,5 +1,7 @@
 from os import listdir, makedirs, walk
 from os.path import split, splitext, basename, join, isdir, isfile
+import io
+import sys
 
 import yaml
 import json
@@ -149,8 +151,15 @@ class jsonGenerator():
         return updated_yml
 
     def write_json_to_file(self, json_dict=None, file_name=None):
-        with open(file_name, 'w') as file_name:
-            json.dump(json_dict, file_name, indent=4)
+        with io.open(file_name, 'w', encoding='utf8') as json_file:
+            text = json.dumps(
+                json_dict,
+                ensure_ascii=False,
+                indent=4
+            )
+            if sys.version.startswith('2'):
+                text=unicode(text)
+            json_file.write(text)
 
     def write_json_for_filename_to_file_with_template(self, file_name=None, yaml_template=None, output_file=None):
         yaml_template = self.return_template_yaml_from(in_file=yaml_template)


Test patch:
diff --git a/tests/data/yaml-test-files/test-special-char.yml b/tests/data/yaml-test-files/test-special-char.yml
new file mode 100644
index 0000000..eee4ea7
--- /dev/null
+++ b/tests/data/yaml-test-files/test-special-char.yml
@@ -0,0 +1,6 @@
+creators:
+  - creatorName: "Müller, Björn"
+    givenName: "Björn"
+    familyName: "Müller"
+    email: "björnmüller@äéèîç.com"
+    affiliation: "kæčœ universität von Lände"
diff --git a/tests/test_generate_CMIP6_json_files.py b/tests/test_generate_CMIP6_json_files.py
index 4d30184..aa27d7f 100644
--- a/tests/test_generate_CMIP6_json_files.py
+++ b/tests/test_generate_CMIP6_json_files.py
@@ -22,7 +22,7 @@ test_file_unique_jsons = [
     'UoM_UoM-ssp534os-1-1-0_input4MIPs_ScenarioMIP',
     'UoM_UoM-ssp585-1-1-0_input4MIPs_ScenarioMIP',
 ]
-test_output_path = './test-json-output-path'
+test_output_path = './tests/test-json-output-path'
 
 @pytest.fixture
 def tear_down_test_path():
diff --git a/tests/test_jsonGenerator.py b/tests/test_jsonGenerator.py
index 5bf97e9..f23d091 100644
--- a/tests/test_jsonGenerator.py
+++ b/tests/test_jsonGenerator.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 from os import listdir, remove
 from os.path import join, isfile, dirname
 from shutil import rmtree
@@ -5,6 +7,7 @@ import re
 import datetime
 import sys
 from tempfile import mkstemp
+import codecs
 
 import pytest
 from pytest import raises
@@ -15,7 +18,8 @@ from utils import captured_output
 from CMIP6_json_data_citation_generator import CMIPPathHandler
 from CMIP6_json_data_citation_generator import jsonGenerator
 
-test_file_path_empty_files = join('.', 'tests', 'data', 'empty-test-files')
+test_data_path = join('.', 'tests', 'data')
+test_file_path_empty_files = join(test_data_path, 'empty-test-files')
 test_file_unique_source_ids = [
     'UoM-ssp119-1-1-0',
     'UoM-ssp245-1-1-0',
@@ -26,11 +30,16 @@ test_file_unique_source_ids = [
 ]
 test_output_path = join('.', 'test-json-output-path')
 
-test_file_path_yaml = join('.', 'tests', 'data', 'yaml-test-files')
+test_file_path_yaml = join(test_data_path, 'yaml-test-files')
 test_data_citation_template_yaml = join(
     test_file_path_yaml,
     'test-data-citation-template.yml'
 )
+test_file_path_yaml_special_char = join(test_data_path, 'yaml-test-files', 'test-special-char.yml')
+test_file_path_yaml_special_char_written = test_file_path_yaml_special_char.replace(
+    '.yml',
+    '-written.yml'
+)
 
 def get_test_file():
     for test_file in listdir(test_file_path_empty_files):
@@ -264,13 +273,13 @@ def test_check_yaml_replace_values():
         assert subbed_yml['fundingReferences'][0]['funderName'] == [value]
 
 def test_write_json_to_file():
-    with patch('CMIP6_json_data_citation_generator.open') as mock_open:
-        with patch('CMIP6_json_data_citation_generator.json.dump') as mock_json_dump:
+    with patch('CMIP6_json_data_citation_generator.io.open') as mock_open:
+        with patch('CMIP6_json_data_citation_generator.json.dumps') as mock_json_dump:
             Generator = jsonGenerator()
             test_fn = 'UoM-ssp119-1-1-0'
             test_dict = {'hi': 'test', 'bye': 'another test'}
             Generator.write_json_to_file(json_dict=test_dict, file_name=test_fn)
-            mock_open.assert_called_with(test_fn, 'w')
+            mock_open.assert_called_with(test_fn, 'w', encoding='utf8')
             mock_json_dump.assert_called_once()
 
 @patch.object(jsonGenerator, 'return_template_yaml_from')
@@ -462,3 +471,49 @@ def test_invalid_name_in_dir(mock_walk, mock_isdir):
 
 
     assert 'Unable to split filename: {}'.format(junk_name) == out.getvalue().strip()
+
+def test_special_yaml_read():
+    Generator = jsonGenerator()
+    actual_result = Generator.return_template_yaml_from(
+        in_file=test_file_path_yaml_special_char
+    )
+    expected_result = {
+        'creators': [
+            {
+                'creatorName': u"Müller, Björn",
+                'givenName': u"Björn",
+                'familyName': u"Müller",
+                'email': u"björnmüller@äéèîç.com",
+                'affiliation': u'kæčœ universität von Lände',
+            },
+        ],
+    }
+    assert actual_result == expected_result
+
+@pytest.fixture
+def remove_written_special_yaml():
+    yield None
+    if isfile(test_file_path_yaml_special_char_written):
+        remove(test_file_path_yaml_special_char_written)
+
+def test_special_yaml_write(remove_written_special_yaml):
+    Generator = jsonGenerator()
+    dict_to_write = Generator.return_template_yaml_from(
+        in_file=test_file_path_yaml_special_char
+    )
+    Generator.write_json_to_file(
+        json_dict=dict_to_write,
+        file_name=test_file_path_yaml_special_char_written
+    )
+    expected_strings = [
+        u"Müller, Björn",
+        u"Björn",
+        u"Müller",
+        u"björnmüller@äéèîç.com",
+        u"kæčœ universität von Lände",
+    ]
+    with codecs.open(test_file_path_yaml_special_char_written, "r", "utf-8") as written_file:
+        written_text = written_file.read()
+
+        for expected_string in expected_strings:
+            assert expected_string in written_text