Repository: mesonbuild/meson

Issue:
meson test fails if LANG variable contains characters not valid in the current locale
To reproduce:

Add an environment variable with an accent:
`export BLOB=éêà`
Set LANG to a something that doesn'T contain the characters
`export LANG=POSIX`

Run meson test
`meson test -C build/`

See surrogates error:
```
root@276e4c1f828a libnice]# meson test -C build/
ninja: Entering directory `/root/libnice/build'
ninja: no work to do.
Traceback (most recent call last):
  File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/mesonbuild/mesonmain.py", line 111, in run
    return options.run_func(options)
  File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/mesonbuild/mtest.py", line 805, in run
    return th.doit()
  File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/mesonbuild/mtest.py", line 555, in doit
    self.run_tests(tests)
  File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/mesonbuild/mtest.py", line 686, in run_tests
    self.open_log_files()
  File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/mesonbuild/mtest.py", line 656, in open_log_files
    self.logfile.write('Inherited environment: {}\n\n'.format(inherit_env))
UnicodeEncodeError: 'utf-8' codec can't encode characters in position 1658-1659: surrogates not allowed
```


Discussion:

Resolving patch:
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -17,6 +17,7 @@
 import importlib
 import traceback
 import argparse
+import codecs
 
 from . import mesonlib
 from . import mlog
@@ -148,6 +149,17 @@ def run_script_command(script_name, script_args):
         mlog.exception(e)
         return 1
 
+def ensure_stdout_accepts_unicode():
+    if sys.stdout.encoding and not sys.stdout.encoding.upper().startswith('UTF-'):
+        if sys.version_info >= (3, 7):
+            sys.stdout.reconfigure(errors='surrogateescape')
+        else:
+            sys.stdout = codecs.getwriter('utf-8')(sys.stdout.detach(),
+                                                   errors='surrogateescape')
+            sys.stdout.encoding = 'UTF-8'
+            if not hasattr(sys.stdout, 'buffer'):
+                sys.stdout.buffer = sys.stdout.raw if hasattr(sys.stdout, 'raw') else sys.stdout
+
 def run(original_args, mainfile):
     if sys.version_info < (3, 5):
         print('Meson works correctly only with python 3.5+.')
@@ -155,6 +167,11 @@ def run(original_args, mainfile):
         print('Please update your environment')
         return 1
 
+    # Meson gets confused if stdout can't output Unicode, if the
+    # locale isn't Unicode, just force stdout to accept it. This tries
+    # to emulate enough of PEP 540 to work elsewhere.
+    ensure_stdout_accepts_unicode()
+
     # https://github.com/mesonbuild/meson/issues/3653
     if sys.platform.lower() == 'msys':
         mlog.error('This python3 seems to be msys/python on MSYS2 Windows, which is known to have path semantics incompatible with Meson')
diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py
--- a/mesonbuild/mtest.py
+++ b/mesonbuild/mtest.py
@@ -647,8 +647,8 @@ def open_log_files(self):
         self.logfilename = logfile_base + '.txt'
         self.jsonlogfilename = logfile_base + '.json'
 
-        self.jsonlogfile = open(self.jsonlogfilename, 'w', encoding='utf-8')
-        self.logfile = open(self.logfilename, 'w', encoding='utf-8')
+        self.jsonlogfile = open(self.jsonlogfilename, 'w', encoding='utf-8', errors='replace')
+        self.logfile = open(self.logfilename, 'w', encoding='utf-8', errors='surrogateescape')
 
         self.logfile.write('Log of Meson test suite run on %s\n\n'
                            % datetime.datetime.now().isoformat())


Test patch: