Repository: ansible/ansible

Issue:
Crash with file module when directories have non utf-8 files
##### Bug Report

##### COMPONENT NAME

file

##### ANSIBLE VERSION

```
ansible 2.4.0 (recursive-crash-non-utf8 78836ec0b9) last updated 2017/04/21 15:31:54 (GMT +200)
  config file = /etc/ansible/ansible.cfg
  python version = 2.7.12 (default, Nov 19 2016, 06:48:10) [GCC 5.4.0 20160609]
```

##### CONFIGURATION

Nothing special

##### OS / ENVIRONMENT

Nothing special

##### SUMMARY

When launching file module with recurse = yes on directories with non utf-8 files, Ansible crash with the following message:

```
LookupError: unknown error handler name 'surrogate_then_strict'
```
##### STEPS TO REPRODUCE

You can retrieve an example at the following location: https://github.com/Yannig/yannig-ansible-playbooks/tree/master/file-encoding

```yaml
- name: "Recursive change with non utf-8 filename"
  hosts: localhost
  gather_facts: no
  tasks:
    - name: "Recursive change (crash)"
      file: path=dir mode=0755 recurse=yes
```

##### EXPECTED RESULTS

Playbook apply change without crash

##### ACTUAL RESULTS

```
exception:
Traceback (most recent call last):
  File "/tmp/ansible_atAUEE/ansible_module_file.py", line 480, in <module>
    main()
  File "/tmp/ansible_atAUEE/ansible_module_file.py", line 362, in main
    changed |= recursive_set_attributes(module, to_bytes(file_args['path'], errors='surrogate_or_strict'), follow, file_args)
  File "/tmp/ansible_atAUEE/ansible_module_file.py", line 171, in recursive_set_attributes
    changed |= module.set_fs_attributes_if_different(tmp_file_args, changed)
  File "/tmp/ansible_atAUEE/ansible_modlib.zip/ansible/module_utils/basic.py", line 1292, in set_fs_attributes_if_different
  File "/tmp/ansible_atAUEE/ansible_modlib.zip/ansible/module_utils/basic.py", line 995, in set_owner_if_different
  File "/tmp/ansible_atAUEE/ansible_modlib.zip/ansible/module_utils/_text.py", line 232, in to_text
  File "/usr/lib/python2.7/encodings/utf_8.py", line 16, in decode
    return codecs.utf_8_decode(input, errors, True)
LookupError: unknown error handler name 'surrogate_then_strict'
```



Discussion:
This issue affect also Ansible 2.3.1.
Is there any way to have a feedback on this issue?

Resolving patch:
diff --git a/lib/ansible/module_utils/_text.py b/lib/ansible/module_utils/_text.py
--- a/lib/ansible/module_utils/_text.py
+++ b/lib/ansible/module_utils/_text.py
@@ -44,7 +44,7 @@
     HAS_SURROGATEESCAPE = False
 
 
-_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_escape',
+_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_replace',
                                       'surrogate_or_strict',
                                       'surrogate_then_replace'))
 
@@ -133,6 +133,9 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'):
             return obj.encode(encoding, errors)
         except UnicodeEncodeError:
             if original_errors in (None, 'surrogate_then_replace'):
+                # We should only reach this if encoding was non-utf8 original_errors was
+                # surrogate_then_escape and errors was surrogateescape
+
                 # Slow but works
                 return_string = obj.encode('utf-8', 'surrogateescape')
                 return_string = return_string.decode('utf-8', 'replace')
diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py
--- a/lib/ansible/module_utils/basic.py
+++ b/lib/ansible/module_utils/basic.py
@@ -1011,7 +1011,7 @@ def selinux_context(self, path):
         return context
 
     def user_and_group(self, path, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
         st = os.lstat(b_path)
@@ -1105,10 +1105,10 @@ def set_context_if_different(self, path, context, changed, diff=None):
         return changed
 
     def set_owner_if_different(self, path, owner, changed, diff=None, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
         if owner is None:
             return changed
         orig_uid, orig_gid = self.user_and_group(path, expand)
@@ -1139,10 +1139,10 @@ def set_owner_if_different(self, path, owner, changed, diff=None, expand=True):
         return changed
 
     def set_group_if_different(self, path, group, changed, diff=None, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
         if group is None:
             return changed
         orig_uid, orig_gid = self.user_and_group(b_path, expand)
@@ -1173,10 +1173,10 @@ def set_group_if_different(self, path, group, changed, diff=None, expand=True):
         return changed
 
     def set_mode_if_different(self, path, mode, changed, diff=None, expand=True):
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
         path_stat = os.lstat(b_path)
 
         if mode is None:
@@ -1253,10 +1253,10 @@ def set_attributes_if_different(self, path, attributes, changed, diff=None, expa
         if attributes is None:
             return changed
 
-        b_path = to_bytes(path, errors='surrogate_then_strict')
+        b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_then_strict')
+        path = to_text(b_path, errors='surrogate_or_strict')
 
         existing = self.get_file_attributes(b_path)
 
diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py
--- a/lib/ansible/plugins/filter/core.py
+++ b/lib/ansible/plugins/filter/core.py
@@ -262,7 +262,7 @@ def get_hash(data, hashtype='sha1'):
     except:
         return None
 
-    h.update(to_bytes(data, errors='surrogate_then_strict'))
+    h.update(to_bytes(data, errors='surrogate_or_strict'))
     return h.hexdigest()
 
 
@@ -453,11 +453,11 @@ def do_groupby(environment, value, attribute):
 
 
 def b64encode(string):
-    return to_text(base64.b64encode(to_bytes(string, errors='surrogate_then_strict')))
+    return to_text(base64.b64encode(to_bytes(string, errors='surrogate_or_strict')))
 
 
 def b64decode(string):
-    return to_text(base64.b64decode(to_bytes(string, errors='surrogate_then_strict')))
+    return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict')))
 
 
 class FilterModule(object):


Test patch: