Repository: ansible/ansible

Issue:
Crash with file module when directories have non utf-8 files
##### Bug Report

##### COMPONENT NAME

file

##### ANSIBLE VERSION

```
ansible 2.4.0 (recursive-crash-non-utf8 78836ec0b9) last updated 2017/04/21 15:31:54 (GMT +200)
  config file = /etc/ansible/ansible.cfg
  python version = 2.7.12 (default, Nov 19 2016, 06:48:10) [GCC 5.4.0 20160609]
```

##### CONFIGURATION

Nothing special

##### OS / ENVIRONMENT

Nothing special

##### SUMMARY

When launching file module with recurse = yes on directories with non utf-8 files, Ansible crash with the following message:

```
LookupError: unknown error handler name 'surrogate_then_strict'
```
##### STEPS TO REPRODUCE

You can retrieve an example at the following location: https://github.com/Yannig/yannig-ansible-playbooks/tree/master/file-encoding

```yaml
- name: "Recursive change with non utf-8 filename"
  hosts: localhost
  gather_facts: no
  tasks:
    - name: "Recursive change (crash)"
      file: path=dir mode=0755 recurse=yes
```

##### EXPECTED RESULTS

Playbook apply change without crash

##### ACTUAL RESULTS

```
exception:
Traceback (most recent call last):
  File "/tmp/ansible_atAUEE/ansible_module_file.py", line 480, in <module>
    main()
  File "/tmp/ansible_atAUEE/ansible_module_file.py", line 362, in main
    changed |= recursive_set_attributes(module, to_bytes(file_args['path'], errors='surrogate_or_strict'), follow, file_args)
  File "/tmp/ansible_atAUEE/ansible_module_file.py", line 171, in recursive_set_attributes
    changed |= module.set_fs_attributes_if_different(tmp_file_args, changed)
  File "/tmp/ansible_atAUEE/ansible_modlib.zip/ansible/module_utils/basic.py", line 1292, in set_fs_attributes_if_different
  File "/tmp/ansible_atAUEE/ansible_modlib.zip/ansible/module_utils/basic.py", line 995, in set_owner_if_different
  File "/tmp/ansible_atAUEE/ansible_modlib.zip/ansible/module_utils/_text.py", line 232, in to_text
  File "/usr/lib/python2.7/encodings/utf_8.py", line 16, in decode
    return codecs.utf_8_decode(input, errors, True)
LookupError: unknown error handler name 'surrogate_then_strict'
```



Discussion:
This issue affect also Ansible 2.3.1.
Is there any way to have a feedback on this issue?

Resolving patch:
diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py
--- a/lib/ansible/module_utils/basic.py
+++ b/lib/ansible/module_utils/basic.py
@@ -1113,19 +1113,19 @@ def set_owner_if_different(self, path, owner, changed, diff=None, expand=True):
         b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_or_strict')
         if owner is None:
             return changed
-        orig_uid, orig_gid = self.user_and_group(path, expand)
+        orig_uid, orig_gid = self.user_and_group(b_path, expand)
         try:
             uid = int(owner)
         except ValueError:
             try:
                 uid = pwd.getpwnam(owner).pw_uid
             except KeyError:
+                path = to_text(b_path)
                 self.fail_json(path=path, msg='chown failed: failed to look up user %s' % owner)
-        if orig_uid != uid:
 
+        if orig_uid != uid:
             if diff is not None:
                 if 'before' not in diff:
                     diff['before'] = {}
@@ -1139,6 +1139,7 @@ def set_owner_if_different(self, path, owner, changed, diff=None, expand=True):
             try:
                 os.lchown(b_path, uid, -1)
             except OSError:
+                path = to_text(b_path)
                 self.fail_json(path=path, msg='chown failed')
             changed = True
         return changed
@@ -1147,7 +1148,6 @@ def set_group_if_different(self, path, group, changed, diff=None, expand=True):
         b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_or_strict')
         if group is None:
             return changed
         orig_uid, orig_gid = self.user_and_group(b_path, expand)
@@ -1157,9 +1157,10 @@ def set_group_if_different(self, path, group, changed, diff=None, expand=True):
             try:
                 gid = grp.getgrnam(group).gr_gid
             except KeyError:
+                path = to_text(b_path)
                 self.fail_json(path=path, msg='chgrp failed: failed to look up group %s' % group)
-        if orig_gid != gid:
 
+        if orig_gid != gid:
             if diff is not None:
                 if 'before' not in diff:
                     diff['before'] = {}
@@ -1173,6 +1174,7 @@ def set_group_if_different(self, path, group, changed, diff=None, expand=True):
             try:
                 os.lchown(b_path, -1, gid)
             except OSError:
+                path = to_text(b_path)
                 self.fail_json(path=path, msg='chgrp failed')
             changed = True
         return changed
@@ -1181,7 +1183,6 @@ def set_mode_if_different(self, path, mode, changed, diff=None, expand=True):
         b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_or_strict')
         path_stat = os.lstat(b_path)
 
         if mode is None:
@@ -1195,12 +1196,14 @@ def set_mode_if_different(self, path, mode, changed, diff=None, expand=True):
                     mode = self._symbolic_mode_to_octal(path_stat, mode)
                 except Exception:
                     e = get_exception()
+                    path = to_text(b_path)
                     self.fail_json(path=path,
                                    msg="mode must be in octal or symbolic form",
                                    details=str(e))
 
                 if mode != stat.S_IMODE(mode):
                     # prevent mode from having extra info orbeing invalid long number
+                    path = to_text(b_path)
                     self.fail_json(path=path, msg="Invalid mode supplied, only permission info is allowed", details=mode)
 
         prev_mode = stat.S_IMODE(path_stat.st_mode)
@@ -1244,6 +1247,7 @@ def set_mode_if_different(self, path, mode, changed, diff=None, expand=True):
                     raise e
             except Exception:
                 e = get_exception()
+                path = to_text(b_path)
                 self.fail_json(path=path, msg='chmod failed', details=str(e))
 
             path_stat = os.lstat(b_path)
@@ -1261,7 +1265,6 @@ def set_attributes_if_different(self, path, attributes, changed, diff=None, expa
         b_path = to_bytes(path, errors='surrogate_or_strict')
         if expand:
             b_path = os.path.expanduser(os.path.expandvars(b_path))
-        path = to_text(b_path, errors='surrogate_or_strict')
 
         existing = self.get_file_attributes(b_path)
 
@@ -1286,6 +1289,7 @@ def set_attributes_if_different(self, path, attributes, changed, diff=None, expa
                             raise Exception("Error while setting attributes: %s" % (out + err))
                     except:
                         e = get_exception()
+                        path = to_text(b_path)
                         self.fail_json(path=path, msg='chattr failed', details=str(e))
         return changed
 


Test patch: