Repository: pre-commit/pre-commit

Issue:
Non-English directories are skipped
Looks like pre-commit skips non-English directories. Please take a look at those:

* https://github.com/GolangShow/golangshow.com/commit/df0529b104739ba8d1de9182d37dfb80bbddf679
* https://travis-ci.org/GolangShow/golangshow.com/builds/271948939

A relevant bit from the full Travis build output:
```
$ pre-commit run --all-files
[…]
Trim Trailing Whitespace.................................................Failed
hookid: trailing-whitespace
Files were modified by this hook. Additional output:
Fixing public/episode/2017/04-14-096/index.html
Fixing public/episode/2016/03-31-050/index.html
[…]
Fixing public/episode/2016/09-01-072/index.html
Fixing public/episode/2015/12-29-036/index.html
The command "pre-commit run --all-files" exited with 1.

$ git status
HEAD detached at df0529b
Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git checkout -- <file>..." to discard changes in working directory)
	modified:   "public/categories/\320\263\320\276\321\201\321\202\320\270/index.html"
	modified:   "public/categories/\320\263\320\276\321\201\321\202\320\270/index.xml"
	modified:   "public/categories/\320\270\320\275\321\202\320\265\321\200\320\262\321\214\321\216/index.html"
	modified:   "public/categories/\320\270\320\275\321\202\320\265\321\200\320\262\321\214\321\216/index.xml"
	modified:   "public/categories/\320\270\321\202\320\276\320\263\320\270/index.html"
	modified:   "public/categories/\320\270\321\202\320\276\320\263\320\270/index.xml"
	modified:   "public/categories/\320\272\320\260\320\274\320\270\320\275/index.html"
	modified:   "public/categories/\320\272\320\260\320\274\320\270\320\275/index.xml"
	modified:   "public/categories/\320\275\320\276\320\262\320\276\321\201\321\202\320\270/index.html"
	modified:   "public/categories/\320\275\320\276\320\262\320\276\321\201\321\202\320\270/index.xml"
	modified:   public/js/highlight.pack.js
	modified:   public/page/1/index.html
no changes added to commit (use "git add" and/or "git commit -a")
```

Those directories in `public/categories/` are `гости`, `итоги`, `камин`, `новости`, and `интервью`.


Discussion:
Thanks for the report!  I should be able to reproduce and fix this pretty easily, let me see if there's a workaround to get you going quicker first :)
Ah definitely a bug!  It has to do with our usage of `git ls-files` and should be easy to fix.

A workaround is to bypass the bug in `--all-files` (though not pretty): `git ls-files -z | xargs -0 pre-commit run --files`

I'll get a fix out for you asap!

Resolving patch:
diff --git a/pre_commit/commands/run.py b/pre_commit/commands/run.py
--- a/pre_commit/commands/run.py
+++ b/pre_commit/commands/run.py
@@ -36,13 +36,6 @@ def _hook_msg_start(hook, verbose):
     )
 
 
-def get_changed_files(new, old):
-    return cmd_output(
-        'git', 'diff', '--no-ext-diff', '--name-only',
-        '{}...{}'.format(old, new),
-    )[1].splitlines()
-
-
 def filter_filenames_by_types(filenames, types, exclude_types):
     types, exclude_types = frozenset(types), frozenset(exclude_types)
     ret = []
@@ -56,7 +49,7 @@ def filter_filenames_by_types(filenames, types, exclude_types):
 def get_filenames(args, include_expr, exclude_expr):
     if args.origin and args.source:
         getter = git.get_files_matching(
-            lambda: get_changed_files(args.origin, args.source),
+            lambda: git.get_changed_files(args.origin, args.source),
         )
     elif args.hook_stage == 'commit-msg':
         def getter(*_):
diff --git a/pre_commit/git.py b/pre_commit/git.py
--- a/pre_commit/git.py
+++ b/pre_commit/git.py
@@ -15,6 +15,14 @@
 logger = logging.getLogger('pre_commit')
 
 
+def zsplit(s):
+    s = s.strip('\0')
+    if s:
+        return s.split('\0')
+    else:
+        return []
+
+
 def get_root():
     try:
         return cmd_output('git', 'rev-parse', '--show-toplevel')[1].strip()
@@ -67,25 +75,32 @@ def get_conflicted_files():
     # If they resolved the merge conflict by choosing a mesh of both sides
     # this will also include the conflicted files
     tree_hash = cmd_output('git', 'write-tree')[1].strip()
-    merge_diff_filenames = cmd_output(
-        'git', 'diff', '--no-ext-diff',
-        '-m', tree_hash, 'HEAD', 'MERGE_HEAD', '--name-only',
-    )[1].splitlines()
+    merge_diff_filenames = zsplit(cmd_output(
+        'git', 'diff', '--name-only', '--no-ext-diff', '-z',
+        '-m', tree_hash, 'HEAD', 'MERGE_HEAD',
+    )[1])
     return set(merge_conflict_filenames) | set(merge_diff_filenames)
 
 
 @memoize_by_cwd
 def get_staged_files():
-    return cmd_output(
-        'git', 'diff', '--staged', '--name-only', '--no-ext-diff',
+    return zsplit(cmd_output(
+        'git', 'diff', '--staged', '--name-only', '--no-ext-diff', '-z',
         # Everything except for D
         '--diff-filter=ACMRTUXB',
-    )[1].splitlines()
+    )[1])
 
 
 @memoize_by_cwd
 def get_all_files():
-    return cmd_output('git', 'ls-files')[1].splitlines()
+    return zsplit(cmd_output('git', 'ls-files', '-z')[1])
+
+
+def get_changed_files(new, old):
+    return zsplit(cmd_output(
+        'git', 'diff', '--name-only', '--no-ext-diff', '-z',
+        '{}...{}'.format(old, new),
+    )[1])
 
 
 def get_files_matching(all_file_list_strategy):


Test patch:
diff --git a/tests/commands/run_test.py b/tests/commands/run_test.py
--- a/tests/commands/run_test.py
+++ b/tests/commands/run_test.py
@@ -15,7 +15,6 @@
 from pre_commit.commands.run import _compute_cols
 from pre_commit.commands.run import _get_skips
 from pre_commit.commands.run import _has_unmerged_paths
-from pre_commit.commands.run import get_changed_files
 from pre_commit.commands.run import run
 from pre_commit.runner import Runner
 from pre_commit.util import cmd_output
@@ -501,18 +500,6 @@ def test_hook_install_failure(mock_out_store_directory, tempdir_factory):
         assert '☃'.encode('UTF-8') + '²'.encode('latin1') in stdout
 
 
-def test_get_changed_files():
-    files = get_changed_files(
-        '78c682a1d13ba20e7cb735313b9314a74365cd3a',
-        '3387edbb1288a580b37fe25225aa0b856b18ad1a',
-    )
-    assert files == ['CHANGELOG.md', 'setup.py']
-
-    # files changed in source but not in origin should not be returned
-    files = get_changed_files('HEAD~10', 'HEAD')
-    assert files == []
-
-
 def test_lots_of_files(mock_out_store_directory, tempdir_factory):
     # windows xargs seems to have a bug, here's a regression test for
     # our workaround
diff --git a/tests/git_test.py b/tests/git_test.py
--- a/tests/git_test.py
+++ b/tests/git_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
@@ -162,3 +163,63 @@ def test_get_conflicted_files_unstaged_files(in_merge_conflict):
 def test_parse_merge_msg_for_conflicts(input, expected_output):
     ret = git.parse_merge_msg_for_conflicts(input)
     assert ret == expected_output
+
+
+def test_get_changed_files():
+    files = git.get_changed_files(
+        '78c682a1d13ba20e7cb735313b9314a74365cd3a',
+        '3387edbb1288a580b37fe25225aa0b856b18ad1a',
+    )
+    assert files == ['CHANGELOG.md', 'setup.py']
+
+    # files changed in source but not in origin should not be returned
+    files = git.get_changed_files('HEAD~10', 'HEAD')
+    assert files == []
+
+
+@pytest.mark.parametrize(
+    ('s', 'expected'),
+    (
+        ('foo\0bar\0', ['foo', 'bar']),
+        ('foo\0', ['foo']),
+        ('', []),
+        ('foo', ['foo']),
+    ),
+)
+def test_zsplit(s, expected):
+    assert git.zsplit(s) == expected
+
+
+@pytest.fixture
+def non_ascii_repo(tmpdir):
+    repo = tmpdir.join('repo').ensure_dir()
+    with repo.as_cwd():
+        cmd_output('git', 'init', '.')
+        cmd_output('git', 'commit', '--allow-empty', '-m', 'initial commit')
+        repo.join('интервью').ensure()
+        cmd_output('git', 'add', '.')
+        cmd_output('git', 'commit', '--allow-empty', '-m', 'initial commit')
+        yield repo
+
+
+def test_all_files_non_ascii(non_ascii_repo):
+    ret = git.get_all_files()
+    assert ret == ['интервью']
+
+
+def test_staged_files_non_ascii(non_ascii_repo):
+    non_ascii_repo.join('интервью').write('hi')
+    cmd_output('git', 'add', '.')
+    assert git.get_staged_files() == ['интервью']
+
+
+def test_changed_files_non_ascii(non_ascii_repo):
+    ret = git.get_changed_files('HEAD', 'HEAD^')
+    assert ret == ['интервью']
+
+
+def test_get_conflicted_files_non_ascii(in_merge_conflict):
+    open('интервью', 'a').close()
+    cmd_output('git', 'add', '.')
+    ret = git.get_conflicted_files()
+    assert ret == {'conflict_file', 'интервью'}