Repository: gitpython-developers/GitPython

Issue:
Diff::a_path is wrong for unicode characters
The problem is slightly mentioned in #865.

Let's consider the following example:
```shell
git init foo
cd ./foo
echo a > a
git add a
git commit -a -m 'Add a'
echo "foo" > špatně.txt
git add špatně.txt
git commit -a -m Add
```

```python3
In [4]: Repo('.').commit().diff('HEAD~')[0].a_path
Out[4]: '"\\305\\241patn\\304\\233.txt"'
```

Unless one uses `git config --global core.quotepath false`, the file name is quoted and I don't see a simple solution how to decode it to unicode? One can use `-z` option that can resolve the problem:

```
git diff HEAD~ --raw -z | cat
:000000 100644 0000000 257cc56 Ašpatně.txt
```

Discussion:

Patch:
diff --git a/git/diff.py b/git/diff.py
index 0fc30b9e..a9dc4b57 100644
--- a/git/diff.py
+++ b/git/diff.py
@@ -108,6 +108,7 @@ class Diffable(object):
             args.append("-p")
         else:
             args.append("--raw")
+            args.append("-z")
 
         # in any way, assure we don't see colored output,
         # fixes https://github.com/gitpython-developers/GitPython/issues/172
@@ -478,55 +479,55 @@ class Diff(object):
 
         index = DiffIndex()
 
-        def handle_diff_line(line):
-            line = line.decode(defenc)
-            if not line.startswith(":"):
-                return
-
-            meta, _, path = line[1:].partition('\t')
-            old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4)
-            # Change type can be R100
-            # R: status letter
-            # 100: score (in case of copy and rename)
-            change_type = _change_type[0]
-            score_str = ''.join(_change_type[1:])
-            score = int(score_str) if score_str.isdigit() else None
-            path = path.strip()
-            a_path = path.encode(defenc)
-            b_path = path.encode(defenc)
-            deleted_file = False
-            new_file = False
-            copied_file = False
-            rename_from = None
-            rename_to = None
-
-            # NOTE: We cannot conclude from the existence of a blob to change type
-            # as diffs with the working do not have blobs yet
-            if change_type == 'D':
-                b_blob_id = None
-                deleted_file = True
-            elif change_type == 'A':
-                a_blob_id = None
-                new_file = True
-            elif change_type == 'C':
-                copied_file = True
-                a_path, b_path = path.split('\t', 1)
-                a_path = a_path.encode(defenc)
-                b_path = b_path.encode(defenc)
-            elif change_type == 'R':
-                a_path, b_path = path.split('\t', 1)
-                a_path = a_path.encode(defenc)
-                b_path = b_path.encode(defenc)
-                rename_from, rename_to = a_path, b_path
-            elif change_type == 'T':
-                # Nothing to do
-                pass
-            # END add/remove handling
-
-            diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode,
-                        new_file, deleted_file, copied_file, rename_from, rename_to,
-                        '', change_type, score)
-            index.append(diff)
+        def handle_diff_line(lines):
+            lines = lines.decode(defenc)
+
+            for line in lines.split(':')[1:]:
+                meta, _, path = line.partition('\x00')
+                path = path.rstrip('\x00')
+                old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4)
+                # Change type can be R100
+                # R: status letter
+                # 100: score (in case of copy and rename)
+                change_type = _change_type[0]
+                score_str = ''.join(_change_type[1:])
+                score = int(score_str) if score_str.isdigit() else None
+                path = path.strip()
+                a_path = path.encode(defenc)
+                b_path = path.encode(defenc)
+                deleted_file = False
+                new_file = False
+                copied_file = False
+                rename_from = None
+                rename_to = None
+
+                # NOTE: We cannot conclude from the existence of a blob to change type
+                # as diffs with the working do not have blobs yet
+                if change_type == 'D':
+                    b_blob_id = None
+                    deleted_file = True
+                elif change_type == 'A':
+                    a_blob_id = None
+                    new_file = True
+                elif change_type == 'C':
+                    copied_file = True
+                    a_path, b_path = path.split('\x00', 1)
+                    a_path = a_path.encode(defenc)
+                    b_path = b_path.encode(defenc)
+                elif change_type == 'R':
+                    a_path, b_path = path.split('\x00', 1)
+                    a_path = a_path.encode(defenc)
+                    b_path = b_path.encode(defenc)
+                    rename_from, rename_to = a_path, b_path
+                elif change_type == 'T':
+                    # Nothing to do
+                    pass
+                # END add/remove handling
+
+                diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode,
+                            new_file, deleted_file, copied_file, rename_from, rename_to,
+                            '', change_type, score)
+                index.append(diff)
 
         handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False)
 


Test patch:
diff --git a/test/test_diff.py b/test/test_diff.py
index 378a58de..c6c9b67a 100644
--- a/test/test_diff.py
+++ b/test/test_diff.py
@@ -26,6 +26,10 @@ from test.lib import with_rw_directory
 import os.path as osp
 
 
+def to_raw(input):
+    return input.replace(b'\t', b'\x00')
+
+
 @ddt.ddt
 class TestDiff(TestBase):
 
@@ -112,7 +116,7 @@ class TestDiff(TestBase):
         self.assertEqual(diff.raw_rename_to, b'm\xc3\xbcller')
         assert isinstance(str(diff), str)
 
-        output = StringProcessAdapter(fixture('diff_rename_raw'))
+        output = StringProcessAdapter(to_raw(fixture('diff_rename_raw')))
         diffs = Diff._index_from_raw_format(self.rorepo, output)
         self.assertEqual(len(diffs), 1)
         diff = diffs[0]
@@ -137,7 +141,7 @@ class TestDiff(TestBase):
         self.assertTrue(diff.b_path, 'test2.txt')
         assert isinstance(str(diff), str)
 
-        output = StringProcessAdapter(fixture('diff_copied_mode_raw'))
+        output = StringProcessAdapter(to_raw(fixture('diff_copied_mode_raw')))
         diffs = Diff._index_from_raw_format(self.rorepo, output)
         self.assertEqual(len(diffs), 1)
         diff = diffs[0]
@@ -165,7 +169,7 @@ class TestDiff(TestBase):
         self.assertIsNotNone(diff.new_file)
         assert isinstance(str(diff), str)
 
-        output = StringProcessAdapter(fixture('diff_change_in_type_raw'))
+        output = StringProcessAdapter(to_raw(fixture('diff_change_in_type_raw')))
         diffs = Diff._index_from_raw_format(self.rorepo, output)
         self.assertEqual(len(diffs), 1)
         diff = diffs[0]
@@ -175,7 +179,7 @@ class TestDiff(TestBase):
         self.assertEqual(len(list(diffs.iter_change_type('T'))), 1)
 
     def test_diff_of_modified_files_not_added_to_the_index(self):
-        output = StringProcessAdapter(fixture('diff_abbrev-40_full-index_M_raw_no-color'))
+        output = StringProcessAdapter(to_raw(fixture('diff_abbrev-40_full-index_M_raw_no-color')))
         diffs = Diff._index_from_raw_format(self.rorepo, output)
 
         self.assertEqual(len(diffs), 1, 'one modification')