Repository: ishepard/pydriller

Issue:
Return undecoded content 
We currently try to decode everything. Of course we silently fail in case the file is a binary, pdf, or whatever can not be decoded to UTF-8. 
Instead of silently failing, we could return an undecoded version of the file.

Discussion:

Patch:
diff --git a/pydriller/domain/commit.py b/pydriller/domain/commit.py
index d8dcf46..ba2c6e5 100644
--- a/pydriller/domain/commit.py
+++ b/pydriller/domain/commit.py
@@ -157,7 +157,7 @@ class ModifiedFile:
             old_path: Optional[str],
             new_path: Optional[str],
             change_type: ModificationType,
-            diff_and_sc: Dict[str, str],
+            diff_and_content: Dict[str, str],
     ):
         """
         Initialize a modified file. A modified file carries on information
@@ -167,11 +167,9 @@ class ModifiedFile:
         self._old_path = Path(old_path) if old_path is not None else None
         self._new_path = Path(new_path) if new_path is not None else None
         self.change_type = change_type
-        self.diff = diff_and_sc["diff"]
-        self.__source_code = diff_and_sc["source_code"]
-        self.__source_code_before = diff_and_sc["source_code_before"]
-        self.content = diff_and_sc["content"]
-        self.content_before = diff_and_sc["content_before"]
+        self.diff = diff_and_content["diff"]
+        self.content = diff_and_content["content"]
+        self.content_before = diff_and_content["content_before"]
 
         self._nloc = None
         self._complexity = None
@@ -192,12 +190,18 @@ class ModifiedFile:
     @property
     def source_code(self):
         warn('The use of `source_code` is deprecated. Use `content` instead.', DeprecationWarning, stacklevel=2)
-        return self.__source_code
+        if type(self.content) == bytes:
+            return self._get_decoded_content(self.content)
+
+        return None
 
     @property
     def source_code_before(self):
         warn('The use of `source_code_before` is deprecated. Use `content_before` instead.', DeprecationWarning, stacklevel=2)
-        return self.__source_code_before
+        if type(self.content_before) == bytes:
+            return self._get_decoded_content(self.content_before)
+
+        return None
 
     @property
     def added_lines(self) -> int:
@@ -475,6 +479,13 @@ class ModifiedFile:
 
             self._function_list_before = [Method(x) for x in anal.function_list]
 
+    def _get_decoded_content(self, content):
+        try:
+            return content.decode("utf-8", "ignore")
+        except (AttributeError, ValueError):
+            logger.debug("Could not load the content for file %s", self.filename)
+            return None
+
     def __eq__(self, other):
         if not isinstance(other, ModifiedFile):
             return NotImplemented
@@ -723,16 +734,14 @@ class Commit:
             new_path = diff.b_path
             change_type = self._from_change_to_modification_type(diff)
 
-            diff_and_sc = {
+            diff_and_content = {
                 "diff": self._get_decoded_str(diff.diff),
-                "source_code_before": self._get_decoded_sc_str(diff.a_blob),
-                "source_code": self._get_decoded_sc_str(diff.b_blob),
-                "content_before": self._get_decoded_sc_str(diff.a_blob),
-                "content": self._get_decoded_sc_str(diff.b_blob),
+                "content_before": self._get_undecoded_content(diff.a_blob),
+                "content": self._get_undecoded_content(diff.b_blob),
             }
 
             modified_files_list.append(
-                ModifiedFile(old_path, new_path, change_type, diff_and_sc)
+                ModifiedFile(old_path, new_path, change_type, diff_and_content)
             )
 
         return modified_files_list
@@ -747,15 +756,8 @@ class Commit:
             )
             return None
 
-    def _get_decoded_sc_str(self, diff):
-        try:
-            return diff.data_stream.read().decode("utf-8", "ignore")
-        except (AttributeError, ValueError):
-            logger.debug(
-                "Could not load source code of a " "file in commit %s",
-                self._c_object.hexsha,
-            )
-            return None
+    def _get_undecoded_content(self, diff):
+        return diff.data_stream.read() if diff is not None else None
 
     @property
     def in_main_branch(self) -> bool:


Test patch:
diff --git a/tests/test_commit.py b/tests/test_commit.py
index 0e89dcf..daef686 100644
--- a/tests/test_commit.py
+++ b/tests/test_commit.py
@@ -44,10 +44,8 @@ def test_equal(repo: Git):
 def test_filename():
     diff_and_sc = {
         'diff': '',
-        'source_code': '',
-        'source_code_before': '',
-        'content': '',
-        'content_before': ''
+        'content': b'',
+        'content_before': b''
     }
     m1 = ModifiedFile('dspadini/pydriller/myfile.py',
                       'dspadini/pydriller/mynewfile.py',
@@ -66,15 +64,13 @@ def test_filename():
 
 
 def test_metrics_python():
-    with open('test-repos/lizard/git_repository.py') as f:
-        sc = f.read()
+    with open('test-repos/lizard/git_repository.py', 'rb') as f:
+        content = f.read()
 
     diff_and_sc = {
         'diff': '',
-        'source_code': sc,
-        'source_code_before': sc,
-        'content': sc,
-        'content_before': sc
+        'content': content,
+        'content_before': content
     }
 
     m1 = ModifiedFile('test-repos/lizard/git_repository.py',
@@ -142,15 +138,13 @@ def test_changed_methods():
 
 
 def test_metrics_cpp():
-    with open('test-repos/lizard/FileCPP.cpp') as f:
-        sc = f.read()
+    with open('test-repos/lizard/FileCPP.cpp', 'rb') as f:
+        content = f.read()
 
     diff_and_sc = {
         'diff': '',
-        'source_code': sc,
-        'source_code_before': sc,
-        'content': sc,
-        'content_before': sc
+        'content': content,
+        'content_before': content
     }
 
     m1 = ModifiedFile('test-repos/lizard/FileCPP.cpp',
@@ -165,15 +159,13 @@ def test_metrics_cpp():
 
 
 def test_metrics_java():
-    with open('test-repos/lizard/FileJava.java') as f:
-        sc = f.read()
+    with open('test-repos/lizard/FileJava.java', 'rb') as f:
+        content = f.read()
 
     diff_and_sc = {
         'diff': '',
-        'source_code': sc,
-        'source_code_before': sc,
-        'content': sc,
-        'content_before': sc
+        'content': content,
+        'content_before': content
     }
 
     m1 = ModifiedFile('test-repos/lizard/FileJava.java',
@@ -188,14 +180,12 @@ def test_metrics_java():
 
 
 def test_metrics_not_supported_file():
-    sc = 'asd !&%@*&^@\n jjdkj'
+    content = b'asd !&%@*&^@\n jjdkj'
 
     diff_and_sc = {
         'diff': '',
-        'source_code': sc,
-        'source_code_before': sc,
-        'content': sc,
-        'content_before': sc
+        'content': content,
+        'content_before': content
     }
 
     m1 = ModifiedFile('test-repos/lizard/NotSupported.pdf',
@@ -319,7 +309,7 @@ def test_content_before_complete(repo: Git):
     m1 = repo.get_commit('ca1f75455f064410360bc56218d0418221cf9484').modified_files[0]
 
     with open('test-repos/source_code_before_commit/'
-              'sc_A_ca1f75455f064410360bc56218d0418221cf9484.txt') as f:
+              'sc_A_ca1f75455f064410360bc56218d0418221cf9484.txt', 'rb') as f:
         sc = f.read()
 
     assert m1.content == sc
@@ -328,7 +318,7 @@ def test_content_before_complete(repo: Git):
     old_sc = sc
     with open(
             'test-repos/source_code_before_commit/'
-            'sc_A_022ebf5fba835c6d95e99eaccc2d85b3db5a2ec0.txt') as f:
+            'sc_A_022ebf5fba835c6d95e99eaccc2d85b3db5a2ec0.txt', 'rb') as f:
         sc = f.read()
 
     m1 = repo.get_commit('022ebf5fba835c6d95e99eaccc2d85b3db5a2ec0').modified_files[0]
diff --git a/tests/test_git_repository.py b/tests/test_git_repository.py
index aaa370b..eeba081 100644
--- a/tests/test_git_repository.py
+++ b/tests/test_git_repository.py
@@ -244,7 +244,7 @@ def test_should_detail_a_commit(repo: Git):
 
     assert commit.modified_files[0].new_path == "Matricula.java"
     assert commit.modified_files[0].diff.startswith("@@ -0,0 +1,62 @@\n+package model;") is True
-    assert commit.modified_files[0].content.startswith("package model;") is True
+    assert commit.modified_files[0].content.decode().startswith("package model;") is True   # type: ignore[attr-defined]
 
     with catch_warnings(record=True) as w:
         assert commit.modified_files[0].source_code.startswith("package model;") is True