Repository: mitmproxy/mitmproxy

Issue:
Error in Auto Content viewer when parsing a JPEG
##### Steps to reproduce the problem:

1. Download those JPEGs:
```
https://img.lemde.fr/2018/10/09/1/0/3008/2005/184/122/60/0/4991a85_fz3CllPV3V0pbxyjvHkXiu7G.jpg
https://img.lemde.fr/2018/10/09/0/56/4770/3180/184/122/60/0/2616eb7_ZEmpnu_LH1c1xeTNOJ5uM5uh.jpg

```
Got this error:
```
ERROR:  Auto Content viewer failed: 	ERROR			
Traceback (most recent call last):	ERROR			
  File "mitmproxy/contentviews/__init__.py", line 130, in get_content_view	ERROR			
  File "mitmproxy/contentviews/auto.py", line 17, in __call__	ERROR			
  File "mitmproxy/contentviews/image/view.py", line 36, in __call__	ERROR			
  File "mitmproxy/contentviews/image/image_parser.py", line 78, in parse_jpeg	ERROR			
  File "mitmproxy/contrib/kaitaistruct/exif_be.py", line 572, in ifd0	ERROR			
  File "mitmproxy/contrib/kaitaistruct/exif_be.py", line 31, in __init__	ERROR			
  File "mitmproxy/contrib/kaitaistruct/exif_be.py", line 522, in __init__	ERROR			
  File "enum.py", line 291, in __call__	ERROR			
  File "enum.py", line 533, in __new__	ERROR			
  File "enum.py", line 546, in _missing_	ERROR			
ValueError: 9 is not a valid FieldTypeEnum	ERROR	
```


##### Any other comments? What have you tried so far?
Not sure if those JPEG are malformed, and how mitmproxy should handle this case


##### System information

<!-- Paste the output of "mitmproxy --version" here. -->
Mitmproxy: 4.0.4 binary
Python:    3.6.5
OpenSSL:   OpenSSL 1.1.0h  27 Mar 2018
Platform:  Darwin-17.7.0-x86_64-i386-64bit

<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->



Discussion:
Thanks for the report. We're using the excellent Kaitai Struct to parse images, it looks like their JPEG definition doesn't work for these files. This is an excellent way for first time contributors to help improving mitmproxy:

1. Fix the kaitai struct definition (https://github.com/kaitai-io/kaitai_struct_formats/blob/master/image/jpeg.ksy) to accept the JPEGs. Not sure what field type 9 is, that needs to be investigated. :-)
2. Recompile the ksy file and update https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/contrib/kaitaistruct/jpeg.py
3. Submit a PR to kaitai and mitmproxy!


I mirrored the first picture at https://uploads.hi.ls/2018-10/4991a85_fz3CllPV3V0pbxyjvHkXiu7G.jpg in case it goes down.
I have started working on this and would update my findings in some time.
@prateek1192 Working on this?

Resolving patch:
diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py
--- a/mitmproxy/contrib/kaitaistruct/exif_be.py
+++ b/mitmproxy/contrib/kaitaistruct/exif_be.py
@@ -1,12 +1,8 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
 
-import array
-import struct
-import zlib
-from enum import Enum
 from pkg_resources import parse_version
-
 from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
+from enum import Enum
 
 
 if parse_version(ks_version) < parse_version('0.7'):
@@ -17,6 +13,9 @@ def __init__(self, _io, _parent=None, _root=None):
         self._io = _io
         self._parent = _parent
         self._root = _root if _root else self
+        self._read()
+
+    def _read(self):
         self.version = self._io.read_u2be()
         self.ifd0_ofs = self._io.read_u4be()
 
@@ -25,6 +24,9 @@ def __init__(self, _io, _parent=None, _root=None):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
             self.num_fields = self._io.read_u2be()
             self.fields = [None] * (self.num_fields)
             for i in range(self.num_fields):
@@ -54,6 +56,9 @@ class FieldTypeEnum(Enum):
             word = 3
             dword = 4
             rational = 5
+            undefined = 7
+            slong = 9
+            srational = 10
 
         class TagEnum(Enum):
             image_width = 256
@@ -518,6 +523,9 @@ def __init__(self, _io, _parent=None, _root=None):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
             self.tag = self._root.IfdField.TagEnum(self._io.read_u2be())
             self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2be())
             self.length = self._io.read_u4be()
@@ -552,7 +560,7 @@ def data(self):
             if hasattr(self, '_m_data'):
                 return self._m_data if hasattr(self, '_m_data') else None
 
-            if not self.is_immediate_data:
+            if not (self.is_immediate_data):
                 io = self._root._io
                 _pos = io.pos()
                 io.seek(self.ofs_or_data)
diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py
--- a/mitmproxy/contrib/kaitaistruct/exif_le.py
+++ b/mitmproxy/contrib/kaitaistruct/exif_le.py
@@ -1,12 +1,8 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
 
-import array
-import struct
-import zlib
-from enum import Enum
 from pkg_resources import parse_version
-
 from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
+from enum import Enum
 
 
 if parse_version(ks_version) < parse_version('0.7'):
@@ -17,6 +13,9 @@ def __init__(self, _io, _parent=None, _root=None):
         self._io = _io
         self._parent = _parent
         self._root = _root if _root else self
+        self._read()
+
+    def _read(self):
         self.version = self._io.read_u2le()
         self.ifd0_ofs = self._io.read_u4le()
 
@@ -25,6 +24,9 @@ def __init__(self, _io, _parent=None, _root=None):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
             self.num_fields = self._io.read_u2le()
             self.fields = [None] * (self.num_fields)
             for i in range(self.num_fields):
@@ -54,6 +56,9 @@ class FieldTypeEnum(Enum):
             word = 3
             dword = 4
             rational = 5
+            undefined = 7
+            slong = 9
+            srational = 10
 
         class TagEnum(Enum):
             image_width = 256
@@ -518,6 +523,9 @@ def __init__(self, _io, _parent=None, _root=None):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
             self.tag = self._root.IfdField.TagEnum(self._io.read_u2le())
             self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2le())
             self.length = self._io.read_u4le()
@@ -552,7 +560,7 @@ def data(self):
             if hasattr(self, '_m_data'):
                 return self._m_data if hasattr(self, '_m_data') else None
 
-            if not self.is_immediate_data:
+            if not (self.is_immediate_data):
                 io = self._root._io
                 _pos = io.pos()
                 io.seek(self.ofs_or_data)


Test patch: