Repository: python-pillow/Pillow

Issue:
TIFF_LZW decompression is incorrect for  16bpc image
Pillow was refusing to open this TIFF in version 4.2.1. In version 4.3.0, it loads it, but then saving it in any format including TIFF produces a silent corruption. Python 2.7.14 in both cases.

This is the reproducing script:
```
from PIL import Image                                                                                                                      

# https://s3-external-1.amazonaws.com/prod-elife-published/articles/32330/elife-32330-fig1-v1.tif 
src_fp = 'elife-32330-fig1-v1.tif'
im = Image.open(src_fp)
target_fp = open('result.tif', 'w')
im.save(target_fp)
```

In 4.2.1:
```
  File "reproducing.py", line 5, in <module>
    im = Image.open(src_fp)
  File "/opt/loris/venv/local/lib/python2.7/site-packages/PIL/Image.py", line 2519, in open
    % (filename if filename else fp))
IOError: cannot identify image file 'elife-32330-fig1-v1.tif'
```

In 4.3.0, no output, file gets stored and corrupted.
[elife-32330-fig1-v1.tif.gz](https://github.com/python-pillow/Pillow/files/1450479/elife-32330-fig1-v1.tif.gz)
[result.tif.gz](https://github.com/python-pillow/Pillow/files/1450480/result.tif.gz)
![converting to PNG](https://user-images.githubusercontent.com/160299/32500515-918e5826-c3cd-11e7-80c2-5995952cf130.png)

I tried to track how the `Image` class changed from 4.2.1 to 4.3.0, but there were too many changes to identify the source. I don't expect it to start to work, but to continue throwing an exception if it can't be read safely.




Discussion:
If you use libtiff instead of the internal tiff decoder, it works:
```
from PIL import Image, TiffImagePlugin
TiffImagePlugin.READ_LIBTIFF=True

im = Image.open('issue_2839.tif')
im.show()
```

This image is a 16bit RGBA image, with tiff_lzw compression.  Pillow 4.3.0 added support for the 16 bit RGBA type, though the output image is a lower precision 8 bit per channel RGBA image. Prior to that, there was _no_ support for this image. 

Pillow (and PIL before it) have had internal support for tiff_lzw compression, but apparently it's either buggy or cannot handle this particular image mode. 

Forcing the TiffImagePlugin to use the libtiff decoder allows Pillow to properly decode this image. 

Resolving patch:
diff --git a/PIL/TiffImagePlugin.py b/PIL/TiffImagePlugin.py
--- a/PIL/TiffImagePlugin.py
+++ b/PIL/TiffImagePlugin.py
@@ -1029,21 +1029,8 @@ def _decoder(self, rawmode, layer, tile=None):
         compression = self._compression
         if compression == "raw":
             args = (rawmode, 0, 1)
-        elif compression == "jpeg":
-            args = rawmode, ""
-            if JPEGTABLES in self.tag_v2:
-                # Hack to handle abbreviated JPEG headers
-                # Definition of JPEGTABLES is that the count
-                # is the number of bytes in the tables datastream
-                # so, it should always be 1 in our tag info
-                self.tile_prefix = self.tag_v2[JPEGTABLES]
         elif compression == "packbits":
             args = rawmode
-        elif compression == "tiff_lzw":
-            args = rawmode
-            if PREDICTOR in self.tag_v2:
-                # Section 14: Differencing Predictor
-                self.decoderconfig = (self.tag_v2[PREDICTOR],)
 
         return args
 
@@ -1244,14 +1231,7 @@ def _setup(self):
             offsets = self.tag_v2[STRIPOFFSETS]
             h = self.tag_v2.get(ROWSPERSTRIP, ysize)
             w = self.size[0]
-            if READ_LIBTIFF or self._compression in ["tiff_ccitt", "group3",
-                                                     "group4", "tiff_jpeg",
-                                                     "tiff_adobe_deflate",
-                                                     "tiff_thunderscan",
-                                                     "tiff_deflate",
-                                                     "tiff_sgilog",
-                                                     "tiff_sgilog24",
-                                                     "tiff_raw_16"]:
+            if READ_LIBTIFF or self._compression != 'raw':
                 # if DEBUG:
                 #     print("Activating g4 compression for whole file")
 
@@ -1285,8 +1265,13 @@ def _setup(self):
                 # we're expecting image byte order. So, if the rawmode
                 # contains I;16, we need to convert from native to image
                 # byte order.
-                if self.mode in ('I;16B', 'I;16') and 'I;16' in rawmode:
+                if rawmode == 'I;16':
                     rawmode = 'I;16N'
+                if ';16B' in rawmode:
+                    rawmode = rawmode.replace(';16B', ';16N')
+                if ';16L' in rawmode:
+                    rawmode = rawmode.replace(';16L', ';16N')
+
 
                 # Offset in the tile tuple is 0, we go from 0,0 to
                 # w,h, and we only do this once -- eds
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@
     "Blend", "Chops", "Convert", "ConvertYCbCr", "Copy", "Crc32", "Crop",
     "Dib", "Draw", "Effects", "EpsEncode", "File", "Fill", "Filter",
     "FliDecode", "Geometry", "GetBBox", "GifDecode", "GifEncode", "HexDecode",
-    "Histo", "JpegDecode", "JpegEncode", "LzwDecode", "Matrix", "ModeFilter",
+    "Histo", "JpegDecode", "JpegEncode", "Matrix", "ModeFilter",
     "Negative", "Offset", "Pack", "PackDecode", "Palette", "Paste", "Quant",
     "QuantOctree", "QuantHash", "QuantHeap", "PcdDecode", "PcxDecode",
     "PcxEncode", "Point", "RankFilter", "RawDecode", "RawEncode", "Storage",


Test patch:
diff --git a/Tests/helper.py b/Tests/helper.py
--- a/Tests/helper.py
+++ b/Tests/helper.py
@@ -14,11 +14,22 @@
 
 
 HAS_UPLOADER = False
-try:
-    import test_image_results
-    HAS_UPLOADER = True
-except ImportError:
-    pass
+
+if os.environ.get('SHOW_ERRORS', None):
+    # local img.show for errors. 
+    HAS_UPLOADER=True
+    class test_image_results:
+        @classmethod
+        def upload(self, a, b):
+            a.show()
+            b.show()
+else:
+    try:
+        import test_image_results
+        HAS_UPLOADER = True
+    except ImportError:
+        pass
+
 
 
 def convert_to_comparable(a, b):
@@ -99,11 +110,17 @@ def assert_image_equal(self, a, b, msg=None):
                 try:
                     url = test_image_results.upload(a, b)
                     logger.error("Url for test images: %s" % url)
-                except:
+                except Exception as msg:
                     pass
 
             self.fail(msg or "got different content")
 
+    def assert_image_equal_tofile(self, a, filename, msg=None, mode=None):
+        with Image.open(filename) as img:
+            if mode:
+                img = img.convert(mode)
+            self.assert_image_equal(a, img, msg)
+            
     def assert_image_similar(self, a, b, epsilon, msg=None):
         epsilon = float(epsilon)
         self.assertEqual(
@@ -136,6 +153,12 @@ def assert_image_similar(self, a, b, epsilon, msg=None):
                     pass
             raise e
 
+    def assert_image_similar_tofile(self, a, filename, epsilon, msg=None, mode=None):
+        with Image.open(filename) as img:
+            if mode:
+                img = img.convert(mode)
+            self.assert_image_similar(a, img, epsilon, msg)
+
     def assert_warning(self, warn_class, func, *args, **kwargs):
         import warnings
 
diff --git a/Tests/images/copyleft.png b/Tests/images/copyleft.png
new file mode 100644
Binary files /dev/null and b/Tests/images/copyleft.png differ
diff --git a/Tests/images/pil136.png b/Tests/images/pil136.png
new file mode 100644
Binary files /dev/null and b/Tests/images/pil136.png differ
diff --git a/Tests/images/pil168.png b/Tests/images/pil168.png
new file mode 100644
Binary files /dev/null and b/Tests/images/pil168.png differ
diff --git a/Tests/images/tiff_16bit_RGBa_target.png b/Tests/images/tiff_16bit_RGBa_target.png
new file mode 100644
Binary files /dev/null and b/Tests/images/tiff_16bit_RGBa_target.png differ
diff --git a/Tests/images/tiff_adobe_deflate.png b/Tests/images/tiff_adobe_deflate.png
new file mode 100644
Binary files /dev/null and b/Tests/images/tiff_adobe_deflate.png differ
diff --git a/Tests/test_file_libtiff.py b/Tests/test_file_libtiff.py
--- a/Tests/test_file_libtiff.py
+++ b/Tests/test_file_libtiff.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 from helper import unittest, PillowTestCase, hopper, py3
+from PIL import features
 
 from ctypes import c_float
 import io
@@ -15,9 +16,7 @@
 class LibTiffTestCase(PillowTestCase):
 
     def setUp(self):
-        codecs = dir(Image.core)
-
-        if "libtiff_encoder" not in codecs or "libtiff_decoder" not in codecs:
+        if not features.check('libtiff'):
             self.skipTest("tiff support not available")
 
     def _assert_noerr(self, im):
@@ -126,6 +125,8 @@ def test_adobe_deflate_tiff(self):
             im.tile[0][:3], ('tiff_adobe_deflate', (0, 0, 278, 374), 0))
         im.load()
 
+        self.assert_image_equal_tofile(im, 'Tests/images/tiff_adobe_deflate.png')
+
     def test_write_metadata(self):
         """ Test metadata writing through libtiff """
         for legacy_api in [False, True]:
@@ -310,12 +311,7 @@ def test_12bit_rawmode(self):
         # imagemagick will auto scale so that a 12bit FFF is 16bit FFF0,
         # so we need to unshift so that the integer values are the same.
 
-        im2 = Image.open('Tests/images/12in16bit.tif')
-
-        logger.debug("%s", [img.getpixel((0, idx))
-                            for img in [im, im2] for idx in range(3)])
-
-        self.assert_image_equal(im, im2)
+        self.assert_image_equal_tofile(im, 'Tests/images/12in16bit.tif')
 
     def test_blur(self):
         # test case from irc, how to do blur on b/w image
@@ -576,6 +572,51 @@ def test_save_tiff_with_jpegtables(self):
         # Should not raise UnicodeDecodeError or anything else
         im.save(outfile)
 
+    def test_16bit_RGBa_tiff(self):
+        im = Image.open("Tests/images/tiff_16bit_RGBa.tiff")
+
+        self.assertEqual(im.mode, "RGBA")
+        self.assertEqual(im.size, (100, 40))
+        self.assertEqual(im.tile, [('tiff_lzw', (0, 0, 100, 40), 0, ('RGBa;16N', 'tiff_lzw', False))])
+        im.load()
+
+        self.assert_image_equal_tofile(im, "Tests/images/tiff_16bit_RGBa_target.png")
+
+    def test_gimp_tiff(self):
+        # Read TIFF JPEG images from GIMP [@PIL168]
+
+        codecs = dir(Image.core)
+        if "jpeg_decoder" not in codecs:
+            self.skipTest("jpeg support not available")
+
+        filename = "Tests/images/pil168.tif"
+        im = Image.open(filename)
+
+        self.assertEqual(im.mode, "RGB")
+        self.assertEqual(im.size, (256, 256))
+        self.assertEqual(
+            im.tile, [('jpeg', (0, 0, 256, 256), 0, ('RGB', 'jpeg', False))]
+        )
+        im.load()
+
+        self.assert_image_equal_tofile(im, "Tests/images/pil168.png")
+
+    def test_sampleformat(self):
+        # https://github.com/python-pillow/Pillow/issues/1466
+        im = Image.open("Tests/images/copyleft.tiff")
+        self.assertEqual(im.mode, 'RGB')
+
+        self.assert_image_equal_tofile(im, "Tests/images/copyleft.png", mode='RGB')
+
+    def test_lzw(self):
+        im = Image.open("Tests/images/hopper_lzw.tif")
+
+        self.assertEqual(im.mode, 'RGB')
+        self.assertEqual(im.size, (128, 128))
+        self.assertEqual(im.format, "TIFF")
+        im2 = hopper()
+        self.assert_image_similar(im, im2, 5)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Tests/test_file_mic.py b/Tests/test_file_mic.py
--- a/Tests/test_file_mic.py
+++ b/Tests/test_file_mic.py
@@ -1,6 +1,6 @@
 from helper import unittest, PillowTestCase, hopper
 
-from PIL import Image, ImagePalette
+from PIL import Image, ImagePalette, features
 
 try:
     from PIL import MicImagePlugin
@@ -13,6 +13,7 @@
 
 
 @unittest.skipUnless(olefile_installed, "olefile package not installed")
+@unittest.skipUnless(features.check('libtiff'), "libtiff not installed")
 class TestFileMic(PillowTestCase):
 
     def test_sanity(self):
diff --git a/Tests/test_file_tiff.py b/Tests/test_file_tiff.py
--- a/Tests/test_file_tiff.py
+++ b/Tests/test_file_tiff.py
@@ -51,13 +51,7 @@ def test_mac_tiff(self):
         self.assertEqual(im.tile, [('raw', (0, 0, 55, 43), 8, ('RGBa', 0, 1))])
         im.load()
 
-    def test_16bit_RGBa_tiff(self):
-        im = Image.open("Tests/images/tiff_16bit_RGBa.tiff")
-
-        self.assertEqual(im.mode, "RGBA")
-        self.assertEqual(im.size, (100, 40))
-        self.assertEqual(im.tile, [('tiff_lzw', (0, 0, 100, 40), 50, 'RGBa;16B')])
-        im.load()
+        self.assert_image_similar_tofile(im, "Tests/images/pil136.png", 1)
 
     def test_wrong_bits_per_sample(self):
         im = Image.open("Tests/images/tiff_wrong_bits_per_sample.tiff")
@@ -67,32 +61,6 @@ def test_wrong_bits_per_sample(self):
         self.assertEqual(im.tile, [('raw', (0, 0, 52, 53), 160, ('RGBA', 0, 1))])
         im.load()
 
-    def test_gimp_tiff(self):
-        # Read TIFF JPEG images from GIMP [@PIL168]
-
-        codecs = dir(Image.core)
-        if "jpeg_decoder" not in codecs:
-            self.skipTest("jpeg support not available")
-
-        filename = "Tests/images/pil168.tif"
-        im = Image.open(filename)
-
-        self.assertEqual(im.mode, "RGB")
-        self.assertEqual(im.size, (256, 256))
-        self.assertEqual(
-            im.tile, [
-                ('jpeg', (0, 0, 256, 64), 8, ('RGB', '')),
-                ('jpeg', (0, 64, 256, 128), 1215, ('RGB', '')),
-                ('jpeg', (0, 128, 256, 192), 2550, ('RGB', '')),
-                ('jpeg', (0, 192, 256, 256), 3890, ('RGB', '')),
-                ])
-        im.load()
-
-    def test_sampleformat(self):
-        # https://github.com/python-pillow/Pillow/issues/1466
-        im = Image.open("Tests/images/copyleft.tiff")
-        self.assertEqual(im.mode, 'RGB')
-
     def test_set_legacy_api(self):
         with self.assertRaises(Exception):
             ImageFileDirectory_v2.legacy_api = None
@@ -225,12 +193,7 @@ def test_12bit_rawmode(self):
         # imagemagick will auto scale so that a 12bit FFF is 16bit FFF0,
         # so we need to unshift so that the integer values are the same.
 
-        im2 = Image.open('Tests/images/12in16bit.tif')
-
-        logger.debug("%s", [img.getpixel((0, idx))
-                            for img in [im, im2] for idx in range(3)])
-
-        self.assert_image_equal(im, im2)
+        self.assert_image_equal_tofile(im, 'Tests/images/12in16bit.tif')
 
     def test_32bit_float(self):
         # Issue 614, specific 32-bit float format
@@ -436,16 +399,6 @@ def test_with_underscores(self):
         self.assertEqual(im.tag_v2[X_RESOLUTION], 72)
         self.assertEqual(im.tag_v2[Y_RESOLUTION], 36)
 
-    def test_lzw(self):
-        # Act
-        im = Image.open("Tests/images/hopper_lzw.tif")
-
-        # Assert
-        self.assertEqual(im.mode, 'RGB')
-        self.assertEqual(im.size, (128, 128))
-        self.assertEqual(im.format, "TIFF")
-        im2 = hopper()
-        self.assert_image_similar(im, im2, 5)
 
     def test_roundtrip_tiff_uint16(self):
         # Test an image of all '0' values