Repository: python-pillow/Pillow

Problem statement:
PIL cannot read BigTIFF
<!--
Thank you for reporting an issue.

Follow these guidelines to ensure your issue is handled properly.

If you have a ...

1.  General question: consider asking the question on Stack Overflow
    with the python-imaging-library tag:

    * https://stackoverflow.com/questions/tagged/python-imaging-library

    Do not ask a question in both places.

    If you think you have found a bug or have an unexplained exception
    then file a bug report here.

2.  Bug report: include a self-contained, copy-pastable example that
    generates the issue if possible. Be concise with code posted.
    Guidelines on how to provide a good bug report:

    * https://stackoverflow.com/help/mcve

    Bug reports which follow these guidelines are easier to diagnose,
    and are often handled much more quickly.

3.  Feature request: do a quick search of existing issues
    to make sure this has not been asked before.

We know asking good questions takes effort, and we appreciate your time.
Thank you.
-->

### What did you do?
I'm trying to open an orthomosaic geotiff into Python to crop into 1000x1000 tiles. I am able to open some tif files in my notebook, but other files return an UnidentifiedImageError. I have set up my notebook using a Dockerfile that creates a conda environment.
### What did you expect to happen?
I expected that my code would work the same on all of my tif files - they are all 4-band 8-bit. This code works with a tif file that is 992 MB, but it doesn't work with a file that's 691 MB. 




### What actually happened?
```
---------------------------------------------------------------------------
UnidentifiedImageError                    Traceback (most recent call last)
<ipython-input-85-c7f17ab4563e> in <module>
      6 
      7 # break it up into crops
----> 8 for k, piece in enumerate(crop(infile, tile_height, tile_width, stride, img_dict, prj_name), start_num):
      9     img=Image.new('RGB', (tile_height, tile_width), (255, 255, 255))
     10     print(img.size)

<ipython-input-83-c9b468169840> in crop(infile, tile_height, tile_width, stride, img_dict, prj_name)
      5 
      6 def crop(infile, tile_height, tile_width, stride, img_dict, prj_name):
----> 7     im = Image.open(infile)
      8     img_width, img_height = im.size
      9     print(im.size)

/opt/conda/envs/geo_env/lib/python3.8/site-packages/PIL/Image.py in open(fp, mode)
   2859     for message in accept_warnings:
   2860         warnings.warn(message)
-> 2861     raise UnidentifiedImageError(
   2862         "cannot identify image file %r" % (filename if filename else fp)
   2863     )

UnidentifiedImageError: cannot identify image file '../data/mosaics/GrandJason_SWRightThird_Nov2019_transparent_mosaic_group1.tif'
```
### What are your OS, Python and Pillow versions?

* OS: Ubuntu 18.04
* Python: 3.8.2
* Pillow: 7.0.0

<!--
Please include **code** that reproduces the issue and whenever possible, an **image** that demonstrates the issue. Please upload images to GitHub, not to third-party file hosting sites. If necessary, add the image to a zip or tar archive.

The best reproductions are self-contained scripts with minimal dependencies. If you are using a framework such as Plone, Django, or Buildout, try to replicate the issue just using Pillow.
-->

```python
from PIL import Image
import os
import argparse
import numpy as np
import json
import csv
import rasterio
import matplotlib
import folium
from pyproj import Proj, transform


%matplotlib inline


Image.MAX_IMAGE_PIXELS = 100000000000
# ingest the image
infile = "../data/mosaics/GrandJason_SWRightThird_Nov2019_transparent_mosaic_group1.tif"

img_dir = '..' + infile.split(".")[2]
prj_name = img_dir.split("/")[-1]
dataset = rasterio.open(infile)

# what is the name of this image
img_name = dataset.name
print('Image filename: {n}\n'.format(n=img_name))

# How many bands does this image have?
num_bands = dataset.count
print('Number of bands in image: {n}\n'.format(n=num_bands))

# How many rows and columns?
rows, cols = dataset.shape
print('Image size is: {r} rows x {c} columns\n'.format(r=rows, c=cols))

# Does the raster have a description or metadata?
desc = dataset.descriptions
metadata = dataset.meta

print('Raster description: {desc}\n'.format(desc=desc))

# What driver was used to open the raster?
driver = dataset.driver
print('Raster driver: {d}\n'.format(d=driver))

# What is the raster's projection?
proj = dataset.crs
print('Image projection:')
print(proj, '\n')

# What is the raster's "geo-transform"
gt = dataset.transform

print('Image geo-transform:\n{gt}\n'.format(gt=gt))

print('All raster metadata:')
print(metadata)
print('\n')

tile_height = tile_width = 1000
overlap = 80
stride = tile_height - overlap
start_num=0

#crop image into tiles
def crop(infile, tile_height, tile_width, stride, img_dict, prj_name):
    im = Image.open(infile) 
    img_width, img_height = im.size
    print(im.size)
    print(img_width * img_height / (tile_height - stride) / (tile_width - stride))
    count = 0
    for r in range(0, img_height-tile_height+1, stride):
        for c in range(0, img_width-tile_width+1, stride):
            #tile = im[r:r+100, c:c+100]
            box = (c, r, c+tile_width, r+tile_height)
            top_pixel = [c,r]
            img_dict[prj_name + "---" + str(count) + ".png"] = top_pixel
            count += 1
            yield im.crop(box)

#split image into heightxwidth patches
img = Image

img_dict = {}

# create the dir if it doesn't already exist
if not os.path.exists(img_dir):
    os.makedirs(img_dir)

# break it up into crops
for k, piece in enumerate(crop(infile, tile_height, tile_width, stride, img_dict, prj_name), start_num):
    img=Image.new('RGB', (tile_height, tile_width), (255, 255, 255))
    print(img.size)
    print(piece.size)
    img.paste(piece)
    image_name = prj_name + "---%s.png" % k
    path=os.path.join(img_dir, image_name)
    img.save(path)
```



Hints:
Could be a [BigTIFF](https://www.awaresystems.be/imaging/tiff/bigtiff.html) file, which is not supported by Pillow. The first 4 bytes in BigTIFF files are `b'II\x2B\x00'` or `b'MM\x00\x2B'`
> Could be a [BigTIFF](https://www.awaresystems.be/imaging/tiff/bigtiff.html) file, which is not supported by Pillow. The first 4 bytes in BigTIFF files are `b'II\x2B\x00'` or `b'MM\x00\x2B'`

None of my tif files are BigTIFF, and they are all way under 4GB size
Here is the information of my raster: 
```Number of bands in image: 4

Image size is: 18679 rows x 31880 columns

Raster description: (None, None, None, None)

Raster driver: GTiff

Image projection:
EPSG:32720 

Image geo-transform:
| 0.02, 0.00, 632879.80|
| 0.00,-0.02, 4341041.62|
| 0.00, 0.00, 1.00|

All raster metadata:
{'driver': 'GTiff', 'dtype': 'uint8', 'nodata': None, 'width': 31880, 'height': 18679, 'count': 4, 'crs': CRS.from_epsg(32720), 'transform': Affine(0.01804, 0.0, 632879.79994,
       0.0, -0.01804, 4341041.621540001)}

bytearray(b'II+\x00\x08')
```
> `bytearray(b'II+\x00\x08')`

That's BigTIFF.
Are there any plans to support bigtiff?  I'd like to add my name to the list of people who would find it very useful.
I've just stumbled upon this issue too.  At a first glance, it seems to me that it should be pretty straightforward to modify src/libImaging/TiffDecode.c to handle the BigTIFF format, which is meant to be very closely compatible with the TIFF format.
Ah, after working on this for about a day, and getting most of the way to a working solution, I realised that it is probably futile, as BigTIFF files are to a great extent used for medical images, and they will include multiple images within the file (for example, thumbnails). But PIL can only handle one image in a file, so this won't work very well.

A better solution is probably to identify BigTIFF files (from the b"MM\x00\x2B" or b"II\x2B\x00" header bytes) and then recommend using something like tifffile to handle the image.

If it's of interest, I could upload what I've done so far to github.  (It's primarily work on TiffImagePlugin.py and TiffTags.py.)

Best wishes!
> But PIL can only handle one image in a file, so this won't work very well.

Pillow can load multiple images. It opens the file at one image, and then uses `seek()` to navigate to a different image.

If you're willing to share what you've done, and an example image, that would be interesting.
OK, shall do when I've had a chance to get at least some of it working!
I've created PR #6097 to resolve this.

Patch:
diff --git a/src/PIL/Image.py b/src/PIL/Image.py
--- a/src/PIL/Image.py
+++ b/src/PIL/Image.py
@@ -49,7 +49,7 @@
 # PILLOW_VERSION was removed in Pillow 9.0.0.
 # Use __version__ instead.
 from . import ImageMode, TiffTags, UnidentifiedImageError, __version__, _plugins
-from ._binary import i32le
+from ._binary import i32le, o32be, o32le
 from ._util import deferred_error, isPath
 
 
@@ -1416,6 +1416,7 @@ def getexif(self):
                     "".join(self.info["Raw profile type exif"].split("\n")[3:])
                 )
             elif hasattr(self, "tag_v2"):
+                self._exif.bigtiff = self.tag_v2._bigtiff
                 self._exif.endian = self.tag_v2._endian
                 self._exif.load_from_fp(self.fp, self.tag_v2._offset)
         if exif_info is not None:
@@ -3423,6 +3424,7 @@ def _apply_env_variables(env=None):
 
 class Exif(MutableMapping):
     endian = None
+    bigtiff = False
 
     def __init__(self):
         self._data = {}
@@ -3458,10 +3460,15 @@ def _get_ifd_dict(self, offset):
             return self._fixup_dict(info)
 
     def _get_head(self):
+        version = b"\x2B" if self.bigtiff else b"\x2A"
         if self.endian == "<":
-            return b"II\x2A\x00\x08\x00\x00\x00"
+            head = b"II" + version + b"\x00" + o32le(8)
         else:
-            return b"MM\x00\x2A\x00\x00\x00\x08"
+            head = b"MM\x00" + version + o32be(8)
+        if self.bigtiff:
+            head += o32le(8) if self.endian == "<" else o32be(8)
+            head += b"\x00\x00\x00\x00"
+        return head
 
     def load(self, data):
         # Extract EXIF information.  This is highly experimental,
diff --git a/src/PIL/TiffImagePlugin.py b/src/PIL/TiffImagePlugin.py
--- a/src/PIL/TiffImagePlugin.py
+++ b/src/PIL/TiffImagePlugin.py
@@ -260,6 +260,8 @@
     b"II\x2A\x00",  # Valid TIFF header with little-endian byte order
     b"MM\x2A\x00",  # Invalid TIFF header, assume big-endian
     b"II\x00\x2A",  # Invalid TIFF header, assume little-endian
+    b"MM\x00\x2B",  # BigTIFF with big-endian byte order
+    b"II\x2B\x00",  # BigTIFF with little-endian byte order
 ]
 
 
@@ -502,11 +504,14 @@ def __init__(self, ifh=b"II\052\0\0\0\0\0", prefix=None, group=None):
             self._endian = "<"
         else:
             raise SyntaxError("not a TIFF IFD")
+        self._bigtiff = ifh[2] == 43
         self.group = group
         self.tagtype = {}
         """ Dictionary of tag types """
         self.reset()
-        (self.next,) = self._unpack("L", ifh[4:])
+        (self.next,) = (
+            self._unpack("Q", ifh[8:]) if self._bigtiff else self._unpack("L", ifh[4:])
+        )
         self._legacy_api = False
 
     prefix = property(lambda self: self._prefix)
@@ -699,6 +704,7 @@ def _register_basic(idx_fmt_name):
                 (TiffTags.FLOAT, "f", "float"),
                 (TiffTags.DOUBLE, "d", "double"),
                 (TiffTags.IFD, "L", "long"),
+                (TiffTags.LONG8, "Q", "long8"),
             ],
         )
     )
@@ -776,8 +782,17 @@ def load(self, fp):
         self._offset = fp.tell()
 
         try:
-            for i in range(self._unpack("H", self._ensure_read(fp, 2))[0]):
-                tag, typ, count, data = self._unpack("HHL4s", self._ensure_read(fp, 12))
+            tag_count = (
+                self._unpack("Q", self._ensure_read(fp, 8))
+                if self._bigtiff
+                else self._unpack("H", self._ensure_read(fp, 2))
+            )[0]
+            for i in range(tag_count):
+                tag, typ, count, data = (
+                    self._unpack("HHQ8s", self._ensure_read(fp, 20))
+                    if self._bigtiff
+                    else self._unpack("HHL4s", self._ensure_read(fp, 12))
+                )
 
                 tagname = TiffTags.lookup(tag, self.group).name
                 typname = TYPES.get(typ, "unknown")
@@ -789,9 +804,9 @@ def load(self, fp):
                     logger.debug(msg + f" - unsupported type {typ}")
                     continue  # ignore unsupported type
                 size = count * unit_size
-                if size > 4:
+                if size > (8 if self._bigtiff else 4):
                     here = fp.tell()
-                    (offset,) = self._unpack("L", data)
+                    (offset,) = self._unpack("Q" if self._bigtiff else "L", data)
                     msg += f" Tag Location: {here} - Data Location: {offset}"
                     fp.seek(offset)
                     data = ImageFile._safe_read(fp, size)
@@ -820,7 +835,11 @@ def load(self, fp):
                 )
                 logger.debug(msg)
 
-            (self.next,) = self._unpack("L", self._ensure_read(fp, 4))
+            (self.next,) = (
+                self._unpack("Q", self._ensure_read(fp, 8))
+                if self._bigtiff
+                else self._unpack("L", self._ensure_read(fp, 4))
+            )
         except OSError as msg:
             warnings.warn(str(msg))
             return
@@ -1042,6 +1061,8 @@ def _open(self):
 
         # Header
         ifh = self.fp.read(8)
+        if ifh[2] == 43:
+            ifh += self.fp.read(8)
 
         self.tag_v2 = ImageFileDirectory_v2(ifh)
 
diff --git a/src/PIL/TiffTags.py b/src/PIL/TiffTags.py
--- a/src/PIL/TiffTags.py
+++ b/src/PIL/TiffTags.py
@@ -74,6 +74,7 @@ def lookup(tag, group=None):
 FLOAT = 11
 DOUBLE = 12
 IFD = 13
+LONG8 = 16
 
 TAGS_V2 = {
     254: ("NewSubfileType", LONG, 1),


Test patch:
diff --git a/Tests/images/hopper_bigtiff.tif b/Tests/images/hopper_bigtiff.tif
new file mode 100644
Binary files /dev/null and b/Tests/images/hopper_bigtiff.tif differ
diff --git a/Tests/test_file_tiff.py b/Tests/test_file_tiff.py
--- a/Tests/test_file_tiff.py
+++ b/Tests/test_file_tiff.py
@@ -87,6 +87,10 @@ def test_mac_tiff(self):
 
             assert_image_similar_tofile(im, "Tests/images/pil136.png", 1)
 
+    def test_bigtiff(self):
+        with Image.open("Tests/images/hopper_bigtiff.tif") as im:
+            assert_image_equal_tofile(im, "Tests/images/hopper.tif")
+
     @pytest.mark.parametrize(
         "file_name,mode,size,offset",
         [