Repository: pydicom/pydicom

Issue:
Add support for Extended Offset Table to encaps module
[CP1818](http://webcache.googleusercontent.com/search?q=cache:xeWXtrAs9G4J:ftp://medical.nema.org/medical/dicom/final/cp1818_ft_whenoffsettabletoosmall.pdf) added the use of an Extended Offset Table for encapsulated pixel data when the Basic Offset Table isn't suitable.


Discussion:
Some notes for myself:

* The EOT is optional and intended to support users where there are multiple compressed frames and the total length of all the frames exceeds the `2**32 - 1` limit available in the BOT item lengths. 
* Only 1 fragment per frame is allowed with an EOT
* The *Extended Table Offset Lengths* is the length (in bytes) of each compressed frame
* It's not usable where each compressed frame is larger than `2**32 - 1` due to the limit in the 4-byte (FFFE,E000) Item Tag length (who has 4 GB compressed images anyway?)

Relevant links: [google groups issue](https://groups.google.com/forum/?nomobile=true#!searchin/comp.protocols.dicom/extended$20offset|sort:date/comp.protocols.dicom/piMk2TmcyEg/daCk33zDBwAJ), [Image Pixel module](http://dicom.nema.org/medical/dicom/current/output/chtml/part03/sect_C.7.6.3.html), [BOT](http://dicom.nema.org/medical/dicom/current/output/chtml/part05/sect_A.4.html)

Implementation shouldn't be difficult, something along the lines of:
```python
from struct import pack
from pydicom.encaps import encapsulate

frame_data: List[bytes] = [...]
ds.PixelData = encapsulate(frame_data, has_bot=False, fragments_per_frame=1)

frame_lengths = [len(f) for f in frame_data]
frame_offsets = [0]
for ii, length in enumerate(frame_lengths[:-1]):
    frame_offsets .append(frame_offsets [ii] + length + 8)
ds.ExtendedOffsetTable =  pack("<{len(frame_offsets )}Q", *frame_offsets )
ds.ExtendedOffsetTableLengths = pack("<{len(frame_lengths)}Q", *frame_lengths)
```

Maybe just add a helper function to ensure conformance? And add a check to `encapsulate()` for data that's too long.
```python
from pydicom.encaps import encapsulate_extended

out: Tuple[bytes, bytes, bytes] = encapsulate_extended(frame_data)
ds.PixelData = out[0]
ds.ExtendedOffsetTable = out[1]
ds.ExtendedOffsetTableLengths = out[2]
```

Parsing should already be handled correctly by `generate_pixel_data()` since it's 1 frame per fragment.

Patch:
diff --git a/pydicom/encaps.py b/pydicom/encaps.py
--- a/pydicom/encaps.py
+++ b/pydicom/encaps.py
@@ -77,13 +77,15 @@ def get_frame_offsets(fp: DicomFileLike) -> Tuple[bool, List[int]]:
     tag = Tag(fp.read_tag())
 
     if tag != 0xfffee000:
-        raise ValueError("Unexpected tag '{}' when parsing the Basic Table "
-                         "Offset item.".format(tag))
+        raise ValueError(
+            f"Unexpected tag '{tag}' when parsing the Basic Table Offset item"
+        )
 
     length = fp.read_UL()
     if length % 4:
-        raise ValueError("The length of the Basic Offset Table item is not "
-                         "a multiple of 4.")
+        raise ValueError(
+            "The length of the Basic Offset Table item is not a multiple of 4"
+        )
 
     offsets = []
     # Always return at least a 0 offset
@@ -116,18 +118,20 @@ def get_nr_fragments(fp: DicomFileLike) -> int:
             # Item
             length = fp.read_UL()
             if length == 0xFFFFFFFF:
-                raise ValueError("Undefined item length at offset {} when "
-                                 "parsing the encapsulated pixel data "
-                                 "fragments.".format(fp.tell() - 4))
+                raise ValueError(
+                    f"Undefined item length at offset {fp.tell() - 4} when "
+                    "parsing the encapsulated pixel data fragments"
+                )
             fp.seek(length, 1)
             nr_fragments += 1
         elif tag == 0xFFFEE0DD:
             # Sequence Delimiter
             break
         else:
-            raise ValueError("Unexpected tag '{}' at offset {} when parsing "
-                             "the encapsulated pixel data fragment items."
-                             .format(tag, fp.tell() - 4))
+            raise ValueError(
+                f"Unexpected tag '{tag}' at offset {fp.tell() - 4} when "
+                "parsing the encapsulated pixel data fragment items"
+            )
 
     fp.seek(start)
     return nr_fragments
@@ -206,9 +210,10 @@ def generate_pixel_data_fragment(
             # Item
             length = fp.read_UL()
             if length == 0xFFFFFFFF:
-                raise ValueError("Undefined item length at offset {} when "
-                                 "parsing the encapsulated pixel data "
-                                 "fragments.".format(fp.tell() - 4))
+                raise ValueError(
+                    f"Undefined item length at offset {fp.tell() - 4} when "
+                    "parsing the encapsulated pixel data fragments"
+                )
             yield fp.read(length)
         elif tag == 0xFFFEE0DD:
             # Sequence Delimiter
@@ -216,9 +221,10 @@ def generate_pixel_data_fragment(
             fp.seek(-4, 1)
             break
         else:
-            raise ValueError("Unexpected tag '{0}' at offset {1} when parsing "
-                             "the encapsulated pixel data fragment items."
-                             .format(tag, fp.tell() - 4))
+            raise ValueError(
+                f"Unexpected tag '{tag}' at offset {fp.tell() - 4} when "
+                "parsing the encapsulated pixel data fragment items"
+            )
 
 
 def generate_pixel_data_frame(
@@ -549,8 +555,10 @@ def fragment_frame(
     frame_length = len(frame)
     # Add 1 to fix odd length frames not being caught
     if nr_fragments > (frame_length + 1) / 2.0:
-        raise ValueError('Too many fragments requested (the minimum fragment '
-                         'size is 2 bytes)')
+        raise ValueError(
+            "Too many fragments requested (the minimum fragment size is "
+            "2 bytes)"
+        )
 
     length = int(frame_length / nr_fragments)
 
@@ -596,7 +604,7 @@ def itemize_fragment(fragment: bytes) -> bytes:
       a 4 byte length.
     """
     # item tag (fffe,e000)
-    item = bytes(b'\xFE\xFF\x00\xE0')
+    item = b'\xFE\xFF\x00\xE0'
     # fragment length '<I' little endian, 4 byte unsigned int
     item += pack('<I', len(fragment))
     # fragment data
@@ -640,7 +648,7 @@ def itemize_frame(
     :dcm:`Annex A.4 <part05/sect_A.4.html>`
     """
     for fragment in fragment_frame(frame, nr_fragments):
-        yield itemise_fragment(fragment)
+        yield itemize_fragment(fragment)
 
 
 itemise_frame = itemize_frame
@@ -664,6 +672,16 @@ def encapsulate(
     For multi-frame data each frame must be encoded separately and then all
     encoded frames encapsulated together.
 
+    When many large frames are to be encapsulated, the total length of
+    encapsulated data may exceed the maximum length available with the
+    :dcm:`Basic Offset Table<part05/sect_A.4.html>` (2**31 - 1 bytes). Under
+    these circumstances you can:
+
+    * Pass ``has_bot=False`` to :func:`~pydicom.encaps.encapsulate`
+    * Use :func:`~pydicom.encaps.encapsulate_extended` and add the
+      :dcm:`Extended Offset Table<part03/sect_C.7.6.3.html>` elements to your
+      dataset (recommended)
+
     Data will be encapsulated with a Basic Offset Table Item at the beginning,
     then one or more fragment items. Each item will be of even length and the
     final fragment of each frame may be padded with ``0x00`` if required.
@@ -688,18 +706,33 @@ def encapsulate(
     ----------
     DICOM Standard, Part 5, :dcm:`Section 7.5 <part05/sect_7.5.html>` and
     :dcm:`Annex A.4 <part05/sect_A.4.html>`
+
+    See Also
+    --------
+    :func:`~pydicom.encaps.encapsulate_extended`
     """
-    no_frames = len(frames)
+    nr_frames = len(frames)
     output = bytearray()
 
     # Add the Basic Offset Table Item
     # Add the tag
     output.extend(b'\xFE\xFF\x00\xE0')
     if has_bot:
+        # Check that the 2**32 - 1 limit in BOT item lengths won't be exceeded
+        total = (nr_frames - 1) * 8 + sum([len(f) for f in frames[:-1]])
+        if total > 2**32 - 1:
+            raise ValueError(
+                f"The total length of the encapsulated frame data ({total} "
+                "bytes) will be greater than the maximum allowed by the Basic "
+                f"Offset Table ({2**32 - 1} bytes), it's recommended that you "
+                "use the Extended Offset Table instead (see the "
+                "'encapsulate_extended' function for more information)"
+            )
+
         # Add the length
-        output.extend(pack('<I', 4 * no_frames))
+        output.extend(pack('<I', 4 * nr_frames))
         # Reserve 4 x len(frames) bytes for the offsets
-        output.extend(b'\xFF\xFF\xFF\xFF' * no_frames)
+        output.extend(b'\xFF\xFF\xFF\xFF' * nr_frames)
     else:
         # Add the length
         output.extend(pack('<I', 0))
@@ -708,7 +741,7 @@ def encapsulate(
     for ii, frame in enumerate(frames):
         # `itemised_length` is the total length of each itemised frame
         itemised_length = 0
-        for item in itemise_frame(frame, fragments_per_frame):
+        for item in itemize_frame(frame, fragments_per_frame):
             itemised_length += len(item)
             output.extend(item)
 
@@ -717,7 +750,67 @@ def encapsulate(
 
     if has_bot:
         # Go back and write the frame offsets - don't need the last offset
-        output[8:8 + 4 * no_frames] = pack('<{}I'.format(no_frames),
-                                           *bot_offsets[:-1])
+        output[8:8 + 4 * nr_frames] = pack(f"<{nr_frames}I", *bot_offsets[:-1])
 
     return bytes(output)
+
+
+def encapsulate_extended(frames: List[bytes]) -> Tuple[bytes, bytes, bytes]:
+    """Return encapsulated image data and values for the Extended Offset Table
+    elements.
+
+    When using a compressed transfer syntax (such as RLE Lossless or one of
+    JPEG formats) then any *Pixel Data* must be :dcm:`encapsulated
+    <part05/sect_A.4.html>`. When many large frames are to be encapsulated, the
+    total length of encapsulated data may exceed the maximum length available
+    with the :dcm:`Basic Offset Table<part05/sect_A.4.html>` (2**32 - 1 bytes).
+    Under these circumstances you can:
+
+    * Pass ``has_bot=False`` to :func:`~pydicom.encaps.encapsulate`
+    * Use :func:`~pydicom.encaps.encapsulate_extended` and add the
+      :dcm:`Extended Offset Table<part03/sect_C.7.6.3.html>` elements to your
+      dataset (recommended)
+
+    Examples
+    --------
+
+    .. code-block:: python
+
+        from pydicom.encaps import encapsulate_extended
+
+        # 'frames' is a list of image frames that have been each been encoded
+        # separately using the compression method corresponding to the Transfer
+        # Syntax UID
+        frames: List[bytes] = [...]
+        out: Tuple[bytes, bytes, bytes] = encapsulate_extended(frames)
+
+        ds.PixelData = out[0]
+        ds.ExtendedOffsetTable = out[1]
+        ds.ExtendedOffsetTableLengths = out[2]
+
+    Parameters
+    ----------
+    frames : list of bytes
+        The compressed frame data to encapsulate, one frame per item.
+
+    Returns
+    -------
+    bytes, bytes, bytes
+        The (encapsulated frames, extended offset table, extended offset
+        table lengths).
+
+    See Also
+    --------
+    :func:`~pydicom.encaps.encapsulate`
+    """
+    nr_frames = len(frames)
+    frame_lengths = [len(frame) for frame in frames]
+    frame_offsets = [0]
+    for ii, length in enumerate(frame_lengths[:-1]):
+        # Extra 8 bytes for the Item tag and length
+        frame_offsets.append(frame_offsets[ii] + length + 8)
+
+    offsets = pack(f"<{nr_frames}Q", *frame_offsets)
+    lengths = pack(f"<{nr_frames}Q", *frame_lengths)
+
+    return encapsulate(frames, has_bot=False), offsets, lengths


Test patch:
diff --git a/pydicom/tests/test_encaps.py b/pydicom/tests/test_encaps.py
--- a/pydicom/tests/test_encaps.py
+++ b/pydicom/tests/test_encaps.py
@@ -1,6 +1,8 @@
-# Copyright 2008-2018 pydicom authors. See LICENSE file for details.
+# Copyright 2008-2020 pydicom authors. See LICENSE file for details.
 """Test for encaps.py"""
 
+from struct import unpack
+
 import pytest
 
 from pydicom import dcmread
@@ -16,7 +18,8 @@
     read_item,
     fragment_frame,
     itemise_frame,
-    encapsulate
+    encapsulate,
+    encapsulate_extended
 )
 from pydicom.filebase import DicomBytesIO
 
@@ -36,7 +39,7 @@ def test_bad_tag(self):
         fp.is_little_endian = True
         with pytest.raises(ValueError,
                            match=r"Unexpected tag '\(fffe, e100\)' when "
-                                 r"parsing the Basic Table Offset item."):
+                                 r"parsing the Basic Table Offset item"):
             get_frame_offsets(fp)
 
     def test_bad_length_multiple(self):
@@ -49,7 +52,7 @@ def test_bad_length_multiple(self):
         fp.is_little_endian = True
         with pytest.raises(ValueError,
                            match="The length of the Basic Offset Table item"
-                                 " is not a multiple of 4."):
+                                 " is not a multiple of 4"):
             get_frame_offsets(fp)
 
     def test_zero_length(self):
@@ -138,7 +141,7 @@ def test_item_bad_tag(self):
         fp.is_little_endian = True
         msg = (
             r"Unexpected tag '\(0010, 0010\)' at offset 12 when parsing the "
-            r"encapsulated pixel data fragment items."
+            r"encapsulated pixel data fragment items"
         )
         with pytest.raises(ValueError, match=msg):
             get_nr_fragments(fp)
@@ -212,7 +215,7 @@ def test_item_undefined_length(self):
         with pytest.raises(ValueError,
                            match="Undefined item length at offset 4 when "
                                  "parsing the encapsulated pixel data "
-                                 "fragments."):
+                                 "fragments"):
             next(fragments)
         pytest.raises(StopIteration, next, fragments)
 
@@ -249,8 +252,7 @@ def test_item_bad_tag(self):
         with pytest.raises(ValueError,
                            match=r"Unexpected tag '\(0010, 0010\)' at offset "
                                  r"12 when parsing the encapsulated pixel "
-                                 r"data "
-                                 r"fragment items."):
+                                 r"data fragment items"):
             next(fragments)
         pytest.raises(StopIteration, next, fragments)
 
@@ -1199,3 +1201,63 @@ def test_encapsulate_bot(self):
             b'\xfe\xff\x00\xe0'  # Next item tag
             b'\xe6\x0e\x00\x00'  # Next item length
         )
+
+    def test_encapsulate_bot_large_raises(self):
+        """Test exception raised if too much pixel data for BOT."""
+
+        class FakeBytes(bytes):
+            length = -1
+
+            def __len__(self):
+                return self.length
+
+            def __getitem__(self, s):
+                return b'\x00' * 5
+
+        frame_a = FakeBytes()
+        frame_a.length = 2**32 - 1 - 8  # 8 for first BOT item tag/length
+        frame_b = FakeBytes()
+        frame_b.length = 10
+        data = encapsulate([frame_a, frame_b], has_bot=True)
+
+        frame_a.length = 2**32 - 1 - 7
+        msg = (
+            r"The total length of the encapsulated frame data \(4294967296 "
+            r"bytes\) will be greater than the maximum allowed by the Basic "
+        )
+        with pytest.raises(ValueError, match=msg):
+            encapsulate([frame_a, frame_b], has_bot=True)
+
+
+class TestEncapsulateExtended:
+    """Tests for encaps.encapsulate_extended."""
+    def test_encapsulate(self):
+        ds = dcmread(JP2K_10FRAME_NOBOT)
+        frames = decode_data_sequence(ds.PixelData)
+        assert len(frames) == 10
+
+        out = encapsulate_extended(frames)
+        # Pixel Data encapsulated OK
+        assert isinstance(out[0], bytes)
+        test_frames = decode_data_sequence(out[0])
+        for a, b in zip(test_frames, frames):
+            assert a == b
+
+        # Extended Offset Table is OK
+        assert isinstance(out[1], bytes)
+        assert [
+            0x0000,  # 0
+            0x0eee,  # 3822
+            0x1df6,  # 7670
+            0x2cf8,  # 11512
+            0x3bfc,  # 15356
+            0x4ade,  # 19166
+            0x59a2,  # 22946
+            0x6834,  # 26676
+            0x76e2,  # 30434
+            0x8594  # 34196
+        ] == list(unpack('<10Q', out[1]))
+
+        # Extended Offset Table Lengths are OK
+        assert isinstance(out[2], bytes)
+        assert [len(f) for f in frames] == list(unpack('<10Q', out[2]))