Repository: pydicom/pydicom

Issue:
Multiple encodings handled incorrectly in text VRs
This came up in #703. While decoding values with text VRs in a dataset with multiple encodings, the first encoding is removed, so the second encoding is used instead of the first one if no escape code is present.

This is the related code in charset.py:
```python
    if data_element.VR in text_VRs:
        # Remove the first encoding if this is a multi-byte encoding
        if len(encodings) > 1:
            del encodings[0]
```

Removing these lines should fix the problem, but we need some tests that check this scenario.

As for why that got in, maybe the following statement in PS3.5, 6.1.2.4:
> If such Code Extension techniques are used, the related Specific Character Set or Sets shall be specified by value 2 to value n of the Attribute Specific Character Set (0008,0005)

has been misinterpreted to mean all used character sets, while this is about the extensions, as far as know.

<!-- Instructions For Filing a Bug: https://github.com/pydicom/pydicom/blob/master/CONTRIBUTING.md#filing-bugs -->

#### Description
<!-- Example: Attribute Error thrown when printing (0x0010, 0x0020) patient Id> 0-->

#### Steps/Code to Reproduce
<!--
Example:
```py
from io import BytesIO
from pydicom import dcmread

bytestream = b'\x02\x00\x02\x00\x55\x49\x16\x00\x31\x2e\x32\x2e\x38\x34\x30\x2e\x31' \
             b'\x30\x30\x30\x38\x2e\x35\x2e\x31\x2e\x31\x2e\x39\x00\x02\x00\x10\x00' \
             b'\x55\x49\x12\x00\x31\x2e\x32\x2e\x38\x34\x30\x2e\x31\x30\x30\x30\x38' \
             b'\x2e\x31\x2e\x32\x00\x20\x20\x10\x00\x02\x00\x00\x00\x01\x00\x20\x20' \
             b'\x20\x00\x06\x00\x00\x00\x4e\x4f\x52\x4d\x41\x4c'

fp = BytesIO(bytestream)
ds = dcmread(fp, force=True)

print(ds.PatientID)
```
If the code is too long, feel free to put it in a public gist and link
it in the issue: https://gist.github.com

When possible use pydicom testing examples to reproduce the errors. Otherwise, provide
an anonymous version of the data in order to replicate the errors.
-->

#### Expected Results
<!-- Please paste or describe the expected results.
Example: No error is thrown and the name of the patient is printed.-->

#### Actual Results
<!-- Please paste or specifically describe the actual output or traceback.
(Use %xmode to deactivate ipython's trace beautifier)
Example: ```AttributeError: 'FileDataset' object has no attribute 'PatientID'```
-->

#### Versions
<!--
Please run the following snippet and paste the output below.
import platform; print(platform.platform())
import sys; print("Python", sys.version)
import pydicom; print("pydicom", pydicom.__version__)
-->


<!-- Thanks for contributing! -->


Discussion:

Patch:
diff --git a/README.md b/README.md
index 801972576..17699a22a 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@ pydicom
 [![codecov](https://codecov.io/gh/pydicom/pydicom/branch/master/graph/badge.svg)](https://codecov.io/gh/pydicom/pydicom)
 [![Python version](https://img.shields.io/pypi/pyversions/pydicom.svg)](https://img.shields.io/pypi/pyversions/pydicom.svg)
 [![PyPI version](https://badge.fury.io/py/pydicom.svg)](https://badge.fury.io/py/pydicom)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1291986.svg)](https://doi.org/10.5281/zenodo.1291986)
 
 pydicom is a pure python package for working with [DICOM](http://medical.nema.org/) files.
 It was made for inspecting and modifying DICOM data in an easy "pythonic" way.
diff --git a/pydicom/charset.py b/pydicom/charset.py
index 8ea85d303..28b38a665 100644
--- a/pydicom/charset.py
+++ b/pydicom/charset.py
@@ -158,10 +158,6 @@ def decode(data_element, dicom_character_set):
                     for value in data_element.value
                 ]
     if data_element.VR in text_VRs:
-        # Remove the first encoding if this is a multi-byte encoding
-        if len(encodings) > 1:
-            del encodings[0]
-
         # You can't re-decode unicode (string literals in py3)
         if data_element.VM == 1:
             if isinstance(data_element.value, compat.text_type):
diff --git a/pydicom/dataset.py b/pydicom/dataset.py
index 24dca240c..af2ae6e75 100644
--- a/pydicom/dataset.py
+++ b/pydicom/dataset.py
@@ -44,8 +44,6 @@ try:
 except ImportError:
     have_numpy = False
 
-sys_is_little_endian = (sys.byteorder == 'little')
-
 
 class PropertyError(Exception):
     """For AttributeErrors caught in a property, so do not go to __getattr__"""
diff --git a/pydicom/filereader.py b/pydicom/filereader.py
index 172df90c8..77461c037 100644
--- a/pydicom/filereader.py
+++ b/pydicom/filereader.py
@@ -32,9 +32,6 @@ from pydicom.util.hexutil import bytes2hex
 from pydicom.valuerep import extra_length_VRs
 
 
-sys_is_little_endian = (byteorder == 'little')
-
-
 class DicomIter(object):
     """Iterator over DICOM data elements created from a file-like object"""
 
diff --git a/pydicom/pixel_data_handlers/jpeg_ls_handler.py b/pydicom/pixel_data_handlers/jpeg_ls_handler.py
index d0c90948e..7db126a6c 100644
--- a/pydicom/pixel_data_handlers/jpeg_ls_handler.py
+++ b/pydicom/pixel_data_handlers/jpeg_ls_handler.py
@@ -4,9 +4,9 @@ Use the jpeg_ls (CharPyLS) python package
 to decode pixel transfer syntaxes.
 """
 
-import sys
 import pydicom
 import pydicom.uid
+from pydicom.pixel_data_handlers.util import dtype_corrected_for_endianess
 
 have_numpy = True
 try:
@@ -21,7 +21,6 @@ try:
 except ImportError:
     have_jpeg_ls = False
     raise
-sys_is_little_endian = (sys.byteorder == 'little')
 
 JPEGLSSupportedTransferSyntaxes = [
     pydicom.uid.JPEGLSLossless,
@@ -108,9 +107,8 @@ def get_pixeldata(dicom_dataset):
                    dicom_dataset.BitsAllocated))
         raise TypeError(msg)
 
-    if (dicom_dataset.is_little_endian !=
-            sys_is_little_endian):
-        numpy_format = numpy_format.newbyteorder('S')
+    numpy_format = dtype_corrected_for_endianess(
+        dicom_dataset.is_little_endian, numpy_format)
 
     # decompress here
     UncompressedPixelData = bytearray()
diff --git a/pydicom/pixel_data_handlers/numpy_handler.py b/pydicom/pixel_data_handlers/numpy_handler.py
index d2d645873..0cc5a1dba 100644
--- a/pydicom/pixel_data_handlers/numpy_handler.py
+++ b/pydicom/pixel_data_handlers/numpy_handler.py
@@ -1,9 +1,10 @@
 # Copyright 2008-2018 pydicom authors. See LICENSE file for details.
 """Use the numpy package to decode pixel transfer syntaxes."""
 
-import sys
 import pydicom.uid
 from pydicom import compat
+from pydicom.pixel_data_handlers.util import dtype_corrected_for_endianess
+
 have_numpy = True
 try:
     import numpy
@@ -11,8 +12,6 @@ except ImportError:
     have_numpy = False
     raise
 
-sys_is_little_endian = (sys.byteorder == 'little')
-
 NumpySupportedTransferSyntaxes = [
     pydicom.uid.ExplicitVRLittleEndian,
     pydicom.uid.ImplicitVRLittleEndian,
@@ -102,8 +101,8 @@ def get_pixeldata(dicom_dataset):
                    dicom_dataset.BitsAllocated))
         raise TypeError(msg)
 
-    if dicom_dataset.is_little_endian != sys_is_little_endian:
-        numpy_dtype = numpy_dtype.newbyteorder('S')
+    numpy_dtype = dtype_corrected_for_endianess(
+        dicom_dataset.is_little_endian, numpy_dtype)
 
     pixel_bytearray = dicom_dataset.PixelData
 
diff --git a/pydicom/pixel_data_handlers/pillow_handler.py b/pydicom/pixel_data_handlers/pillow_handler.py
index d16481dc6..682edc2b0 100644
--- a/pydicom/pixel_data_handlers/pillow_handler.py
+++ b/pydicom/pixel_data_handlers/pillow_handler.py
@@ -1,11 +1,13 @@
 # Copyright 2008-2018 pydicom authors. See LICENSE file for details.
 """Use the pillow python package to decode pixel transfer syntaxes."""
 
-import sys
 import io
 import pydicom.encaps
 import pydicom.uid
 import logging
+
+from pydicom.pixel_data_handlers.util import dtype_corrected_for_endianess
+
 have_numpy = True
 logger = logging.getLogger('pydicom')
 try:
@@ -40,7 +42,6 @@ PillowJPEGTransferSyntaxes = [
     pydicom.uid.JPEGBaseLineLossy12bit,
 ]
 
-sys_is_little_endian = (sys.byteorder == 'little')
 have_pillow_jpeg_plugin = False
 have_pillow_jpeg2000_plugin = False
 try:
@@ -150,8 +151,8 @@ def get_pixeldata(dicom_dataset):
                    dicom_dataset.BitsAllocated))
         raise TypeError(msg)
 
-    if dicom_dataset.is_little_endian != sys_is_little_endian:
-        numpy_format = numpy_format.newbyteorder('S')
+    numpy_format = dtype_corrected_for_endianess(
+        dicom_dataset.is_little_endian, numpy_format)
 
     # decompress here
     if (dicom_dataset.file_meta.TransferSyntaxUID in
diff --git a/pydicom/pixel_data_handlers/rle_handler.py b/pydicom/pixel_data_handlers/rle_handler.py
index 066bb4ae3..20469c966 100644
--- a/pydicom/pixel_data_handlers/rle_handler.py
+++ b/pydicom/pixel_data_handlers/rle_handler.py
@@ -1,6 +1,5 @@
 # Copyright 2008-2018 pydicom authors. See LICENSE file for details.
 
-import sys
 import pydicom.uid
 import pydicom.encaps
 from struct import unpack
@@ -11,8 +10,6 @@ except ImportError:
     have_numpy = False
     raise
 
-sys_is_little_endian = (sys.byteorder == 'little')
-
 RLESupportedTransferSyntaxes = [
     pydicom.uid.RLELossless,
 ]
diff --git a/pydicom/pixel_data_handlers/util.py b/pydicom/pixel_data_handlers/util.py
new file mode 100644
index 000000000..5cc0e5dc2
--- /dev/null
+++ b/pydicom/pixel_data_handlers/util.py
@@ -0,0 +1,39 @@
+# Copyright 2008-2018 pydicom authors. See LICENSE file for details.
+"""Utility functions used in the pixel data handlers."""
+
+import sys
+
+sys_is_little_endian = (sys.byteorder == 'little')
+
+
+def dtype_corrected_for_endianess(is_little_endian, numpy_dtype):
+    """Adapts the given numpy data type for changing the endianess of the
+    dataset, if needed.
+
+        Parameters
+        ----------
+        is_little_endian : bool
+            The endianess of the affected dataset.
+        numpy_dtype : numpy.dtype
+            The numpy data type used for the pixel data without considering
+            endianess.
+
+        Raises
+        ------
+        ValueError
+            If `is_little_endian` id None, e.g. not initialized.
+
+        Returns
+        -------
+        numpy.dtype
+            The numpy data type to be used for the pixel data, considering
+            the endianess.
+    """
+    if is_little_endian is None:
+        raise ValueError("Dataset attribute 'is_little_endian' "
+                         "has to be set before writing the dataset")
+
+    if is_little_endian != sys_is_little_endian:
+        return numpy_dtype.newbyteorder('S')
+
+    return numpy_dtype
diff --git a/pydicom/valuerep.py b/pydicom/valuerep.py
index fde29912d..f0337ff0f 100644
--- a/pydicom/valuerep.py
+++ b/pydicom/valuerep.py
@@ -732,9 +732,6 @@ class PersonNameUnicode(PersonNameBase, compat.text_type):
         if len(encodings) == 2:
             encodings.append(encodings[1])
         components = val.split(b"=")
-        # Remove the first encoding if only one component is present
-        if (len(components) == 1):
-            del encodings[0]
 
         comps = [
             clean_escseq(C.decode(enc), encodings)


Test patch:
diff --git a/pydicom/tests/test_charset.py b/pydicom/tests/test_charset.py
index 6237664de..e1ebfc648 100644
--- a/pydicom/tests/test_charset.py
+++ b/pydicom/tests/test_charset.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2008-2018 pydicom authors. See LICENSE file for details.
 """unittest cases for pydicom.charset module"""
 
@@ -26,7 +27,7 @@ class CharsetTests(unittest.TestCase):
         ds = dcmread(latin1_file)
         ds.decode()
         # Make sure don't get unicode encode error on converting to string
-        expected = u'Buc^J\xe9r\xf4me'
+        expected = u'Buc^Jérôme'
         got = ds.PatientName
         self.assertEqual(expected, got,
                          "Expected %r, got %r" % (expected, got))
@@ -120,7 +121,7 @@ class CharsetTests(unittest.TestCase):
         elem = DataElement(0x00100010, 'PN', b'Buc^J\xc3\xa9r\xc3\xb4me')
         pydicom.charset.decode(elem, ['ISO_IR 192'])
         # correct encoding
-        assert u'Buc^J\xe9r\xf4me' == elem.value
+        assert u'Buc^Jérôme' == elem.value
 
         # patched encoding shall behave correctly, but a warning is issued
         elem = DataElement(0x00100010, 'PN', b'Buc^J\xc3\xa9r\xc3\xb4me')
@@ -128,7 +129,7 @@ class CharsetTests(unittest.TestCase):
                           match='Incorrect value for Specific Character Set '
                                 "'ISO IR 192' - assuming 'ISO_IR 192'"):
             pydicom.charset.decode(elem, ['ISO IR 192'])
-            assert u'Buc^J\xe9r\xf4me' == elem.value
+            assert u'Buc^Jérôme' == elem.value
 
         elem = DataElement(0x00100010, 'PN', b'Buc^J\xe9r\xf4me')
         with pytest.warns(UserWarning,
@@ -146,7 +147,18 @@ class CharsetTests(unittest.TestCase):
         # Python encoding also can be used directly
         elem = DataElement(0x00100010, 'PN', b'Buc^J\xc3\xa9r\xc3\xb4me')
         pydicom.charset.decode(elem, ['utf8'])
-        assert u'Buc^J\xe9r\xf4me' == elem.value
+        assert u'Buc^Jérôme' == elem.value
+
+    def test_multi_charset_default_value(self):
+        """Test that the first value is used if no escape code is given"""
+        # regression test for #707
+        elem = DataElement(0x00100010, 'PN', b'Buc^J\xe9r\xf4me')
+        pydicom.charset.decode(elem, ['ISO 2022 IR 100', 'ISO 2022 IR 144'])
+        assert u'Buc^Jérôme' == elem.value
+
+        elem = DataElement(0x00081039, 'LO', b'R\xf6ntgenaufnahme')
+        pydicom.charset.decode(elem, ['ISO 2022 IR 100', 'ISO 2022 IR 144'])
+        assert u'Röntgenaufnahme' == elem.value
 
 
 if __name__ == "__main__":
diff --git a/pydicom/tests/test_jpeg_ls_pixel_data.py b/pydicom/tests/test_jpeg_ls_pixel_data.py
index 5f0f28a9f..f23d1eae9 100644
--- a/pydicom/tests/test_jpeg_ls_pixel_data.py
+++ b/pydicom/tests/test_jpeg_ls_pixel_data.py
@@ -182,6 +182,11 @@ class jpeg_ls_JPEG_LS_Tests_with_jpeg_ls(unittest.TestCase):
     def tearDown(self):
         pydicom.config.image_handlers = self.original_handlers
 
+    def test_raises_if_endianess_not_set(self):
+        self.jpeg_ls_lossless.is_little_endian = None
+        with pytest.raises(ValueError):
+            _ = self.jpeg_ls_lossless.pixel_array
+
     def test_JPEG_LS_PixelArray(self):
         a = self.jpeg_ls_lossless.pixel_array
         b = self.mr_small.pixel_array
diff --git a/pydicom/tests/test_numpy_pixel_data.py b/pydicom/tests/test_numpy_pixel_data.py
index d4725b0bb..7ae52f369 100644
--- a/pydicom/tests/test_numpy_pixel_data.py
+++ b/pydicom/tests/test_numpy_pixel_data.py
@@ -301,6 +301,11 @@ class numpy_LittleEndian_Tests(unittest.TestCase):
     def tearDown(self):
         pydicom.config.image_handlers = self.original_handlers
 
+    def test_raises_if_endianess_not_set(self):
+        self.odd_size_image.is_little_endian = None
+        with pytest.raises(ValueError):
+            _ = self.odd_size_image.pixel_array
+
     def test_little_endian_PixelArray_odd_data_size(self):
         pixel_data = self.odd_size_image.pixel_array
         assert pixel_data.nbytes == 27
diff --git a/pydicom/tests/test_pillow_pixel_data.py b/pydicom/tests/test_pillow_pixel_data.py
index 2cf185fd3..8a85e6341 100644
--- a/pydicom/tests/test_pillow_pixel_data.py
+++ b/pydicom/tests/test_pillow_pixel_data.py
@@ -289,6 +289,11 @@ class Test_JPEG2000Tests_with_pillow(object):
         """Reset the pixel data handlers."""
         pydicom.config.image_handlers = self.original_handlers
 
+    def test_raises_if_endianess_not_set(self):
+        self.jpeg_2k_lossless.is_little_endian = None
+        with pytest.raises(ValueError):
+            _ = self.jpeg_2k_lossless.pixel_array
+
     def testJPEG2000(self):
         """Test reading element values works OK with pillow pixel handler."""
         # XX also tests multiple-valued AT data element