Repository: pydicom/pydicom

Issue:
Deferred Read Fails For File-Like Objects
#### Description
Deferred reads are failing when dcmread is passed a file-like object (instead of a filepath).  There are two old issues from 2014 which describe the same issue which were apparently fixed, but I'm still seeing it on v1.3:
https://github.com/pydicom/pydicom/issues/104
https://github.com/pydicom/pydicom/issues/74

#### Steps/Code to Reproduce
```
import io
import pydicom

with open("./0.dcm", "rb") as fp:
   data = fp.read()
filelike = io.BytesIO(data)

dataset = pydicom.dcmread(filelike, defer_size=1024)
print(len(dataset.PixelData))
```

#### Expected Results
Pydicom should hold onto the supplied file-like and use that for the deferred read, rather than trying to grab the file-like's .name/.filename attr and use that to re-open.  It could also hold onto it's own open'd file-like (if supplied a file_path) and use that for deferred reads to simplify things.

#### Actual Results
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.6/dist-packages/pydicom/dataset.py", line 747, in __getattr__
    data_elem = self[tag]
  File "/usr/local/lib/python3.6/dist-packages/pydicom/dataset.py", line 826, in __getitem__
    data_elem)
  File "/usr/local/lib/python3.6/dist-packages/pydicom/filereader.py", line 911, in read_deferred_data_element
    raise IOError("Deferred read -- original filename not stored. "
OSError: Deferred read -- original filename not stored. Cannot re-open

#### Versions
Linux-4.18.0-25-generic-x86_64-with-Ubuntu-18.10-cosmic
Python 3.6.7 (default, Oct 22 2018, 11:32:17) 
pydicom 1.3.0

Discussion:

Patch:
diff --git a/doc/release_notes/v1.4.0.rst b/doc/release_notes/v1.4.0.rst
index 44fd87977..cac106f84 100644
--- a/doc/release_notes/v1.4.0.rst
+++ b/doc/release_notes/v1.4.0.rst
@@ -19,4 +19,5 @@ Enhancements
 ............
 
 * Added support for converting (60xx,3000) *Overlay Data* to a numpy ndarray
-  using `Dataset.overlay_array()` (issue:`912`).
+  using `Dataset.overlay_array()` (issue:`912`)
+* Added support for deferred reading in file-like objects (:issue:`932`)
diff --git a/pydicom/dataelem.py b/pydicom/dataelem.py
index 7f23c35de..e99601f2a 100644
--- a/pydicom/dataelem.py
+++ b/pydicom/dataelem.py
@@ -22,7 +22,8 @@ from pydicom.config import logger
 from pydicom.datadict import (dictionary_has_tag, dictionary_description,
                               dictionary_keyword, dictionary_is_retired,
                               private_dictionary_description, dictionary_VR,
-                              dictionary_VM, repeater_has_tag)
+                              repeater_has_tag)
+from pydicom.jsonrep import JsonDataElementConverter
 from pydicom.multival import MultiValue
 from pydicom.tag import Tag, BaseTag
 from pydicom.uid import UID
@@ -48,9 +49,9 @@ def empty_value_for_VR(VR, raw=False):
     The behavior of this property depends on the setting of
     :attr:`config.use_none_as_empty_value`. If that is set to ``True``,
     an empty value is always represented by ``None``, otherwise it depends
-    on `VR`. For text VRs (not including VRs that represent a number in
-    textual representation) an empty string is used as empty value
-    representation, for all other VRs, ``None``.
+    on `VR`. For text VRs (this includes 'AE', 'AS', 'CS', 'DA', 'DT', 'LO',
+    'LT', 'PN', 'SH', 'ST', 'TM', 'UC', 'UI', 'UR' and 'UT') an empty string
+    is used as empty value representation, for all other VRs, ``None``.
     Note that this is used only if decoding the element - it is always
     possible to set the value to another empty value representation,
     which will be preserved during the element object lifetime.
@@ -253,14 +254,14 @@ class DataElement(object):
 
     @classmethod
     def from_json(cls, dataset_class, tag, vr, value, value_key,
-                  bulk_data_uri_handler=None, encodings=None):
+                  bulk_data_uri_handler=None):
         """Return a :class:`DataElement` from JSON.
 
         Parameters
         ----------
-        dataset_class : Dataset derived class
+        dataset_class : dataset.Dataset derived class
             Class used to create sequence items.
-        tag : BaseTag
+        tag : BaseTag or int
             The data element tag.
         vr : str
             The data element value representation.
@@ -279,127 +280,22 @@ class DataElement(object):
         DataElement
         """
         # TODO: test wado-rs retrieve wrapper
+        converter = JsonDataElementConverter(dataset_class, tag, vr, value,
+                                             value_key, bulk_data_uri_handler)
+        elem_value = converter.get_element_values()
         try:
-            vm = dictionary_VM(tag)
-        except KeyError:
-            # Private tag
-            vm = str(len(value))
-        if value_key == 'Value':
-            if not(isinstance(value, list)):
-                fmt = '"{}" of data element "{}" must be a list.'
-                raise TypeError(fmt.format(value_key, tag))
-        elif value_key in {'InlineBinary', 'BulkDataURI'}:
-            if isinstance(value, list):
-                fmt = '"{}" of data element "{}" must be a {}.'
-                expected_type = ('string' if value_key == 'BulkDataURI'
-                                 else 'bytes-like object')
-                raise TypeError(fmt.format(value_key, tag, expected_type))
-        if vr == 'SQ':
-            elem_value = []
-            for value_item in value:
-                ds = dataset_class()
-                if value_item:
-                    for key, val in value_item.items():
-                        if 'vr' not in val:
-                            fmt = 'Data element "{}" must have key "vr".'
-                            raise KeyError(fmt.format(tag))
-                        unique_value_keys = tuple(
-                            set(val.keys()) & set(jsonrep.JSON_VALUE_KEYS)
-                        )
-                        if len(unique_value_keys) == 0:
-                            logger.debug(
-                                'data element has neither key "{}".'.format(
-                                    '" nor "'.join(jsonrep.JSON_VALUE_KEYS)
-                                )
-                            )
-                            elem = DataElement(tag=tag, value='', VR=vr)
-                        else:
-                            value_key = unique_value_keys[0]
-                            elem = cls.from_json(
-                                dataset_class, key, val['vr'],
-                                val[value_key], value_key
-                            )
-                        ds.add(elem)
-                elem_value.append(ds)
-        elif vr == 'PN':
-            # Special case, see DICOM Part 18 Annex F2.2
-            elem_value = []
-            for v in value:
-                if not isinstance(v, dict):
-                    # Some DICOMweb services get this wrong, so we
-                    # workaround the issue and warn the user
-                    # rather than raising an error.
-                    logger.warning(
-                        'value of data element "{}" with VR Person Name (PN) '
-                        'is not formatted correctly'.format(tag)
-                    )
-                    elem_value.append(v)
-                else:
-                    if 'Phonetic' in v:
-                        comps = ['', '', '']
-                    elif 'Ideographic' in v:
-                        comps = ['', '']
-                    else:
-                        comps = ['']
-                    if 'Alphabetic' in v:
-                        comps[0] = v['Alphabetic']
-                    if 'Ideographic' in v:
-                        comps[1] = v['Ideographic']
-                    if 'Phonetic' in v:
-                        comps[2] = v['Phonetic']
-                    elem_value.append('='.join(comps))
-            if len(elem_value) == 1:
-                elem_value = elem_value[0]
-            elif not elem_value:
-                elem_value = empty_value_for_VR(vr)
-        elif vr == 'AT':
-            elem_value = []
-            for v in value:
-                try:
-                    elem_value.append(int(v, 16))
-                except ValueError:
-                    warnings.warn('Invalid value "{}" for AT element - '
-                                  'ignoring it'.format(v))
-                value = value[0]
-            if not elem_value:
-                elem_value = empty_value_for_VR(vr)
-            elif len(elem_value) == 1:
-                elem_value = elem_value[0]
-        else:
-            if isinstance(value, list) and len(value) == 1:
-                value = value[0]
-            if value_key == 'InlineBinary':
-                elem_value = base64.b64decode(value)
-            elif value_key == 'BulkDataURI':
-                if bulk_data_uri_handler is None:
-                    logger.warning(
-                        'no bulk data URI handler provided for retrieval '
-                        'of value of data element "{}"'.format(tag)
-                    )
-                    elem_value = empty_value_for_VR(vr, raw=True)
-                else:
-                    elem_value = bulk_data_uri_handler(value)
-            else:
-                elem_value = value
-        if elem_value is None:
-            elem_value = empty_value_for_VR(vr)
-
-        elem_value = jsonrep.convert_to_python_number(elem_value, vr)
-
-        try:
-            if compat.in_py2 and vr == "PN":
-                elem_value = PersonNameUnicode(elem_value, 'UTF8')
             return DataElement(tag=tag, value=elem_value, VR=vr)
         except Exception:
             raise ValueError(
                 'Data element "{}" could not be loaded from JSON: {}'.format(
                     tag, elem_value
-                    )
+                )
             )
 
-    def to_json(self, bulk_data_element_handler,
-                bulk_data_threshold, dump_handler):
-        """Converts a :class:`DataElement` to JSON representation.
+    def to_json_dict(self, bulk_data_element_handler, bulk_data_threshold):
+        """Return a dictionary representation of the :class:`DataElement`
+        conforming to the DICOM JSON Model as described in the DICOM
+        Standard, Part 18, :dcm:`Annex F<part18/chaptr_F.html>`.
 
         Parameters
         ----------
@@ -446,14 +342,14 @@ class DataElement(object):
                     )
                     json_element['InlineBinary'] = encoded_value
         elif self.VR == 'SQ':
-            # recursive call to co-routine to format sequence contents
+            # recursive call to get sequence item JSON dicts
             value = [
-                json.loads(e.to_json(
+                ds.to_json(
                     bulk_data_element_handler=bulk_data_element_handler,
                     bulk_data_threshold=bulk_data_threshold,
-                    dump_handler=dump_handler
-                ))
-                for e in self
+                    dump_handler=lambda d: d
+                )
+                for ds in self
             ]
             json_element['Value'] = value
         elif self.VR == 'PN':
@@ -492,6 +388,49 @@ class DataElement(object):
             )
         return json_element
 
+    def to_json(self, bulk_data_threshold=1, bulk_data_element_handler=None,
+                dump_handler=None):
+        """Return a JSON representation of the :class:`DataElement`.
+
+        Parameters
+        ----------
+        bulk_data_element_handler: callable or None
+            Callable that accepts a bulk data element and returns the
+            "BulkDataURI" for retrieving the value of the data element
+            via DICOMweb WADO-RS
+        bulk_data_threshold: int
+            Size of base64 encoded data element above which a value will be
+            provided in form of a "BulkDataURI" rather than "InlineBinary"
+        dump_handler : callable, optional
+            Callable function that accepts a :class:`dict` and returns the
+            serialized (dumped) JSON string (by default uses
+            :func:`json.dumps`).
+
+        Returns
+        -------
+        dict
+            Mapping representing a JSON encoded data element
+
+        Raises
+        ------
+        TypeError
+            When size of encoded data element exceeds `bulk_data_threshold`
+            but `bulk_data_element_handler` is ``None`` and hence not callable
+
+        See also
+        --------
+        Dataset.to_json
+        """
+        if dump_handler is None:
+            def json_dump(d):
+                return json.dumps(d, sort_keys=True)
+
+            dump_handler = json_dump
+
+        return dump_handler(
+            self.to_json_dict(bulk_data_threshold, bulk_data_element_handler))
+
+
     @property
     def value(self):
         """Return the element's value."""
@@ -628,9 +567,8 @@ class DataElement(object):
             return True
 
         if isinstance(other, self.__class__):
-            if self.tag == other.tag and self.VR == other.VR \
-                    and self.value == other.value:
-                return True
+            return (self.tag == other.tag and self.VR == other.VR
+                    and self.value == other.value)
 
         return NotImplemented
 
diff --git a/pydicom/dataset.py b/pydicom/dataset.py
index 33af68926..3c5c6d2f2 100644
--- a/pydicom/dataset.py
+++ b/pydicom/dataset.py
@@ -1556,7 +1556,8 @@ class Dataset(dict):
                                getattr(data_element, x))
                               for x in dir(data_element)
                               if not x.startswith("_")
-                              and x not in ('from_json', 'to_json', 'clear')])
+                              and x not in ('from_json', 'to_json',
+                                            'to_json_dict', 'clear')])
             if data_element.VR == "SQ":
                 yield sequence_element_format % elem_dict
             else:
@@ -1949,8 +1950,7 @@ class Dataset(dict):
                         dataset.walk(callback)
 
     @classmethod
-    def from_json(cls, json_dataset, bulk_data_uri_handler=None,
-                  encodings=None):
+    def from_json(cls, json_dataset, bulk_data_uri_handler=None):
         """Add elements to the :class:`Dataset` from DICOM JSON format.
 
         See the DICOM Standard, Part 18, :dcm:`Annex F<part18/chapter_F.html>`.
@@ -1964,9 +1964,6 @@ class Dataset(dict):
             Callable function that accepts the "BulkDataURI" of the JSON
             representation of a data element and returns the actual value of
             data element (retrieved via DICOMweb WADO-RS).
-        encodings : list of str, optional
-            Encodings from (0008,0005) *Specific Character Set*, or ``None``
-            (default).
 
         Returns
         -------
@@ -1992,6 +1989,39 @@ class Dataset(dict):
             dataset.add(data_element)
         return dataset
 
+    def to_json_dict(self, bulk_data_threshold=1,
+                     bulk_data_element_handler=None):
+        """Return a dictionary representation of the :class:`Dataset`
+        conforming to the DICOM JSON Model as described in the DICOM
+        Standard, Part 18, :dcm:`Annex F<part18/chaptr_F.html>`.
+
+        Parameters
+        ----------
+        bulk_data_threshold : int, optional
+            Threshold for the length of a base64-encoded binary data element
+            above which the element should be considered bulk data and the
+            value provided as a URI rather than included inline (default:
+            ``1``).
+        bulk_data_element_handler : callable, optional
+            Callable function that accepts a bulk data element and returns a
+            JSON representation of the data element (dictionary including the
+            "vr" key and either the "InlineBinary" or the "BulkDataURI" key).
+
+        Returns
+        -------
+        dict
+            :class:`Dataset` representation based on the DICOM JSON Model.
+        """
+        json_dataset = {}
+        for key in self.keys():
+            json_key = '{:08X}'.format(key)
+            data_element = self[key]
+            json_dataset[json_key] = data_element.to_json_dict(
+                bulk_data_element_handler=bulk_data_element_handler,
+                bulk_data_threshold=bulk_data_threshold
+            )
+        return json_dataset
+
     def to_json(self, bulk_data_threshold=1, bulk_data_element_handler=None,
                 dump_handler=None):
         """Return a JSON representation of the :class:`Dataset`.
@@ -2014,6 +2044,11 @@ class Dataset(dict):
             serialized (dumped) JSON string (by default uses
             :func:`json.dumps`).
 
+            .. note:
+
+                Make sure to use a dump handler that sorts the keys (see
+                example below) to create DICOM-conformant JSON.
+
         Returns
         -------
         str
@@ -2023,29 +2058,17 @@ class Dataset(dict):
         Examples
         --------
         >>> def my_json_dumps(data):
-        ...     return json.dumps(data, indent=4)
+        ...     return json.dumps(data, indent=4, sort_keys=True)
         >>> ds.to_json(dump_handler=my_json_dumps)
         """
         if dump_handler is None:
-            logger.debug('using default json.dumps function')
-            dump_handler = json.dumps
-        json_dataset = {}
-        for key in self.keys():
-            json_key = '{0:04x}{1:04x}'.format(key.group, key.element).upper()
-            # FIXME: with pydicom 1.x the referenced image sequence
-            # causes a recursion error
-            if json_key == '00081140':
-                logger.warning(
-                    'currently can\'t serialize data element "{}"'.format(key)
-                )
-                continue
-            data_element = self[key]
-            json_dataset[json_key] = data_element.to_json(
-                bulk_data_element_handler=bulk_data_element_handler,
-                bulk_data_threshold=bulk_data_threshold,
-                dump_handler=dump_handler
-            )
-        return dump_handler(json_dataset)
+            def json_dump(d):
+                return json.dumps(d, sort_keys=True)
+
+            dump_handler = json_dump
+
+        return dump_handler(
+            self.to_json_dict(bulk_data_threshold, bulk_data_element_handler))
 
     __repr__ = __str__
 
@@ -2114,11 +2137,12 @@ class FileDataset(Dataset):
         self.file_meta = file_meta
         self.is_implicit_VR = is_implicit_VR
         self.is_little_endian = is_little_endian
+        filename = None
         if isinstance(filename_or_obj, compat.string_types):
-            self.filename = filename_or_obj
+            filename = filename_or_obj
             self.fileobj_type = open
         elif isinstance(filename_or_obj, io.BufferedReader):
-            self.filename = filename_or_obj.name
+            filename = filename_or_obj.name
             # This is the appropriate constructor for io.BufferedReader
             self.fileobj_type = open
         else:
@@ -2126,17 +2150,20 @@ class FileDataset(Dataset):
             # http://docs.python.org/reference/datamodel.html
             self.fileobj_type = filename_or_obj.__class__
             if getattr(filename_or_obj, "name", False):
-                self.filename = filename_or_obj.name
+                filename = filename_or_obj.name
             elif getattr(filename_or_obj, "filename",
                          False):  # gzip python <2.7?
-                self.filename = filename_or_obj.filename
+                filename = filename_or_obj.filename
             else:
                 # e.g. came from BytesIO or something file-like
-                self.filename = None
+                self.filename = filename_or_obj
+
         self.timestamp = None
-        if self.filename and os.path.exists(self.filename):
-            statinfo = os.stat(self.filename)
-            self.timestamp = statinfo.st_mtime
+        if filename:
+            self.filename = filename
+            if os.path.exists(filename):
+                statinfo = os.stat(filename)
+                self.timestamp = statinfo.st_mtime
 
     def __eq__(self, other):
         """Compare `self` and `other` for equality.
diff --git a/pydicom/filereader.py b/pydicom/filereader.py
index 06bc2bee9..6f42575a4 100644
--- a/pydicom/filereader.py
+++ b/pydicom/filereader.py
@@ -44,7 +44,7 @@ def data_element_generator(fp,
     Parameters
     ----------
     fp : file-like
-        The file like to read from.
+        The file-like to read from.
     is_implicit_VR : bool
         ``True`` if the data is encoded as implicit VR, ``False`` otherwise.
     is_little_endian : bool
@@ -912,28 +912,63 @@ def data_element_offset_to_value(is_implicit_VR, VR):
     return offset
 
 
-def read_deferred_data_element(fileobj_type, filename, timestamp,
+def read_deferred_data_element(fileobj_type, filename_or_obj, timestamp,
                                raw_data_elem):
     """Read the previously deferred value from the file into memory
-    and return a raw data element"""
+    and return a raw data element.
+
+    .. note:
+
+        This is called internally by pydicom and will normally not be
+        needed in user code.
+
+    Parameters
+    ----------
+    fileobj_type : type
+        The type of the original file object.
+    filename_or_obj : str or file-like
+        The filename of the original file if one exists, or the file-like
+        object where the data element persists.
+    timestamp : time or None
+        The time the original file has been read, if not a file-like.
+    raw_data_elem : dataelem.RawDataElement
+        The raw data element with no value set.
+
+    Returns
+    -------
+    dataelem.RawDataElement
+        The data element with the value set.
+
+    Raises
+    ------
+    IOError
+        If `filename_or_obj` is ``None``.
+    IOError
+        If `filename_or_obj` is a filename and the corresponding file does
+        not exist.
+    ValueError
+        If the VR or tag of `raw_data_elem` does not match the read value.
+    """
     logger.debug("Reading deferred element %r" % str(raw_data_elem.tag))
     # If it wasn't read from a file, then return an error
-    if filename is None:
+    if filename_or_obj is None:
         raise IOError("Deferred read -- original filename not stored. "
                       "Cannot re-open")
+    is_filename = isinstance(filename_or_obj, compat.string_types)
+
     # Check that the file is the same as when originally read
-    if not os.path.exists(filename):
+    if is_filename and not os.path.exists(filename_or_obj):
         raise IOError(u"Deferred read -- original file "
-                      "{0:s} is missing".format(filename))
+                      "{0:s} is missing".format(filename_or_obj))
     if timestamp is not None:
-        statinfo = os.stat(filename)
+        statinfo = os.stat(filename_or_obj)
         if statinfo.st_mtime != timestamp:
             warnings.warn("Deferred read warning -- file modification time "
                           "has changed.")
 
     # Open the file, position to the right place
-    # fp = self.typefileobj(self.filename, "rb")
-    fp = fileobj_type(filename, 'rb')
+    fp = (fileobj_type(filename_or_obj, 'rb')
+          if is_filename else filename_or_obj)
     is_implicit_VR = raw_data_elem.is_implicit_VR
     is_little_endian = raw_data_elem.is_little_endian
     offset = data_element_offset_to_value(is_implicit_VR, raw_data_elem.VR)
diff --git a/pydicom/jsonrep.py b/pydicom/jsonrep.py
index d96e79cfa..2d5c12c8e 100644
--- a/pydicom/jsonrep.py
+++ b/pydicom/jsonrep.py
@@ -1,9 +1,14 @@
 # Copyright 2008-2019 pydicom authors. See LICENSE file for details.
 """Methods for converting Datasets and DataElements to/from json"""
 
-# Order of keys is significant!
+import base64
+import warnings
+
+from pydicom import compat
 from pydicom.compat import int_type
+from pydicom.valuerep import PersonNameUnicode
 
+# Order of keys is significant!
 JSON_VALUE_KEYS = ('Value', 'BulkDataURI', 'InlineBinary',)
 
 BINARY_VR_VALUES = ['OW', 'OB', 'OD', 'OF', 'OL', 'UN',
@@ -41,3 +46,203 @@ def convert_to_python_number(value, vr):
         else:
             value = number_type(value)
     return value
+
+
+class JsonDataElementConverter(object):
+    """Handles conversion between JSON struct and :class:`DataElement`."""
+
+    def __init__(self, dataset_class, tag, vr, value, value_key,
+                 bulk_data_uri_handler):
+        """Create a new converter instance.
+
+        Parameters
+        ----------
+        dataset_class : dataset.Dataset derived class
+            Class used to create sequence items.
+        tag : BaseTag
+            The data element tag.
+        vr : str
+            The data element value representation.
+        value : list
+            The data element's value(s).
+        value_key : str or None
+            Key of the data element that contains the value
+            (options: ``{"Value", "InlineBinary", "BulkDataURI"}``)
+        bulk_data_uri_handler: callable or None
+            Callable function that accepts the "BulkDataURI" of the JSON
+            representation of a data element and returns the actual value of
+            that data element (retrieved via DICOMweb WADO-RS)
+        """
+        self.dataset_class = dataset_class
+        self.tag = tag
+        self.vr = vr
+        self.value = value
+        self.value_key = value_key
+        self.bulk_data_uri_handler = bulk_data_uri_handler
+
+    def get_element_values(self):
+        """Return a the data element value or list of values.
+
+        Returns
+        -------
+        str or bytes or int or float or dataset_class
+        or PersonName3 or PersonNameUnicode or list of any of these types
+            The value or value list of the newly created data element.
+        """
+        from pydicom.dataelem import empty_value_for_VR
+        if self.value_key == 'Value':
+            if not isinstance(self.value, list):
+                fmt = '"{}" of data element "{}" must be a list.'
+                raise TypeError(fmt.format(self.value_key, self.tag))
+            if not self.value:
+                return empty_value_for_VR(self.vr)
+            element_value = [self.get_regular_element_value(v)
+                             for v in self.value]
+            if len(element_value) == 1 and self.vr != 'SQ':
+                element_value = element_value[0]
+            return convert_to_python_number(element_value, self.vr)
+
+        # The value for "InlineBinary" shall be encoded as a base64 encoded
+        # string, as shown in PS3.18, Table F.3.1-1, but the example in
+        # PS3.18, Annex F.4 shows the string enclosed in a list.
+        # We support both variants, as the standard is ambiguous here,
+        # and do the same for "BulkDataURI".
+        value = self.value
+        if isinstance(value, list):
+            value = value[0]
+
+        if self.value_key == 'InlineBinary':
+            if not isinstance(value, compat.char_types):
+                fmt = '"{}" of data element "{}" must be a bytes-like object.'
+                raise TypeError(fmt.format(self.value_key, self.tag))
+            return base64.b64decode(value)
+
+        if self.value_key == 'BulkDataURI':
+            if not isinstance(value, compat.string_types):
+                fmt = '"{}" of data element "{}" must be a string.'
+                raise TypeError(fmt.format(self.value_key, self.tag))
+            if self.bulk_data_uri_handler is None:
+                warnings.warn(
+                    'no bulk data URI handler provided for retrieval '
+                    'of value of data element "{}"'.format(self.tag)
+                )
+                return empty_value_for_VR(self.vr, raw=True)
+            return self.bulk_data_uri_handler(value)
+        return empty_value_for_VR(self.vr)
+
+    def get_regular_element_value(self, value):
+        """Return a the data element value created from a json "Value" entry.
+
+        Parameters
+        ----------
+        value : str or int or float or dict
+            The data element's value from the json entry.
+
+        Returns
+        -------
+        dataset_class or PersonName3 or PersonNameUnicode
+        or str or int or float
+            A single value of the corresponding :class:`DataElement`.
+        """
+        if self.vr == 'SQ':
+            return self.get_sequence_item(value)
+
+        if self.vr == 'PN':
+            return self.get_pn_element_value(value)
+
+        if self.vr == 'AT':
+            try:
+                return int(value, 16)
+            except ValueError:
+                warnings.warn('Invalid value "{}" for AT element - '
+                              'ignoring it'.format(value))
+            return
+        return value
+
+    def get_sequence_item(self, value):
+        """Return a sequence item for the JSON dict `value`.
+
+        Parameters
+        ----------
+        value : dict or None
+            The sequence item from the JSON entry.
+
+        Returns
+        -------
+        dataset_class
+            The decoded dataset item.
+
+        Raises
+        ------
+        KeyError
+            If the "vr" key is missing for a contained element
+        """
+        ds = self.dataset_class()
+        if value:
+            for key, val in value.items():
+                if 'vr' not in val:
+                    fmt = 'Data element "{}" must have key "vr".'
+                    raise KeyError(fmt.format(self.tag))
+                vr = val['vr']
+                unique_value_keys = tuple(
+                    set(val.keys()) & set(JSON_VALUE_KEYS)
+                )
+                from pydicom import DataElement
+                from pydicom.dataelem import empty_value_for_VR
+                if not unique_value_keys:
+                    # data element with no value
+                    elem = DataElement(
+                        tag=int(key, 16),
+                        value=empty_value_for_VR(vr),
+                        VR=vr)
+                else:
+                    value_key = unique_value_keys[0]
+                    elem = DataElement.from_json(
+                        self.dataset_class, key, vr,
+                        val[value_key], value_key
+                    )
+                ds.add(elem)
+        return ds
+
+    def get_pn_element_value(self, value):
+        """Return PersonName value from JSON value.
+
+        Values with VR PN have a special JSON encoding, see the DICOM Standard,
+        Part 18, :dcm:`Annex F.2.2<part18/sect_F.2.2.html>`.
+
+        Parameters
+        ----------
+        value : dict
+            The person name components in the JSON entry.
+
+        Returns
+        -------
+        PersonName3 or PersonNameUnicode or str
+            The decoded PersonName object or an empty string.
+        """
+        if not isinstance(value, dict):
+            # Some DICOMweb services get this wrong, so we
+            # workaround the issue and warn the user
+            # rather than raising an error.
+            warnings.warn(
+                'value of data element "{}" with VR Person Name (PN) '
+                'is not formatted correctly'.format(self.tag)
+            )
+            return value
+        else:
+            if 'Phonetic' in value:
+                comps = ['', '', '']
+            elif 'Ideographic' in value:
+                comps = ['', '']
+            else:
+                comps = ['']
+            if 'Alphabetic' in value:
+                comps[0] = value['Alphabetic']
+            if 'Ideographic' in value:
+                comps[1] = value['Ideographic']
+            if 'Phonetic' in value:
+                comps[2] = value['Phonetic']
+            elem_value = '='.join(comps)
+            if compat.in_py2:
+                elem_value = PersonNameUnicode(elem_value, 'UTF8')
+            return elem_value


Test patch:
diff --git a/pydicom/tests/test_filereader.py b/pydicom/tests/test_filereader.py
index cc35fbba2..ad3f350d0 100644
--- a/pydicom/tests/test_filereader.py
+++ b/pydicom/tests/test_filereader.py
@@ -3,6 +3,7 @@
 """Unit tests for the pydicom.filereader module."""
 
 import gzip
+import io
 from io import BytesIO
 import os
 import shutil
@@ -918,6 +919,14 @@ class TestDeferredRead(object):
         # the right place, it was re-opened as a normal file, not a zip file
         ds.InstanceNumber
 
+    def test_filelike_deferred(self):
+        """Deferred values work with file-like objects."""
+        with open(ct_name, 'rb') as fp:
+            data = fp.read()
+        filelike = io.BytesIO(data)
+        dataset = pydicom.dcmread(filelike, defer_size=1024)
+        assert 32768 == len(dataset.PixelData)
+
 
 class TestReadTruncatedFile(object):
     def testReadFileWithMissingPixelData(self):
diff --git a/pydicom/tests/test_json.py b/pydicom/tests/test_json.py
index 3a078d6db..924e3b480 100644
--- a/pydicom/tests/test_json.py
+++ b/pydicom/tests/test_json.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Copyright 2008-2019 pydicom authors. See LICENSE file for details.
 import json
+import sys
 
 import pytest
 
@@ -13,7 +14,7 @@ from pydicom.valuerep import PersonNameUnicode, PersonName3
 
 
 class TestPersonName(object):
-    def test_json_PN_from_file(self):
+    def test_json_pn_from_file(self):
         with open(get_testdata_files("test_PN.json")[0]) as s:
             ds = Dataset.from_json(s.read())
         assert isinstance(ds[0x00080090].value,
@@ -24,7 +25,7 @@ class TestPersonName(object):
         dataelem = inner_seq[0][0x00100010]
         assert isinstance(dataelem.value, (PersonNameUnicode, PersonName3))
 
-    def test_PN_components_to_json(self):
+    def test_pn_components_to_json(self):
         def check_name(tag, components):
             # we cannot directly compare the dictionaries, as they are not
             # ordered in Python 2
@@ -50,7 +51,7 @@ class TestPersonName(object):
         ds.add_new(0x00091005, 'PN', u'==やまだ^たろう')
         ds.add_new(0x00091006, 'PN', u'=山田^太郎')
         ds.add_new(0x00091007, 'PN', u'Yamada^Tarou=山田^太郎')
-        ds_json = json.loads(ds.to_json())
+        ds_json = ds.to_json_dict()
         check_name('00100010', ['Yamada^Tarou', u'山田^太郎', u'やまだ^たろう'])
         check_name('00091001', ['Yamada^Tarou'])
         check_name('00091002', ['Yamada^Tarou'])
@@ -60,7 +61,7 @@ class TestPersonName(object):
         check_name('00091006', ['', u'山田^太郎'])
         check_name('00091007', ['Yamada^Tarou', u'山田^太郎'])
 
-    def test_PN_components_from_json(self):
+    def test_pn_components_from_json(self):
         # this is the encoded dataset from the previous test, with some
         # empty components omitted
         ds_json = (u'{"00100010": {"vr": "PN", "Value": [{"Alphabetic": '
@@ -100,7 +101,7 @@ class TestPersonName(object):
     def test_empty_value(self):
         ds = Dataset()
         ds.add_new(0x00100010, 'PN', '')
-        ds_json = json.loads(ds.to_json())
+        ds_json = ds.to_json_dict()
         assert '00100010' in ds_json
         assert 'Value' not in ds_json['00100010']
 
@@ -108,7 +109,7 @@ class TestPersonName(object):
         ds = Dataset()
         patient_names = [u'Buc^Jérôme', u'Διονυσιος', u'Люкceмбypг']
         ds.add_new(0x00091001, 'PN', patient_names)
-        ds_json = json.loads(ds.to_json())
+        ds_json = ds.to_json_dict()
         assert [{'Alphabetic': u'Buc^Jérôme'},
                 {'Alphabetic': u'Διονυσιος'},
                 {'Alphabetic': u'Люкceмбypг'}] == ds_json['00091001']['Value']
@@ -128,7 +129,7 @@ class TestAT(object):
         ds.add_new(0x00091002, 'AT', Tag(0x28, 0x02))
         ds.add_new(0x00091003, 'AT', BaseTag(0x00280002))
         ds.add_new(0x00091004, 'AT', [0x00280002, Tag('PatientName')])
-        ds_json = json.loads(ds.to_json())
+        ds_json = ds.to_json_dict()
 
         assert ['00100010', '00100020'] == ds_json['00091001']['Value']
         assert ['00280002'] == ds_json['00091002']['Value']
@@ -160,6 +161,10 @@ class TestDataSetToJson(object):
         ds2 = Dataset.from_json(ds_json)
         assert ds1 == ds2
 
+        ds_json = ds1.to_json_dict(bulk_data_threshold=100000)
+        ds2 = Dataset.from_json(ds_json)
+        assert ds1 == ds2
+
     def test_roundtrip(self):
         ds = Dataset()
         ds.add_new(0x00080005, 'CS', 'ISO_IR 100')
@@ -202,6 +207,7 @@ class TestDataSetToJson(object):
 
         json_string = ds.to_json(bulk_data_threshold=100)
         json_model = json.loads(json_string)
+
         assert json_model['00080005']['Value'] == ['ISO_IR 100']
         assert json_model['00091007']['Value'] == ['1.2.3.4.5.6']
         assert json_model['0009100A']['Value'] == ['20200101115500.000000']
@@ -214,13 +220,112 @@ class TestDataSetToJson(object):
 
         ds2 = Dataset.from_json(json_string)
         assert ds == ds2
+        ds2 = Dataset.from_json(json_model)
+        assert ds == ds2
+
+        json_model2 = ds.to_json_dict(bulk_data_threshold=100)
+        if compat.in_py2:
+            # in Python 2, the encoding of this is slightly different
+            # (single vs double quotation marks)
+            del json_model['00091015']
+            del json_model2['00091015']
+
+        assert json_model == json_model2
+
+    def test_dataset_dumphandler(self):
+        ds = Dataset()
+        ds.add_new(0x00100010, 'PN', 'Jane^Doe')
+        # as the order of the keys is not defined, we have to check both
+        assert ds.to_json() in ('{"00100010": {"vr": "PN", "Value": [{'
+                                '"Alphabetic": "Jane^Doe"}]}}',
+                                '{"00100010": {"Value": [{'
+                                '"Alphabetic": "Jane^Doe"}], "vr": "PN"}}')
+
+        assert {
+                   "00100010": {
+                       "vr": "PN", "Value": [{"Alphabetic": "Jane^Doe"}]}
+               } == ds.to_json(dump_handler=lambda d: d)
+
+    def test_dataelement_dumphandler(self):
+        element = DataElement(0x00100010, 'PN', 'Jane^Doe')
+        # as the order of the keys is not defined, we have to check both
+        assert element.to_json() in ('{"vr": "PN", "Value": [{'
+                                     '"Alphabetic": "Jane^Doe"}]}',
+                                     '{"Value": [{'
+                                     '"Alphabetic": "Jane^Doe"}], "vr": "PN"}')
+
+        assert {
+                   "vr": "PN", "Value": [{"Alphabetic": "Jane^Doe"}]
+               } == element.to_json(dump_handler=lambda d: d)
+
+    def test_sort_order(self):
+        """Test that tags are serialized in ascending order."""
+        ds = Dataset()
+        ds.add_new(0x00100040, 'CS', 'F')
+        ds.add_new(0x00100030, 'DA', '20000101')
+        ds.add_new(0x00100020, 'LO', '0017')
+        ds.add_new(0x00100010, 'PN', 'Jane^Doe')
+
+        ds_json = ds.to_json()
+        assert ds_json.index('"00100010"') < ds_json.index('"00100020"')
+        assert ds_json.index('"00100020"') < ds_json.index('"00100030"')
+        assert ds_json.index('"00100030"') < ds_json.index('"00100040"')
 
-    def test_json_private_DS_VM(self):
+
+class TestSequence(object):
+    def test_nested_sequences(self):
         test1_json = get_testdata_files("test1.json")[0]
-        jsonmodel = open(test1_json, 'r').read()
-        ds = Dataset.from_json(jsonmodel)
-        jsonmodel2 = ds.to_json(dump_handler=lambda d: json.dumps(d, indent=2))
-        ds2 = Dataset.from_json(jsonmodel2)
+        with open(test1_json) as f:
+            with pytest.warns(UserWarning,
+                              match='no bulk data URI handler provided '):
+                ds = Dataset.from_json(f.read())
+            del ds.PixelData
+        ds2 = Dataset.from_json(ds.to_json())
+        assert ds == ds2
+
+
+class TestBinary(object):
+    def test_inline_binary(self):
+        ds = Dataset()
+        ds.add_new(0x00091002, 'OB', b'BinaryContent')
+        ds_json = ds.to_json_dict(bulk_data_threshold=20)
+        assert "00091002" in ds_json
+        assert "QmluYXJ5Q29udGVudA==" == ds_json["00091002"]["InlineBinary"]
+        ds1 = Dataset.from_json(ds_json)
+        assert ds == ds1
+        # also test if the binary is enclosed in a list
+        ds_json["00091002"]["InlineBinary"] = ["QmluYXJ5Q29udGVudA=="]
+        ds1 = Dataset.from_json(ds_json)
+        assert ds == ds1
+
+    def test_invalid_inline_binary(self):
+        msg = ('"InlineBinary" of data element "00091002" '
+               'must be a bytes-like object.')
+        ds_json = '{"00091002": {"vr": "OB", "InlineBinary": 42}}'
+        with pytest.raises(TypeError, match=msg):
+            Dataset.from_json(ds_json)
+
+        ds_json = '{"00091002": {"vr": "OB", "InlineBinary": [42]}}'
+        with pytest.raises(TypeError, match=msg):
+            Dataset.from_json(ds_json)
+
+    def test_valid_bulkdata_uri(self):
+        ds_json = ('{"00091002": {"vr": "OB", "BulkDataURI": '
+                   '"http://example.com/bulkdatahandler"}}')
+        ds = Dataset.from_json(ds_json)
+        assert 0x00091002 in ds
+        ds_json = ('{"00091002": {"vr": "OB", "BulkDataURI": '
+                   '["http://example.com/bulkdatahandler"]}}')
+        ds = Dataset.from_json(ds_json)
+        assert 0x00091002 in ds
+
+    def test_invalid_bulkdata_uri(self):
+        msg = ('"BulkDataURI" of data element "00091002" '
+               'must be a string.')
+        ds_json = '{"00091002": {"vr": "OB", "BulkDataURI": 42}}'
+        with pytest.raises(TypeError, match=msg):
+            Dataset.from_json(ds_json)
 
-        assert ds.PatientIdentityRemoved == 'YES'
-        assert ds2.PatientIdentityRemoved == 'YES'
+        ds_json = '{"00091002": {"vr": "OB", "BulkDataURI": [42]}}'
+        with pytest.raises(TypeError, match=msg):
+            Dataset.from_json(ds_json)