Repository: spyder-ide/spyder

Issue:
Spyder stops executing commands if Variable explorer shows complicated objects
## Description
I often found that spyder sometimes becomes unusable because some complicated object is present in the variable explorer
even running 1+1 just hangs and takes ages to be executed
Here is a working example

run the following code:

```python
from collections import defaultdict
import numpy as np
bug = defaultdict(list)
for i in range(50000):
    a = {j:np.random.rand(10) for j in range(10)}
    bug[i] = a
```
    
Now running even the simplest stuff in the ipython console is unbearably slow.
changing `bug` to be of type dict solves it


**Please provide any additional information below**
I dont have autorefresh enabled (I didn't change any options in th Varibale explorer)

My guess is that the code that creates the string representation for variable explorer doesn't have an optimization for very large defaultdict.
This makes defaultdict unusable for me.
## Version and main components

* Spyder Version: 3.1.3 
* Python Version: 2.7.12
* Qt Versions: 5.6.2, PyQt5 5.6 on Linux

## Dependencies
```
pyflakes >=0.5.0 :  1.3.0 (OK)
pep8 >=0.6       :  1.7.0 (OK)
pygments >=2.0   :  2.1.3 (OK)
qtconsole >=4.2.0:  4.2.1 (OK)
nbconvert >=4.0  :  4.2.0 (OK)
pandas >=0.13.1  :  0.19.2 (OK)
numpy >=1.7      :  1.11.2 (OK)
sphinx >=0.6.6   :  1.4.6 (OK)
rope >=0.9.4     :  0.9.4 (OK)
jedi =0.9.0      :  0.9.0 (OK)
psutil >=0.3     :  4.3.1 (OK)
matplotlib >=1.0 :  1.5.3 (OK)
sympy >=0.7.3    :  1.0 (OK)
pylint >=0.25    :  1.5.4 (OK)

```



Discussion:
Thanks for the report and for the reproducible example. We'll take a look at this problem for Spyder 3.2

Resolving patch:
diff --git a/spyder/widgets/variableexplorer/utils.py b/spyder/widgets/variableexplorer/utils.py
--- a/spyder/widgets/variableexplorer/utils.py
+++ b/spyder/widgets/variableexplorer/utils.py
@@ -5,18 +5,19 @@
 # (see spyder/__init__.py for details)
 
 """
-Utilities for the Collections editor widget and dialog
+Utilities
 """
 
 from __future__ import print_function
 
+from itertools import islice
 import re
 
 # Local imports
 from spyder.config.base import get_supported_types
 from spyder.py3compat import (NUMERIC_TYPES, TEXT_TYPES, to_text_string,
-                              is_text_string, is_binary_string, reprlib,
-                              PY2, to_binary_string)
+                              is_text_string, is_binary_string, PY2,
+                              to_binary_string, iteritems)
 from spyder.utils import programs
 from spyder import dependencies
 from spyder.config.base import _
@@ -152,17 +153,6 @@ def get_object_attrs(obj):
     return attrs
 
 
-# =============================================================================
-# Set limits for the amount of elements in the repr of collections (lists,
-# dicts, tuples and sets) and Numpy arrays
-# =============================================================================
-CollectionsRepr = reprlib.Repr()
-CollectionsRepr.maxlist = 10
-CollectionsRepr.maxdict = 10
-CollectionsRepr.maxtuple = 10
-CollectionsRepr.maxset = 10
-
-
 #==============================================================================
 # Date and datetime objects support
 #==============================================================================
@@ -256,10 +246,71 @@ def unsorted_unique(lista):
 #==============================================================================
 # Display <--> Value
 #==============================================================================
-def value_to_display(value, minmax=False):
+def default_display(value, with_module=True):
+    """Default display for unknown objects."""
+    object_type = type(value)
+    try:
+        name = object_type.__name__
+        module = object_type.__module__
+        if with_module:
+            return name + ' object of ' + module + ' module'
+        else:
+            return name
+    except:
+        type_str = to_text_string(object_type)
+        return type_str[1:-1]
+
+
+def collections_display(value, level):
+    """Display for collections (i.e. list, tuple and dict)."""
+    is_dict = isinstance(value, dict)
+
+    # Get elements
+    if is_dict:
+        elements = iteritems(value)
+    else:
+        elements = value
+
+    # Truncate values
+    truncate = False
+    if level == 1 and len(value) > 10:
+        elements = islice(elements, 10) if is_dict else value[:10]
+        truncate = True
+    elif level == 2 and len(value) > 5:
+        elements = islice(elements, 5) if is_dict else value[:5]
+        truncate = True
+
+    # Get display of each element
+    if level <= 2:
+        if is_dict:
+            displays = [value_to_display(k, level=level) + ':' +
+                        value_to_display(v, level=level)
+                        for (k, v) in list(elements)]
+        else:
+            displays = [value_to_display(e, level=level)
+                        for e in elements]
+        if truncate:
+            displays.append('...')
+        display = ', '.join(displays)
+    else:
+        display = '...'
+
+    # Return display
+    if is_dict:
+        display = '{' + display + '}'
+    elif isinstance(value, list):
+        display = '[' + display + ']'
+    else:
+        display = '(' + display + ')'
+
+    return display
+
+
+def value_to_display(value, minmax=False, level=0):
     """Convert value for display purpose"""
     # To save current Numpy threshold
     np_threshold = FakeObject
+
     try:
         numeric_numpy_types = (int64, int32, float64, float32,
                                complex128, complex64)
@@ -270,68 +321,93 @@ def value_to_display(value, minmax=False):
             # in our display
             set_printoptions(threshold=10)
         if isinstance(value, recarray):
-            fields = value.names
-            display = 'Field names: ' + ', '.join(fields)
+            if level == 0:
+                fields = value.names
+                display = 'Field names: ' + ', '.join(fields)
+            else:
+                display = 'Recarray'
         elif isinstance(value, MaskedArray):
             display = 'Masked array'
         elif isinstance(value, ndarray):
-            if minmax:
-                try:
-                    display = 'Min: %r\nMax: %r' % (value.min(), value.max())
-                except (TypeError, ValueError):
+            if level == 0:
+                if minmax:
+                    try:
+                        display = 'Min: %r\nMax: %r' % (value.min(), value.max())
+                    except (TypeError, ValueError):
+                        if value.dtype.type in numeric_numpy_types:
+                            display = repr(value)
+                        else:
+                            display = default_display(value)
+                elif value.dtype.type in numeric_numpy_types:
                     display = repr(value)
+                else:
+                    display = default_display(value)
             else:
-                display = repr(value)
-        elif isinstance(value, (list, tuple, dict, set)):
-            display = CollectionsRepr.repr(value)
+                display = 'Numpy array'
+        elif any([type(value) == t for t in [list, tuple, dict]]):
+            display = collections_display(value, level+1)
         elif isinstance(value, Image):
-            display = '%s  Mode: %s' % (address(value), value.mode)
+            if level == 0:
+                display = '%s  Mode: %s' % (address(value), value.mode)
+            else:
+                display = 'Image'
         elif isinstance(value, DataFrame):
-            cols = value.columns
-            if PY2 and len(cols) > 0:
-                # Get rid of possible BOM utf-8 data present at the
-                # beginning of a file, which gets attached to the first
-                # column header when headers are present in the first
-                # row.
-                # Fixes Issue 2514
-                try:
-                    ini_col = to_text_string(cols[0], encoding='utf-8-sig')
-                except:
-                    ini_col = to_text_string(cols[0])
-                cols = [ini_col] + [to_text_string(c) for c in cols[1:]]
+            if level == 0:
+                cols = value.columns
+                if PY2 and len(cols) > 0:
+                    # Get rid of possible BOM utf-8 data present at the
+                    # beginning of a file, which gets attached to the first
+                    # column header when headers are present in the first
+                    # row.
+                    # Fixes Issue 2514
+                    try:
+                        ini_col = to_text_string(cols[0], encoding='utf-8-sig')
+                    except:
+                        ini_col = to_text_string(cols[0])
+                    cols = [ini_col] + [to_text_string(c) for c in cols[1:]]
+                else:
+                    cols = [to_text_string(c) for c in cols]
+                display = 'Column names: ' + ', '.join(list(cols))
             else:
-                cols = [to_text_string(c) for c in cols]
-            display = 'Column names: ' + ', '.join(list(cols))
+                display = 'Dataframe'
         elif isinstance(value, NavigableString):
             # Fixes Issue 2448
             display = to_text_string(value)
+            if level > 0:
+                display = u"'" + display + u"'"
         elif isinstance(value, DatetimeIndex):
-            display = value.summary()
+            if level == 0:
+                display = value.summary()
+            else:
+                display = 'DatetimeIndex'
         elif is_binary_string(value):
             try:
                 display = to_text_string(value, 'utf8')
             except:
                 display = value
+            if level > 0:
+                display = (to_binary_string("'") + display +
+                           to_binary_string("'"))
         elif is_text_string(value):
             display = value
-        elif isinstance(value, NUMERIC_TYPES) or isinstance(value, bool) or \
-          isinstance(value, datetime.date) or \
-          isinstance(value, numeric_numpy_types):
+            if level > 0:
+                display = u"'" + display + u"'"
+        elif (isinstance(value, NUMERIC_TYPES) or isinstance(value, bool) or
+              isinstance(value, datetime.date) or
+              isinstance(value, numeric_numpy_types)):
             display = repr(value)
         else:
-            # Note: Don't trust on repr's. They can be inefficient and
-            # so freeze Spyder quite easily
-            # display = repr(value)
-            type_str = to_text_string(type(value))
-            display = type_str[1:-1]
+            if level == 0:
+                display = default_display(value)
+            else:
+                display = default_display(value, with_module=False)
     except:
-        type_str = to_text_string(type(value))
-        display = type_str[1:-1]
+        display = default_display(value)
 
-    # Truncate display at 80 chars to avoid freezing Spyder
+    # Truncate display at 70 chars to avoid freezing Spyder
     # because of large displays
-    if len(display) > 80:
-        display = display[:80].rstrip() + ' ...'
+    if len(display) > 70:
+        display = display[:70].rstrip() + ' ...'
 
     # Restore Numpy threshold
     if np_threshold is not FakeObject:
@@ -340,7 +416,6 @@ def value_to_display(value, minmax=False):
     return display
 
 
-
 def display_to_value(value, default_value, ignore_errors=True):
     """Convert back to value"""
     from qtpy.compat import from_qvariant


Test patch:
diff --git a/spyder/widgets/variableexplorer/tests/test_utils.py b/spyder/widgets/variableexplorer/tests/test_utils.py
--- a/spyder/widgets/variableexplorer/tests/test_utils.py
+++ b/spyder/widgets/variableexplorer/tests/test_utils.py
@@ -7,12 +7,31 @@
 Tests for utils.py
 """
 
+from collections import defaultdict
+
 # Third party imports
+import numpy as np
+import pandas as pd
 import pytest
 
 # Local imports
 from spyder.config.base import get_supported_types
-from spyder.widgets.variableexplorer.utils import sort_against, is_supported
+from spyder.widgets.variableexplorer.utils import (sort_against,
+    is_supported, value_to_display)
+
+
+def generate_complex_object():
+    """Taken from issue #4221."""
+    bug = defaultdict(list)
+    for i in range(50000):
+        a = {j:np.random.rand(10) for j in range(10)}
+        bug[i] = a
+    return bug
+
+
+COMPLEX_OBJECT = generate_complex_object()
+DF = pd.DataFrame([1,2,3])
+PANEL = pd.Panel({0: pd.DataFrame([1,2]), 1:pd.DataFrame([3,4])})
 
 
 # --- Tests
@@ -32,7 +51,7 @@ def test_sort_against_is_stable():
 
 
 def test_none_values_are_supported():
-    # None values should be displayed by default
+    """Tests that None values are displayed by default"""
     supported_types = get_supported_types()
     mode = 'editable'
     none_var = None
@@ -45,5 +64,90 @@ def test_none_values_are_supported():
     assert is_supported(none_tuple, filters=tuple(supported_types[mode]))
 
 
+def test_default_display():
+    """Tests for default_display."""
+    # Display of defaultdict
+    assert (value_to_display(COMPLEX_OBJECT) ==
+            'defaultdict object of collections module')
+
+    # Display of array of COMPLEX_OBJECT
+    assert (value_to_display(np.array(COMPLEX_OBJECT)) ==
+            'ndarray object of numpy module')
+
+    # Display of Panel
+    assert (value_to_display(PANEL) ==
+            'Panel object of pandas.core.panel module')
+
+
+def test_list_display():
+    """Tests for display of lists."""
+    long_list = list(range(100))
+
+    # Simple list
+    assert value_to_display([1, 2, 3]) == '[1, 2, 3]'
+
+    # Long list
+    assert (value_to_display(long_list) ==
+            '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...]')
+
+    # Short list of lists
+    assert (value_to_display([long_list] * 3) ==
+            '[[0, 1, 2, 3, 4, ...], [0, 1, 2, 3, 4, ...], [0, 1, 2, 3, 4, ...]]')
+
+    # Long list of lists
+    result = '[' + ''.join('[0, 1, 2, 3, 4, ...], '*10)[:-2] + ']'
+    assert value_to_display([long_list] * 10) == result[:70] + ' ...'
+
+    # Multiple level lists
+    assert (value_to_display([[1, 2, 3, [4], 5]] + long_list) ==
+            '[[1, 2, 3, [...], 5], 0, 1, 2, 3, 4, 5, 6, 7, 8, ...]')
+    assert value_to_display([1, 2, [DF]]) == '[1, 2, [Dataframe]]'
+    assert value_to_display([1, 2, [[DF], PANEL]]) == '[1, 2, [[...], Panel]]'
+
+    # List of complex object
+    assert value_to_display([COMPLEX_OBJECT]) == '[defaultdict]'
+
+    # List of composed objects
+    li = [COMPLEX_OBJECT, PANEL, 1, {1:2, 3:4}, DF]
+    result = '[defaultdict, Panel, 1, {1:2, 3:4}, Dataframe]'
+    assert value_to_display(li) == result
+
+
+def test_dict_display():
+    """Tests for display of dicts."""
+    long_list = list(range(100))
+    long_dict = dict(zip(list(range(100)), list(range(100))))
+
+    # Simple dict
+    assert value_to_display({0:0, 'a':'b'}) == "{0:0, 'a':'b'}"
+
+    # Long dict
+    assert (value_to_display(long_dict) ==
+            '{0:0, 1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8, 9:9, ...}')
+
+    # Short list of lists
+    assert (value_to_display({1:long_dict, 2:long_dict}) ==
+            '{1:{0:0, 1:1, 2:2, 3:3, 4:4, ...}, 2:{0:0, 1:1, 2:2, 3:3, 4:4, ...}}')
+
+    # Long dict of dicts
+    result = ('{(0, 0, 0, 0, 0, ...):[0, 1, 2, 3, 4, ...], '
+               '(1, 1, 1, 1, 1, ...):[0, 1, 2, 3, 4, ...]}')
+    assert value_to_display({(0,)*100:long_list, (1,)*100:long_list}) == result[:70] + ' ...'
+
+    # Multiple level dicts
+    assert (value_to_display({0: {1:1, 2:2, 3:3, 4:{0:0}, 5:5}, 1:1}) ==
+            '{0:{1:1, 2:2, 3:3, 4:{...}, 5:5}, 1:1}')
+    assert value_to_display({0:0, 1:1, 2:2, 3:DF}) == '{0:0, 1:1, 2:2, 3:Dataframe}'
+    assert value_to_display({0:0, 1:1, 2:[[DF], PANEL]}) == '{0:0, 1:1, 2:[[...], Panel]}'
+
+    # Dict of complex object
+    assert value_to_display({0:COMPLEX_OBJECT}) == '{0:defaultdict}'
+
+    # Dict of composed objects
+    li = {0:COMPLEX_OBJECT, 1:PANEL, 2:2, 3:{0:0, 1:1}, 4:DF}
+    result = '{0:defaultdict, 1:Panel, 2:2, 3:{0:0, 1:1}, 4:Dataframe}'
+    assert value_to_display(li) == result
+
+
 if __name__ == "__main__":
     pytest.main()