Repository: streamlit/streamlit

Issue:
Lists inside of DataFrames are unhashable and break st.cache
# Summary

A `DataFrame` which contains a list is unhashable and therefore breaks `st.cache`.

## Steps to reproduce

Run this code
```python
import streamlit as st
import pandas as pd

@st.cache
def return_a_list(n):
    return pd.DataFrame({'n': [n], 'result': [list(range(n))]})


st.text(return_a_list(10))
```

# Behavior

## Actual behavior

We get an error with the following stack trace
```
TypeError: unhashable type: 'list'
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/ScriptRunner.py", line 317, in _run_script exec(code, module.__dict__)
File "/Users/adrien/Desktop/streamlit/streamlit-staging/test_list_bug.py", line 9, in <module> st.text(return_a_list(10))
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/caching.py", line 412, in wrapped_func key, return_value, persist, ignore_hash, args_mutated)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/caching.py", line 307, in _write_to_cache _write_to_mem_cache(key, value, ignore_hash, args_mutated)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/caching.py", line 237, in _write_to_mem_cache hash=None if ignore_hash else get_hash(value),
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/hashing.py", line 89, in get_hash hasher.update(f, context)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/hashing.py", line 159, in update self._update(self.hasher, obj, context)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/hashing.py", line 190, in _update b = self.to_bytes(obj, context)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/hashing.py", line 179, in to_bytes b = self._to_bytes(obj, context)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/streamlit/hashing.py", line 230, in _to_bytes return pd.util.hash_pandas_object(obj).sum()
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/pandas/core/util/hashing.py", line 115, in hash_pandas_object h = _combine_hash_arrays(hashes, num_items)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/pandas/core/util/hashing.py", line 41, in _combine_hash_arrays for i, a in enumerate(arrays):
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/pandas/core/util/hashing.py", line 104, in <genexpr> hashes = (hash_array(series.values) for _, series in obj.iteritems())
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/pandas/core/util/hashing.py", line 286, in hash_array codes, categories = factorize(vals, sort=False)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/pandas/util/_decorators.py", line 178, in wrapper return func(*args, **kwargs)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/pandas/core/algorithms.py", line 630, in factorize na_value=na_value)
File "/Users/adrien/.pyenv/versions/3.6.3/envs/streamlit-staging/lib/python3.6/site-packages/pandas/core/algorithms.py", line 476, in _factorize_array na_value=na_value)
File "pandas/_libs/hashtable_class_helper.pxi", line 1601, in pandas._libs.hashtable.PyObjectHashTable.get_labels
```

## Expected behavior

This should work, just as if `return_a_list` weren't decorated with `@st.cache`.

## Is this a regression?

Not sure.

# Debug info
```
$ streamlit version && python --version && pyenv --version && sw_vers && "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --version
Streamlit, version 0.45.0
Python 3.6.3
pyenv 1.2.3
ProductName:	Mac OS X
ProductVersion:	10.14.6
BuildVersion:	18G95
Google Chrome 76.0.3809.132 
```




Discussion:
This issue was first observed here: https://discuss.streamlit.io/t/using-caching-with-api-calls-and-messy-dataframes/75

Resolving patch:
diff --git a/lib/streamlit/hashing.py b/lib/streamlit/hashing.py
--- a/lib/streamlit/hashing.py
+++ b/lib/streamlit/hashing.py
@@ -142,6 +142,17 @@ def is_simple(obj):
     return None
 
 
+def _hashing_error_message(start):
+    return (
+        start,
+        "\n\n**More information:** to prevent unexpected behavior, Streamlit tries to detect mutations in cached objects so it can alert the user if needed. However, something went wrong while performing this check.\n\n"
+        "Please [file a bug](https://github.com/streamlit/streamlit/issues/new/choose).\n\n"
+        "To stop this warning from showing in the meantime, try one of the following:\n"
+        "* **Preferred:** modify your code to avoid using this type of object.\n"
+        "* Or add the argument `ignore_cache=True` to the `st.cache` decorator.",
+    )
+
+
 class CodeHasher:
     """A hasher that can hash code objects including dependencies."""
 
@@ -202,125 +213,138 @@ def _to_bytes(self, obj, context):
         Python's built in `hash` does not produce consistent results across
         runs."""
 
-        if _is_magicmock(obj):
-            # MagicMock can result in objects that appear to be infinitely
-            # deep, so we don't try to hash them at all.
-            return self.to_bytes(id(obj))
-        elif isinstance(obj, bytes) or isinstance(obj, bytearray):
-            return obj
-        elif isinstance(obj, string_types):
-            return obj.encode()
-        elif isinstance(obj, float):
-            return self.to_bytes(hash(obj))
-        elif isinstance(obj, int):
-            return _int_to_bytes(obj)
-        elif isinstance(obj, list) or isinstance(obj, tuple):
-            h = hashlib.new(self.name)
-            # add type to distingush x from [x]
-            self._update(h, type(obj).__name__.encode() + b":")
-            for e in obj:
-                self._update(h, e, context)
-            return h.digest()
-        elif obj is None:
-            # Special string since hashes change between sessions.
-            # We don't use Python's `hash` since hashes are not consistent
-            # across runs.
-            return b"none:"
-        elif obj is True:
-            return b"bool:1"
-        elif obj is False:
-            return b"bool:0"
-        elif util.is_type(obj, "pandas.core.frame.Series"):
-            import pandas as pd
-            return pd.util.hash_pandas_object(obj).sum()
-        elif util.is_type(obj, "pandas.core.frame.DataFrame"):
-            import pandas as pd
-
-            if len(obj) >= PANDAS_ROWS_LARGE:
-                obj = obj.sample(n=PANDAS_SAMPLE_SIZE, random_state=0)
-            return pd.util.hash_pandas_object(obj).sum()
-        elif util.is_type(obj, "numpy.ndarray"):
-            h = hashlib.new(self.name)
-            self._update(h, obj.shape)
-
-            if obj.size >= NP_SIZE_LARGE:
-                import numpy as np
-
-                state = np.random.RandomState(0)
-                obj = state.choice(obj.flat, size=NP_SAMPLE_SIZE)
-
-            self._update(h, obj.tobytes())
-            return h.digest()
-        elif inspect.isbuiltin(obj):
-            return self.to_bytes(obj.__name__)
-        elif hasattr(obj, "name") and (
-            isinstance(obj, io.IOBase) or
-            (isinstance(obj.name, string_types) and os.path.exists(obj.name))
-        ):
-            # Hash files as name + last modification date + offset.
-            h = hashlib.new(self.name)
-            self._update(h, obj.name)
-            self._update(h, os.path.getmtime(obj.name))
-            self._update(h, obj.tell())
-            return h.digest()
-        elif inspect.isroutine(obj):
-            if hasattr(obj, "__wrapped__"):
-                # Ignore the wrapper of wrapped functions.
-                return self.to_bytes(obj.__wrapped__)
-
-            if obj.__module__.startswith("streamlit"):
-                # Ignore streamlit modules even if they are in the CWD
-                # (e.g. during development).
-                return self.to_bytes("%s.%s" % (obj.__module__, obj.__name__))
-
-            h = hashlib.new(self.name)
-            # TODO: This may be too restrictive for libraries in development.
-            if os.path.abspath(obj.__code__.co_filename).startswith(os.getcwd()):
-                context = _get_context(obj)
-                if obj.__defaults__:
-                    self._update(h, obj.__defaults__, context)
-                h.update(self._code_to_bytes(obj.__code__, context))
-            else:
-                # Don't hash code that is not in the current working directory.
-                self._update(h, obj.__module__)
-                self._update(h, obj.__name__)
-            return h.digest()
-        elif inspect.iscode(obj):
-            return self._code_to_bytes(obj, context)
-        elif inspect.ismodule(obj):
-            # TODO: Figure out how to best show this kind of warning to the
-            # user. In the meantime, show nothing. This scenario is too common,
-            # so the current warning is quite annoying...
-            # st.warning(('Streamlit does not support hashing modules. '
-            #             'We did not hash `%s`.') % obj.__name__)
-            # TODO: Hash more than just the name for internal modules.
-            return self.to_bytes(obj.__name__)
-        elif inspect.isclass(obj):
-            # TODO: Figure out how to best show this kind of warning to the
-            # user. In the meantime, show nothing. This scenario is too common,
-            # (e.g. in every "except" statement) so the current warning is
-            # quite annoying...
-            # st.warning(('Streamlit does not support hashing classes. '
-            #             'We did not hash `%s`.') % obj.__name__)
-            # TODO: Hash more than just the name of classes.
-            return self.to_bytes(obj.__name__)
-        elif isinstance(obj, functools.partial):
-            # The return value of functools.partial is not a plain function:
-            # it's a callable object that remembers the original function plus
-            # the values you pickled into it. So here we need to special-case it.
-            h = hashlib.new(self.name)
-            self._update(h, obj.args)
-            self._update(h, obj.func)
-            self._update(h, obj.keywords)
-            return h.digest()
-        else:
-            try:
-                # As a last resort, we pickle the object to hash it.
-                return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
-            except Exception:
+        try:
+            if _is_magicmock(obj):
+                # MagicMock can result in objects that appear to be infinitely
+                # deep, so we don't try to hash them at all.
+                return self.to_bytes(id(obj))
+            elif isinstance(obj, bytes) or isinstance(obj, bytearray):
+                return obj
+            elif isinstance(obj, string_types):
+                return obj.encode()
+            elif isinstance(obj, float):
+                return self.to_bytes(hash(obj))
+            elif isinstance(obj, int):
+                return _int_to_bytes(obj)
+            elif isinstance(obj, list) or isinstance(obj, tuple):
+                h = hashlib.new(self.name)
+                # add type to distingush x from [x]
+                self._update(h, type(obj).__name__.encode() + b":")
+                for e in obj:
+                    self._update(h, e, context)
+                return h.digest()
+            elif obj is None:
+                # Special string since hashes change between sessions.
+                # We don't use Python's `hash` since hashes are not consistent
+                # across runs.
+                return b"none:"
+            elif obj is True:
+                return b"bool:1"
+            elif obj is False:
+                return b"bool:0"
+            elif util.is_type(obj, "pandas.core.frame.DataFrame") or util.is_type(
+                obj, "pandas.core.series.Series"
+            ):
+                import pandas as pd
+
+                if len(obj) >= PANDAS_ROWS_LARGE:
+                    obj = obj.sample(n=PANDAS_SAMPLE_SIZE, random_state=0)
+                try:
+                    return pd.util.hash_pandas_object(obj).sum()
+                except TypeError:
+                    # Use pickle if pandas cannot hash the object for example if
+                    # it contains unhashable objects.
+                    return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
+            elif util.is_type(obj, "numpy.ndarray"):
+                h = hashlib.new(self.name)
+                self._update(h, obj.shape)
+
+                if obj.size >= NP_SIZE_LARGE:
+                    import numpy as np
+
+                    state = np.random.RandomState(0)
+                    obj = state.choice(obj.flat, size=NP_SAMPLE_SIZE)
+
+                self._update(h, obj.tobytes())
+                return h.digest()
+            elif inspect.isbuiltin(obj):
+                return self.to_bytes(obj.__name__)
+            elif hasattr(obj, "name") and (
+                isinstance(obj, io.IOBase)
+                or (isinstance(obj.name, string_types) and os.path.exists(obj.name))
+            ):
+                # Hash files as name + last modification date + offset.
+                h = hashlib.new(self.name)
+                self._update(h, obj.name)
+                self._update(h, os.path.getmtime(obj.name))
+                self._update(h, obj.tell())
+                return h.digest()
+            elif inspect.isroutine(obj):
+                if hasattr(obj, "__wrapped__"):
+                    # Ignore the wrapper of wrapped functions.
+                    return self.to_bytes(obj.__wrapped__)
+
+                if obj.__module__.startswith("streamlit"):
+                    # Ignore streamlit modules even if they are in the CWD
+                    # (e.g. during development).
+                    return self.to_bytes("%s.%s" % (obj.__module__, obj.__name__))
+
+                h = hashlib.new(self.name)
+                # TODO: This may be too restrictive for libraries in development.
+                if os.path.abspath(obj.__code__.co_filename).startswith(os.getcwd()):
+                    context = _get_context(obj)
+                    if obj.__defaults__:
+                        self._update(h, obj.__defaults__, context)
+                    h.update(self._code_to_bytes(obj.__code__, context))
+                else:
+                    # Don't hash code that is not in the current working directory.
+                    self._update(h, obj.__module__)
+                    self._update(h, obj.__name__)
+                return h.digest()
+            elif inspect.iscode(obj):
+                return self._code_to_bytes(obj, context)
+            elif inspect.ismodule(obj):
+                # TODO: Figure out how to best show this kind of warning to the
+                # user. In the meantime, show nothing. This scenario is too common,
+                # so the current warning is quite annoying...
+                # st.warning(('Streamlit does not support hashing modules. '
+                #             'We did not hash `%s`.') % obj.__name__)
+                # TODO: Hash more than just the name for internal modules.
+                return self.to_bytes(obj.__name__)
+            elif inspect.isclass(obj):
                 # TODO: Figure out how to best show this kind of warning to the
-                # user.
-                st.warning("Streamlit cannot hash an object of type %s." % type(obj))
+                # user. In the meantime, show nothing. This scenario is too common,
+                # (e.g. in every "except" statement) so the current warning is
+                # quite annoying...
+                # st.warning(('Streamlit does not support hashing classes. '
+                #             'We did not hash `%s`.') % obj.__name__)
+                # TODO: Hash more than just the name of classes.
+                return self.to_bytes(obj.__name__)
+            elif isinstance(obj, functools.partial):
+                # The return value of functools.partial is not a plain function:
+                # it's a callable object that remembers the original function plus
+                # the values you pickled into it. So here we need to special-case it.
+                h = hashlib.new(self.name)
+                self._update(h, obj.args)
+                self._update(h, obj.func)
+                self._update(h, obj.keywords)
+                return h.digest()
+            else:
+                try:
+                    # As a last resort, we pickle the object to hash it.
+                    return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
+                except:
+                    st.warning(
+                        _hashing_error_message(
+                            "Streamlit cannot hash an object of type %s." % type(obj)
+                        )
+                    )
+        except:
+            st.warning(
+                _hashing_error_message(
+                    "Streamlit failed to hash an object of type %s." % type(obj)
+                )
+            )
 
     def _code_to_bytes(self, code, context):
         h = hashlib.new(self.name)


Test patch:
diff --git a/lib/tests/streamlit/hashing_test.py b/lib/tests/streamlit/hashing_test.py
--- a/lib/tests/streamlit/hashing_test.py
+++ b/lib/tests/streamlit/hashing_test.py
@@ -27,7 +27,7 @@
 from mock import MagicMock
 
 import streamlit as st
-from streamlit.hashing import NP_SIZE_LARGE, get_hash
+from streamlit.hashing import NP_SIZE_LARGE, PANDAS_ROWS_LARGE, get_hash
 
 
 class HashTest(unittest.TestCase):
@@ -77,6 +77,11 @@ def test_pandas_dataframe(self):
         self.assertEqual(get_hash(df1), get_hash(df3))
         self.assertNotEqual(get_hash(df1), get_hash(df2))
 
+        df4 = pd.DataFrame(np.zeros((PANDAS_ROWS_LARGE, 4)), columns=list("ABCD"))
+        df5 = pd.DataFrame(np.zeros((PANDAS_ROWS_LARGE, 4)), columns=list("ABCD"))
+
+        self.assertEqual(get_hash(df4), get_hash(df5))
+
     def test_pandas_series(self):
         series1 = pd.Series([1, 2])
         series2 = pd.Series([1, 3])
@@ -85,6 +90,11 @@ def test_pandas_series(self):
         self.assertEqual(get_hash(series1), get_hash(series3))
         self.assertNotEqual(get_hash(series1), get_hash(series2))
 
+        series4 = pd.Series(range(PANDAS_ROWS_LARGE))
+        series5 = pd.Series(range(PANDAS_ROWS_LARGE))
+
+        self.assertEqual(get_hash(series4), get_hash(series5))
+
     def test_numpy(self):
         np1 = np.zeros(10)
         np2 = np.zeros(11)