Repository: bokeh/bokeh

Issue:
Streaming numpy datetime64 data does not work
Related to #6667 - but in addition to that issue the datetime data also needs special treatment. 

My comment on #6667 contains a complete example and also the workaround, but I thought a ticket explicitly mentioning the datetime64 problem would be useful.


Discussion:
This seems like it should be a quick fix (unless I am mistaken) so I will take a stab at it for the coming release.
@0zeroth I did start looking at this. It turns our there are some complications, but I will push a PR soon so they can be discussed in a more focused way. 

Resolving patch:
diff --git a/bokeh/core/property/containers.py b/bokeh/core/property/containers.py
--- a/bokeh/core/property/containers.py
+++ b/bokeh/core/property/containers.py
@@ -374,17 +374,11 @@ def _stream(self, doc, source, new_data, rollover=None, setter=None):
 
         import numpy as np
 
-        # pandas/issues/13918
-        if pd and isinstance(new_data, pd.DataFrame):
-            new_items = new_data.iteritems()
-        else:
-            new_items = new_data.items()
-
         # TODO (bev) Currently this reports old differently for array vs list
         # For arrays is reports the actual old value. For lists, the old value
         # is actually the already updated value. This is because the method
         # self._saved_copy() makes a shallow copy.
-        for k, v in new_items:
+        for k, v in  new_data.items():
             if isinstance(self[k], np.ndarray):
                 data = np.append(self[k], new_data[k])
                 if rollover and len(data) > rollover:
diff --git a/bokeh/models/sources.py b/bokeh/models/sources.py
--- a/bokeh/models/sources.py
+++ b/bokeh/models/sources.py
@@ -6,6 +6,7 @@
 from ..core.properties import Any, Bool, ColumnData, Dict, Enum, Instance, Int, JSON, List, Seq, String
 from ..model import Model
 from ..util.dependencies import import_optional
+from ..util.serialization import convert_datetime_array
 from ..util.warnings import BokehUserWarning
 
 from .callbacks import Callback
@@ -171,7 +172,6 @@ def _data_from_df(df):
 
         '''
         _df = df.copy()
-        index = _df.index
         tmp_data = {c: v.values for c, v in _df.iteritems()}
 
         new_data = {}
@@ -180,15 +180,8 @@ def _data_from_df(df):
                 k = "_".join(k)
             new_data[k] = v
 
-        if index.name:
-            new_data[index.name] = index.values
-        elif index.names:
-            try:
-                new_data["_".join(index.names)] = index.values
-            except TypeError:
-                new_data["index"] = index.values
-        else:
-            new_data["index"] = index.values
+        index_name = ColumnDataSource._df_index_name(df)
+        new_data[index_name] = _df.index.values
         return new_data
 
     @staticmethod
@@ -208,6 +201,38 @@ def _data_from_groupby(group):
         '''
         return ColumnDataSource._data_from_df(group.describe())
 
+    @staticmethod
+    def _df_index_name(df):
+        ''' Return the Bokeh-appropriate column name for a DataFrame index
+
+        If there is no named index, then `"index" is returned.
+
+        If there is a single named index, then ``df.index.name`` is returned.
+
+        If there is a multi-index, and the index names are all strings, then
+        the names are joined with '_' and the result is returned, e.g. for a
+        multi-index ``['ind1', 'ind2']`` the result will be "ind1_ind2".
+        Otherwise if any index name is not a string, the fallback name "index"
+        is returned.
+
+        Args:
+            df (DataFrame) : the DataFrame to find an index name for
+
+        Returns:
+            str
+
+        '''
+        if df.index.name:
+            return df.index.name
+        elif df.index.names:
+            try:
+                return "_".join(df.index.names)
+            except TypeError:
+                return "index"
+        else:
+            return "index"
+
+
     @classmethod
     def from_df(cls, data):
         ''' Create a ``dict`` of columns from a Pandas DataFrame,
@@ -392,13 +417,24 @@ def _stream(self, new_data, rollover=None, setter=None):
             source.stream(new_data)
 
         '''
+        needs_length_check = True
+
         if pd and isinstance(new_data, pd.Series):
             new_data = new_data.to_frame().T
+
         if pd and isinstance(new_data, pd.DataFrame):
-            newkeys = set(new_data.columns)
+            needs_length_check = False # DataFrame lengths equal by definition
+            _df = new_data
+            newkeys = set(_df.columns)
+            index_name = ColumnDataSource._df_index_name(_df)
+            newkeys.add(index_name)
+            new_data = dict(_df.iteritems())
+            new_data[index_name] = _df.index.values
         else:
             newkeys = set(new_data.keys())
+
         oldkeys = set(self.data.keys())
+
         if newkeys != oldkeys:
             missing = oldkeys - newkeys
             extra = newkeys - oldkeys
@@ -411,7 +447,7 @@ def _stream(self, new_data, rollover=None, setter=None):
             else:
                 raise ValueError("Must stream updates to all existing columns (extra: %s)" % ", ".join(sorted(extra)))
 
-        if not (pd and isinstance(new_data, pd.DataFrame)):
+        if needs_length_check:
             import numpy as np
 
             lengths = set()
@@ -427,6 +463,11 @@ def _stream(self, new_data, rollover=None, setter=None):
             if len(lengths) > 1:
                 raise ValueError("All streaming column updates must be the same length")
 
+        # slighgly awkward that we have to call converty_datetime_array here ourselves
+        # but the downstream code expects things to already be ms-since-epoch
+        for key in new_data:
+            new_data[key] = convert_datetime_array(new_data[key])
+
         self.data._stream(self.document, self, new_data, rollover, setter)
 
     def patch(self, patches, setter=None):
diff --git a/bokeh/util/serialization.py b/bokeh/util/serialization.py
--- a/bokeh/util/serialization.py
+++ b/bokeh/util/serialization.py
@@ -124,6 +124,50 @@ def convert_datetime_type(obj):
     elif isinstance(obj, dt.time):
         return (obj.hour * 3600 + obj.minute * 60 + obj.second) * 1000 + obj.microsecond / 1000.
 
+def convert_datetime_array(array):
+    ''' Convert NumPy datetime arrays to arrays to milliseconds since epoch.
+
+    Args:
+        array : (obj)
+            A NumPy array of datetime to convert
+
+            If the value passed in is not a NumPy array, it will be returned as-is.
+
+    Returns:
+        array
+
+    '''
+    if not isinstance(array, np.ndarray):
+        return array
+
+    try:
+        dt2001 = np.datetime64('2001')
+        legacy_datetime64 = (dt2001.astype('int64') ==
+                             dt2001.astype('datetime64[ms]').astype('int64'))
+    except AttributeError as e:
+        if e.args == ("'module' object has no attribute 'datetime64'",):
+            # for compatibility with PyPy that doesn't have datetime64
+            if 'PyPy' in sys.version:
+                legacy_datetime64 = False
+                pass
+            else:
+                raise e
+        else:
+            raise e
+
+    # not quite correct, truncates to ms..
+    if array.dtype.kind == 'M':
+        if legacy_datetime64:
+            if array.dtype == np.dtype('datetime64[ns]'):
+                array = array.astype('int64') / 10**6.0
+        else:
+            array =  array.astype('datetime64[us]').astype('int64') / 1000.
+
+    elif array.dtype.kind == 'm':
+        array = array.astype('timedelta64[us]').astype('int64') / 1000.
+
+    return array
+
 def make_id():
     ''' Return a new unique ID for a Bokeh object.
 
@@ -199,32 +243,7 @@ def transform_array(array, force_list=False, buffers=None):
 
     '''
 
-    # Check for astype failures (putative Numpy < 1.7)
-    try:
-        dt2001 = np.datetime64('2001')
-        legacy_datetime64 = (dt2001.astype('int64') ==
-                             dt2001.astype('datetime64[ms]').astype('int64'))
-    except AttributeError as e:
-        if e.args == ("'module' object has no attribute 'datetime64'",):
-            # for compatibility with PyPy that doesn't have datetime64
-            if 'PyPy' in sys.version:
-                legacy_datetime64 = False
-                pass
-            else:
-                raise e
-        else:
-            raise e
-
-    # not quite correct, truncates to ms..
-    if array.dtype.kind == 'M':
-        if legacy_datetime64:
-            if array.dtype == np.dtype('datetime64[ns]'):
-                array = array.astype('int64') / 10**6.0
-        else:
-            array =  array.astype('datetime64[us]').astype('int64') / 1000.
-
-    elif array.dtype.kind == 'm':
-        array = array.astype('timedelta64[us]').astype('int64') / 1000.
+    array = convert_datetime_array(array)
 
     return serialize_array(array, force_list=force_list, buffers=buffers)
 


Test patch:
diff --git a/bokeh/models/tests/test_sources.py b/bokeh/models/tests/test_sources.py
--- a/bokeh/models/tests/test_sources.py
+++ b/bokeh/models/tests/test_sources.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import io
 import unittest
 from unittest import skipIf
 import warnings
@@ -158,6 +159,38 @@ def test_stream_bad_data(self):
             str(cm.exception).startswith("stream(...) only supports 1d sequences, got ndarray with size (")
         )
 
+    @skipIf(not is_pandas, "pandas not installed")
+    def test__df_index_name_with_named_index(self):
+        df = pd.DataFrame(dict(a=[10], b=[20], c=[30])).set_index('c')
+        assert ColumnDataSource._df_index_name(df) == "c"
+
+    @skipIf(not is_pandas, "pandas not installed")
+    def test__df_index_name_with_unnamed_index(self):
+        df = pd.DataFrame(dict(a=[10], b=[20], c=[30]))
+        assert ColumnDataSource._df_index_name(df) == "index"
+
+    skipIf(not is_pandas, "pandas not installed")
+    def test__df_index_name_with_named_multi_index(self):
+        data = io.StringIO(u'''
+Fruit,Color,Count,Price
+Apple,Red,3,$1.29
+Apple,Green,9,$0.99
+Pear,Red,25,$2.59
+Pear,Green,26,$2.79
+Lime,Green,99,$0.39
+''')
+        df = pd.read_csv(data).set_index(['Fruit', 'Color'])
+        assert df.index.names == ['Fruit', 'Color']
+        assert ColumnDataSource._df_index_name(df) == "Fruit_Color"
+
+    skipIf(not is_pandas, "pandas not installed")
+    def test__df_index_name_with_unnamed_multi_index(self):
+        arrays = [np.array(['bar', 'bar', 'baz', 'baz', 'foo', 'foo', 'qux', 'qux']),
+                  np.array(['one', 'two', 'one', 'two', 'one', 'two', 'one', 'two'])]
+        df = pd.DataFrame(np.random.randn(8, 4), index=arrays)
+        assert df.index.names == [None, None]
+        assert ColumnDataSource._df_index_name(df) == "index"
+
     def test__stream_good_data(self):
         ds = ColumnDataSource(data=dict(a=[10], b=[20]))
         ds._document = "doc"
@@ -247,7 +280,7 @@ def stream_wrapper(*args, **kwargs):
 
     @skipIf(not is_pandas, "pandas not installed")
     def test_stream_series_to_ds_created_from_df(self):
-        data = pd.DataFrame(dict(a=[10], b=[20], c=[30])).set_index('c')
+        data = pd.DataFrame(dict(a=[10], b=[20], c=[30]))
         ds = ColumnDataSource(data)
         ds._document = "doc"
 
@@ -272,13 +305,14 @@ def stream_wrapper(*args, **kwargs):
         ds._stream(pd.Series([11, 21, 31], index=list('abc')), 7)
 
         self.assertEqual(len(stream_stuff['args']), 5)
-        expected_stream_args = ("doc", ds, dict(a=np.array([11]),
+        expected_df = pd.DataFrame(dict(a=np.array([11]),
                                                 b=np.array([21]),
-                                                c=np.array([31])), 7, None)
-        for i, (arg, ex_arg) in enumerate(zip(stream_stuff['args'],
-                                              expected_stream_args)):
+                                                c=np.array([31])))
+        expected_stream_data = expected_df.to_dict('series')
+        expected_stream_data['index'] = expected_df.index.values
+        expected_args = ("doc", ds, expected_stream_data, 7, None)
+        for i, (arg, ex_arg) in enumerate(zip(stream_stuff['args'], expected_args)):
             if i == 2:
-                arg = {k: v.values for k, v in arg.to_dict('series').items()}
                 self._assert_equal_dicts_of_arrays(arg, ex_arg)
             else:
                 self.assertEqual(arg, ex_arg)
@@ -289,15 +323,17 @@ def stream_wrapper(*args, **kwargs):
         self._assert_equal_dicts_of_arrays(notify_owners_stuff['args'][0],
                                            dict(a=np.array([10]),
                                                 b=np.array([20]),
-                                                c=np.array([30])))
+                                                c=np.array([30]),
+                                                index=np.array([0])))
 
         self._assert_equal_dicts_of_arrays(dict(ds.data),
                                            dict(a=np.array([10, 11]),
                                                 b=np.array([20, 21]),
-                                                c=np.array([30, 31])))
+                                                c=np.array([30, 31]),
+                                                index=np.array([0, 0])))
 
     @skipIf(not is_pandas, "pandas not installed")
-    def test_stream_df_to_ds_created_from_df(self):
+    def test_stream_df_to_ds_created_from_df_named_index(self):
         data = pd.DataFrame(dict(a=[10], b=[20], c=[30])).set_index('c')
         ds = ColumnDataSource(data)
         ds._document = "doc"
@@ -322,7 +358,7 @@ def stream_wrapper(*args, **kwargs):
 
         ds._stream(pd.DataFrame(dict(a=[11, 12],
                                      b=[21, 22],
-                                     c=[31, 32])), 7)
+                                     c=[31, 32])).set_index('c'), 7)
 
         self.assertEqual(len(stream_stuff['args']), 5)
         expected_steam_data = dict(a=np.array([11, 12]),
@@ -331,7 +367,6 @@ def stream_wrapper(*args, **kwargs):
         expected_args = ("doc", ds, expected_steam_data, 7, None)
         for i, (arg, ex_arg) in enumerate(zip(stream_stuff['args'], expected_args)):
             if i == 2:
-                arg = {k: v.values for k, v in arg.to_dict('series').items()}
                 self.assertEqual(arg.keys(), ex_arg.keys())
                 for k, v in arg.items():
                     self.assertTrue(np.array_equal(v, ex_arg[k]))
@@ -351,6 +386,63 @@ def stream_wrapper(*args, **kwargs):
                                                 b=np.array([20, 21, 22]),
                                                 c=np.array([30, 31, 32])))
 
+    @skipIf(not is_pandas, "pandas not installed")
+    def test_stream_df_to_ds_created_from_df_default_index(self):
+        data = pd.DataFrame(dict(a=[10], b=[20], c=[30]))
+        ds = ColumnDataSource(data)
+        ds._document = "doc"
+
+        notify_owners_stuff = {}
+
+        def notify_owners_mock(*args, **kw):
+            notify_owners_stuff['args'] = args
+            notify_owners_stuff['kw'] = kw
+
+        ds.data._notify_owners = notify_owners_mock
+
+        stream_stuff = {}
+        data_stream = ds.data._stream
+
+        def stream_wrapper(*args, **kwargs):
+            stream_stuff['args'] = args
+            stream_stuff['kwargs'] = kwargs
+            data_stream(*args, **kwargs)
+
+        ds.data._stream = stream_wrapper
+
+        ds._stream(pd.DataFrame(dict(a=[11, 12],
+                                     b=[21, 22],
+                                     c=[31, 32])), 7)
+
+        self.assertEqual(len(stream_stuff['args']), 5)
+        expected_df = pd.DataFrame(dict(a=np.array([11, 12]),
+                                        b=np.array([21, 22]),
+                                        c=np.array([31, 32])))
+        expected_stream_data = expected_df.to_dict('series')
+        expected_stream_data['index'] = expected_df.index.values
+        expected_args = ("doc", ds, expected_stream_data, 7, None)
+        for i, (arg, ex_arg) in enumerate(zip(stream_stuff['args'], expected_args)):
+            if i == 2:
+                for k, v in arg.items():
+                    self.assertTrue(np.array_equal(v, ex_arg[k]))
+            else:
+                self.assertEqual(stream_stuff['args'][i], expected_args[i])
+
+        self.assertEqual(stream_stuff['kwargs'], {})
+
+        self.assertEqual(len(notify_owners_stuff['args']), 1)
+        self._assert_equal_dicts_of_arrays(notify_owners_stuff['args'][0],
+                                           dict(a=np.array([10]),
+                                                b=np.array([20]),
+                                                c=np.array([30]),
+                                                index=np.array([0])))
+
+        self._assert_equal_dicts_of_arrays(dict(ds.data),
+                                           dict(a=np.array([10, 11, 12]),
+                                                b=np.array([20, 21, 22]),
+                                                c=np.array([30, 31, 32]),
+                                                index=np.array([0, 0, 1])))
+
     def test_patch_bad_columns(self):
         ds = ColumnDataSource(data=dict(a=[10, 11], b=[20, 21]))
         with self.assertRaises(ValueError) as cm:
diff --git a/bokeh/util/tests/test_serialization.py b/bokeh/util/tests/test_serialization.py
--- a/bokeh/util/tests/test_serialization.py
+++ b/bokeh/util/tests/test_serialization.py
@@ -63,6 +63,22 @@ def test_convert_datetime_type():
     assert bus.convert_datetime_type(pd.Timedelta("3000ms")) == 3000.0
     assert bus.convert_datetime_type(bus._pd_timestamp(3000000)) == 3.0
 
+@pytest.mark.parametrize('obj', [[1,2], (1,2), dict(), set(), 10.2, "foo"])
+def test_convert_datetime_type_array_ignores_non_array(obj):
+    assert bus.convert_datetime_array(obj) is obj
+
+def test_convert_datetime_type_array_ignores_non_datetime_array():
+    a = np.arange(0,10,100)
+    assert bus.convert_datetime_array(a) is a
+
+def test_convert_datetime_type_array():
+    a = np.array(['2018-01-03T15:37:59', '2018-01-03T15:37:59.922452', '2016-05-11'], dtype='datetime64')
+    r = bus.convert_datetime_array(a)
+    assert r[0] == 1514993879000.0
+    assert r[1] == 1514993879922.452
+    assert r[2] == 1462924800000.0
+    assert r.dtype == 'float64'
+
 def test_convert_datetime_type_with_tz():
     # This ensures datetimes are sent to BokehJS timezone-naive
     # see https://github.com/bokeh/bokeh/issues/6480