Repository: bokeh/bokeh

Issue:
Regression streaming datetime columns
The changes in https://github.com/bokeh/bokeh/pull/7379 introduced a regression streaming datetime columns expressed as a dictionary of arrays. This is because the PR introduced special handling for datetime types into the ColumnDataSource._stream method:

```python
        # slighgly awkward that we have to call converty_datetime_array here ourselves
        # but the downstream code expects things to already be ms-since-epoch
        for key in new_data:
            new_data[key] = convert_datetime_array(new_data[key])  
```

This causes issues when the data held by the ColumnDataSource has not also had the same transformation applied, triggering errors with type promotion when concatenating the two arrays:

```
  File "/Users/philippjfr/miniconda/envs/pyviz/lib/python3.6/site-packages/bokeh/core/property/containers.py", line 384, in _stream
    data = np.append(self[k], new_data[k])
  File "/Users/philippjfr/miniconda/envs/pyviz/lib/python3.6/site-packages/numpy/lib/function_base.py", line 5166, in append
    return concatenate((arr, values), axis=axis)
TypeError: invalid type promotion
```

Here is a simple reproducible example:

```python
from bokeh.io import curdoc
from bokeh.models import ColumnDataSource
from bokeh.plotting import Figure

import pandas as pd
import numpy as np

def modify_doc(doc):
    """

    :param Document doc:
    :return:
    """
    df = pd.DataFrame(
        index=pd.date_range('now', periods=30, freq='T'),
        columns=['A', 'B', 'C'],
        data=np.cumsum(np.random.standard_normal((30,3)), axis=0)
    )

    cds = ColumnDataSource({'index': df.index, 'A': df.A})
    figure = Figure(x_axis_type="datetime")
    figure.line("index", "A", source=cds)

    doc.add_root(figure)

    def callback():
        new_df = pd.DataFrame(
            index=df.index + pd.to_timedelta('30m'),
            columns=df.columns,
            data=np.random.standard_normal((30,3))
        )
        cds.stream({'index': new_df.index, 'A': new_df.A})

    doc.add_periodic_callback(callback, 1 * 1000)

modify_doc(curdoc())



Discussion:
Hrm. I'm not sure what a good solution will be. 
I will try moving the conversion into `PropertyValueDict._stream`, after the python columns have been updated. 
I'm not sure about the analysis. This diff, does not solve the problem:

```
diff --git a/bokeh/core/property/containers.py b/bokeh/core/property/containers.py
index 05c42a18a..7e329004f 100644
--- a/bokeh/core/property/containers.py
+++ b/bokeh/core/property/containers.py
@@ -392,6 +392,11 @@ class PropertyValueColumnData(PropertyValueDict):

         from ...document.events import ColumnsStreamedEvent

+        # slighgly awkward that we have to call converty_datetime_array here ourselves
+        # but the downstream code expects things to already be ms-since-epoch
+        for key in new_data:
+            new_data[key] = convert_datetime_array(new_data[key])
+
         self._notify_owners(old,
                             hint=ColumnsStreamedEvent(doc, source, new_data, rollover, setter))

diff --git a/bokeh/models/sources.py b/bokeh/models/sources.py
index ec174ad94..eaa55dcc0 100644
--- a/bokeh/models/sources.py
+++ b/bokeh/models/sources.py
@@ -439,11 +439,6 @@ class ColumnDataSource(ColumnarDataSource):
             if len(lengths) > 1:
                 raise ValueError("All streaming column updates must be the same length")

-        # slighgly awkward that we have to call converty_datetime_array here ourselves
-        # but the downstream code expects things to already be ms-since-epoch
-        for key in new_data:
-            new_data[key] = convert_datetime_array(new_data[key])
-
         self.data._stream(self.document, self, new_data, rollover, setter)

     def patch(self, patches, setter=None):
```
Even with this diff, which puts the conversion after the append, the types are still different:
```
[  1.51990290e+12   1.51990296e+12   1.51990302e+12   1.51990308e+12
   1.51990314e+12   1.51990320e+12   1.51990326e+12   1.51990332e+12
   1.51990338e+12   1.51990344e+12   1.51990350e+12   1.51990356e+12
   1.51990362e+12   1.51990368e+12   1.51990374e+12   1.51990380e+12
   1.51990386e+12   1.51990392e+12   1.51990398e+12   1.51990404e+12
   1.51990410e+12   1.51990416e+12   1.51990422e+12   1.51990428e+12
   1.51990434e+12   1.51990440e+12   1.51990446e+12   1.51990452e+12
   1.51990458e+12   1.51990464e+12] DatetimeIndex(['2018-03-01 11:44:57.683711', '2018-03-01 11:45:57.683711',
               '2018-03-01 11:46:57.683711', '2018-03-01 11:47:57.683711',
               '2018-03-01 11:48:57.683711', '2018-03-01 11:49:57.683711',
               '2018-03-01 11:50:57.683711', '2018-03-01 11:51:57.683711',
               '2018-03-01 11:52:57.683711', '2018-03-01 11:53:57.683711',
               '2018-03-01 11:54:57.683711', '2018-03-01 11:55:57.683711',
               '2018-03-01 11:56:57.683711', '2018-03-01 11:57:57.683711',
               '2018-03-01 11:58:57.683711', '2018-03-01 11:59:57.683711',
               '2018-03-01 12:00:57.683711', '2018-03-01 12:01:57.683711',
               '2018-03-01 12:02:57.683711', '2018-03-01 12:03:57.683711',
               '2018-03-01 12:04:57.683711', '2018-03-01 12:05:57.683711',
               '2018-03-01 12:06:57.683711', '2018-03-01 12:07:57.683711',
               '2018-03-01 12:08:57.683711', '2018-03-01 12:09:57.683711',
               '2018-03-01 12:10:57.683711', '2018-03-01 12:11:57.683711',
               '2018-03-01 12:12:57.683711', '2018-03-01 12:13:57.683711'],
              dtype='datetime64[ns]', freq='T')
```
I think the issue is actually that the conversion is *not* happening. In the above, the first list (the converted timestamps) is the value in `self.data`. The second is a DTIndex, with or without the move. 
Yah, that seems to be the case:
```
In [6]: convert_datetime_array(df.index)
Out[6]:
DatetimeIndex(['2018-03-01 11:21:51.046410', '2018-03-01 11:22:51.046410',
               '2018-03-01 11:23:51.046410', '2018-03-01 11:24:51.046410',
               '2018-03-01 11:25:51.046410', '2018-03-01 11:26:51.046410',
               '2018-03-01 11:27:51.046410', '2018-03-01 11:28:51.046410',
               '2018-03-01 11:29:51.046410', '2018-03-01 11:30:51.046410',
               '2018-03-01 11:31:51.046410', '2018-03-01 11:32:51.046410',
               '2018-03-01 11:33:51.046410', '2018-03-01 11:34:51.046410',
               '2018-03-01 11:35:51.046410', '2018-03-01 11:36:51.046410',
               '2018-03-01 11:37:51.046410', '2018-03-01 11:38:51.046410',
               '2018-03-01 11:39:51.046410', '2018-03-01 11:40:51.046410',
               '2018-03-01 11:41:51.046410', '2018-03-01 11:42:51.046410',
               '2018-03-01 11:43:51.046410', '2018-03-01 11:44:51.046410',
               '2018-03-01 11:45:51.046410', '2018-03-01 11:46:51.046410',
               '2018-03-01 11:47:51.046410', '2018-03-01 11:48:51.046410',
               '2018-03-01 11:49:51.046410', '2018-03-01 11:50:51.046410'],
              dtype='datetime64[ns]', freq='T')
```

Not sure why this worked before. Are we sure it worked before?
Yes, it definitely used to work, we have an entire user guide with examples like this in holoviews and I presented it at two live tutorials.
Well, tangentially, they should probably be changed as this is unecessarily verbose now. 

I'm not sure why `self.data` is a list of timestamps in the above, that is actually the problem afaict and my guess for what actually changed. 
>I'm not sure why self.data is a list of timestamps in the above

Still can't explain that, but as far as I can tell that's an array repr.
> Still can't explain that, but as far as I can tell that's an array repr.

Yes might be an array after all

@philippjfr It seems to work with this diff:
```
diff --git a/bokeh/models/sources.py b/bokeh/models/sources.py
index ec174ad94..2dda212ed 100644
--- a/bokeh/models/sources.py
+++ b/bokeh/models/sources.py
@@ -442,7 +442,10 @@ class ColumnDataSource(ColumnarDataSource):
         # slighgly awkward that we have to call converty_datetime_array here ourselves
         # but the downstream code expects things to already be ms-since-epoch
         for key in new_data:
-            new_data[key] = convert_datetime_array(new_data[key])
+            if hasattr(new_data[key], 'values'):
+                new_data[key] = convert_datetime_array(new_data[key].values)
+            else:
+                new_data[key] = convert_datetime_array(new_data[key])

         self.data._stream(self.document, self, new_data, rollover, setter)
```
So I believe the notion is validated that the problem is actually a *lack* of conversion. We should find a better way to do this than a hacky `hasattr` tho
I'll have a look at this tomorrow morning, that fix seems sensible. I suppose we could just replace the hasattr with a ``if pd and isinstance(new_data[key], Series)``, where pd is None when not installed?
The suggested change above does not seem sufficient to fix the issue at least for me. I think we might be running into to separate issues here. In my case the initial ``CDS.data`` is still of datetime64 type but the 'index' **is** converted to ints, which seems to be the exact opposite issue to the one you seem to have uncovered. I'll keep digging.
I suppose the difference is between running this on the server where the data will be converted, sent to the frontend and then end up being synced back to python and in a notebook where the data will remain the original datetime type.
The only option to make this work consistently is to always convert the data at the earliest point, i.e. during setting, updating, streaming and patching operations.
This fix works for me in all cases (although patch may have to be handled too):

```
diff --git a/bokeh/core/property/containers.py b/bokeh/core/property/containers.py
index 05c42a18a..ff751ca44 100644
--- a/bokeh/core/property/containers.py
+++ b/bokeh/core/property/containers.py
@@ -46,6 +46,8 @@ The classes in this module provide this functionality.
 '''
 from __future__ import absolute_import, print_function
 from ...util.dependencies import import_optional
+from ...util.serialization import convert_datetime_array
+
 
 pd = import_optional('pandas')
 
@@ -319,7 +321,13 @@ class PropertyValueColumnData(PropertyValueDict):
     def update(self, *args, **kwargs):
         old = self._saved_copy()
 
-        result = super(PropertyValueDict, self).update(*args, **kwargs)
+        new = dict(args[0])
+        for key, values in new.items():
+            if pd and isinstance(values, (pd.Series, pd.Index)):
+                values = values.values
+            new[key] = convert_datetime_array(values)
+
+        result = super(PropertyValueDict, self).update(new)
 
         from ...document.events import ColumnDataChangedEvent
 
diff --git a/bokeh/models/sources.py b/bokeh/models/sources.py
index ec174ad94..6de2703f0 100644
--- a/bokeh/models/sources.py
+++ b/bokeh/models/sources.py
@@ -439,10 +439,12 @@ class ColumnDataSource(ColumnarDataSource):
             if len(lengths) > 1:
                 raise ValueError("All streaming column updates must be the same length")
 
-        # slighgly awkward that we have to call converty_datetime_array here ourselves
+        # slightly awkward that we have to call converty_datetime_array here ourselves
         # but the downstream code expects things to already be ms-since-epoch
-        for key in new_data:
-            new_data[key] = convert_datetime_array(new_data[key])
+        for key, values in new_data.items():
+            if pd and isinstance(values, (pd.Series, pd.Index)):
+                values = values.values
+            new_data[key] = convert_datetime_array(values)
 
         self.data._stream(self.document, self, new_data, rollover, setter)
``` 

Let me know what you think. It's not really ideal but I don't currently have another suggestion. It's also not clear to me how this worked in the past.
> The suggested change above does not seem sufficient to fix the issue at least for me.

Can you elaborate? I just tested again, and your MRE works with the first diff above. Do you have a different example that displays problems?
Basically the same example except in the notebook:

```python
from bokeh.io import show, push_notebook, output_notebook
from bokeh.models import ColumnDataSource
from bokeh.plotting import Figure
from bokeh.resources import INLINE

import pandas as pd
import numpy as np

output_notebook(resources=INLINE)

df = pd.DataFrame(
    index=pd.date_range('now', periods=30, freq='T'),
    columns=['A', 'B', 'C'],
    data=np.cumsum(np.random.standard_normal((30,3)), axis=0)
)

cds = ColumnDataSource({'index': df.index, 'A': df.A})
figure = Figure(x_axis_type="datetime")
figure.line("index", "A", source=cds)

handle = show(figure, notebook_handle=True)


new_df = pd.DataFrame(
    index=df.index + pd.to_timedelta('30m'),
    columns=df.columns,
    data=np.random.standard_normal((30,3))
)
cds.stream({'index': new_df.index, 'A': new_df.A})
push_notebook(handle=handle)
```
OK well nothing about the OP mentioned the notebook or `push_notebook`. I'd rather search more for a different solution as changing the type in the CDS would be "surprising behavior" as well as potentially break people's existing code. FWIW it's never occurred to me to use `stream` in conjunction with `push_noteook`
I originally thought it was the same issue, but it's now clear to me that there are two distinct issues here. One on the server when passing in a Series or Index with datetime types and one in the notebook when streaming any datetime data. I think  the easiest solution is to only apply the transform in the stream method if the existing column data has also been converted.

We use streaming in notebooks extensively. 
This works for me:

```
diff --git a/bokeh/models/sources.py b/bokeh/models/sources.py
index ec174ad94..27d32cfc3 100644
--- a/bokeh/models/sources.py
+++ b/bokeh/models/sources.py
@@ -439,10 +439,16 @@ class ColumnDataSource(ColumnarDataSource):
             if len(lengths) > 1:
                 raise ValueError("All streaming column updates must be the same length")
 
-        # slighgly awkward that we have to call converty_datetime_array here ourselves
+        # slightly awkward that we have to call converty_datetime_array here ourselves
         # but the downstream code expects things to already be ms-since-epoch
-        for key in new_data:
-            new_data[key] = convert_datetime_array(new_data[key])
+        for key, values in new_data.items():
+            if pd and isinstance(values, (pd.Series, pd.Index)):
+                values = values.values
+            if (isinstance(values, np.ndarray) and values.dtype.kind.lower() == 'm' and
+                isinstance(self.data[key], np.ndarray) and self.data[key].dtype.kind.lower() != 'm'):
+                new_data[key] = convert_datetime_array(values)
+            else:
+                new_data[key] = values
```
@philippjfr If I run the notebook code with that patch and change the time delta value, e.g.
```
index=df.index + pd.to_timedelta('3000m'),
```
 I would expect to see the last point be much further out, but I don't. Am I missing something?
Can't reproduce that, that seems to behave as expected when I try it:

![bokeh_plot](https://user-images.githubusercontent.com/1550771/37168722-be030958-22fd-11e8-9dcd-3dfb75ec827f.png)

Wow, that's weird then I must have some env issue on my end. Can you start a PR? That section of code needs an explanatory comment and definitely need to try to cover as many of these cases in unit tests as we can. 

Resolving patch:
diff --git a/bokeh/models/sources.py b/bokeh/models/sources.py
--- a/bokeh/models/sources.py
+++ b/bokeh/models/sources.py
@@ -423,9 +423,8 @@ def _stream(self, new_data, rollover=None, setter=None):
             else:
                 raise ValueError("Must stream updates to all existing columns (extra: %s)" % ", ".join(sorted(extra)))
 
+        import numpy as np
         if needs_length_check:
-            import numpy as np
-
             lengths = set()
             arr_types = (np.ndarray, pd.Series) if pd else np.ndarray
             for k, x in new_data.items():
@@ -439,10 +438,19 @@ def _stream(self, new_data, rollover=None, setter=None):
             if len(lengths) > 1:
                 raise ValueError("All streaming column updates must be the same length")
 
-        # slighgly awkward that we have to call converty_datetime_array here ourselves
+        # slightly awkward that we have to call convert_datetime_array here ourselves
         # but the downstream code expects things to already be ms-since-epoch
-        for key in new_data:
-            new_data[key] = convert_datetime_array(new_data[key])
+        for key, values in new_data.items():
+            if pd and isinstance(values, (pd.Series, pd.Index)):
+                values = values.values
+            old_values = self.data[key]
+            # Apply the transformation if the new data contains datetimes
+            # but the current data has already been transformed
+            if (isinstance(values, np.ndarray) and values.dtype.kind.lower() == 'm' and
+                isinstance(old_values, np.ndarray) and old_values.dtype.kind.lower() != 'm'):
+                new_data[key] = convert_datetime_array(values)
+            else:
+                new_data[key] = values
 
         self.data._stream(self.document, self, new_data, rollover, setter)
 


Test patch:
diff --git a/bokeh/models/tests/test_sources.py b/bokeh/models/tests/test_sources.py
--- a/bokeh/models/tests/test_sources.py
+++ b/bokeh/models/tests/test_sources.py
@@ -2,6 +2,7 @@
 
 import io
 import unittest
+import datetime as dt
 from unittest import skipIf
 import warnings
 
@@ -13,7 +14,7 @@
     is_pandas = False
 
 from bokeh.models.sources import DataSource, ColumnDataSource
-from bokeh.util.serialization import transform_column_source_data
+from bokeh.util.serialization import transform_column_source_data, convert_datetime_array
 
 class TestColumnDataSource(unittest.TestCase):
 
@@ -220,10 +221,123 @@ def mock(*args, **kw):
         self.assertEqual(stuff['args'], ("doc", ds, dict(a=[11, 12], b=[21, 22]), "foo", None))
         self.assertEqual(stuff['kw'], {})
 
+    def test__stream_good_datetime64_data(self):
+        now = dt.datetime.now()
+        dates = np.array([now+dt.timedelta(i) for i in range(1, 10)], dtype='datetime64')
+        ds = ColumnDataSource(data=dict(index=dates, b=list(range(1, 10))))
+        ds._document = "doc"
+        stuff = {}
+        mock_setter = object()
+
+        def mock(*args, **kw):
+            stuff['args'] = args
+            stuff['kw'] = kw
+        ds.data._stream = mock
+        # internal implementation of stream
+        new_date = np.array([now+dt.timedelta(10)], dtype='datetime64')
+        ds._stream(dict(index=new_date, b=[10]), "foo", mock_setter)
+        self.assertTrue(np.array_equal(stuff['args'][2]['index'], new_date))
+
+    def test__stream_good_datetime64_data_transformed(self):
+        now = dt.datetime.now()
+        dates = np.array([now+dt.timedelta(i) for i in range(1, 10)], dtype='datetime64')
+        dates = convert_datetime_array(dates)
+        ds = ColumnDataSource(data=dict(index=dates, b=list(range(1, 10))))
+        ds._document = "doc"
+        stuff = {}
+        mock_setter = object()
+
+        def mock(*args, **kw):
+            stuff['args'] = args
+            stuff['kw'] = kw
+        ds.data._stream = mock
+        # internal implementation of stream
+        new_date = np.array([now+dt.timedelta(10)], dtype='datetime64')
+        ds._stream(dict(index=new_date, b=[10]), "foo", mock_setter)
+        transformed_date = convert_datetime_array(new_date)
+        self.assertTrue(np.array_equal(stuff['args'][2]['index'], transformed_date))
+
+    @skipIf(not is_pandas, "pandas not installed")
+    def test__stream_good_df_with_date_index_data(self):
+        df = pd.DataFrame(
+            index=pd.date_range('now', periods=30, freq='T'),
+            columns=['A'],
+            data=np.cumsum(np.random.standard_normal(30), axis=0)
+        )
+        ds = ColumnDataSource(data=df)
+        ds._document = "doc"
+        stuff = {}
+        mock_setter = object()
+
+        def mock(*args, **kw):
+            stuff['args'] = args
+            stuff['kw'] = kw
+        ds.data._stream = mock
+        new_df = pd.DataFrame(
+            index=df.index + pd.to_timedelta('30m'),
+            columns=df.columns,
+            data=np.random.standard_normal(30)
+        )
+        ds._stream(new_df, "foo", mock_setter)
+        self.assertTrue(np.array_equal(stuff['args'][2]['index'], new_df.index.values))
+        self.assertTrue(np.array_equal(stuff['args'][2]['A'], new_df.A.values))
+
+    @skipIf(not is_pandas, "pandas not installed")
+    def test__stream_good_dict_of_index_and_series_data(self):
+        df = pd.DataFrame(
+            index=pd.date_range('now', periods=30, freq='T'),
+            columns=['A'],
+            data=np.cumsum(np.random.standard_normal(30), axis=0)
+        )
+        ds = ColumnDataSource(data=df)
+        ds._document = "doc"
+        stuff = {}
+        mock_setter = object()
+
+        def mock(*args, **kw):
+            stuff['args'] = args
+            stuff['kw'] = kw
+        ds.data._stream = mock
+        new_df = pd.DataFrame(
+            index=df.index + pd.to_timedelta('30m'),
+            columns=df.columns,
+            data=np.random.standard_normal(30)
+        )
+        ds._stream({'index': new_df.index, 'A': new_df.A}, "foo", mock_setter)
+        self.assertTrue(np.array_equal(stuff['args'][2]['index'], new_df.index.values))
+        self.assertTrue(np.array_equal(stuff['args'][2]['A'], new_df.A.values))
+
+    @skipIf(not is_pandas, "pandas not installed")
+    def test__stream_good_dict_of_index_and_series_data_transformed(self):
+        df = pd.DataFrame(
+            index=pd.date_range('now', periods=30, freq='T'),
+            columns=['A'],
+            data=np.cumsum(np.random.standard_normal(30), axis=0)
+        )
+        ds = ColumnDataSource(data={'index': convert_datetime_array(df.index.values),
+                                    'A': df.A})
+        ds._document = "doc"
+        stuff = {}
+        mock_setter = object()
+
+        def mock(*args, **kw):
+            stuff['args'] = args
+            stuff['kw'] = kw
+        ds.data._stream = mock
+        new_df = pd.DataFrame(
+            index=df.index + pd.to_timedelta('30m'),
+            columns=df.columns,
+            data=np.random.standard_normal(30)
+        )
+        ds._stream({'index': new_df.index, 'A': new_df.A}, "foo", mock_setter)
+        self.assertTrue(np.array_equal(stuff['args'][2]['index'],
+                                       convert_datetime_array(new_df.index.values)))
+        self.assertTrue(np.array_equal(stuff['args'][2]['A'], new_df.A.values))
+
     def _assert_equal_dicts_of_arrays(self, d1, d2):
         self.assertEqual(d1.keys(), d2.keys())
         for k, v in d1.items():
-            self.assertEqual(type(v), type(d2[k]))
+            self.assertEqual(type(v), np.ndarray)
             self.assertTrue(np.array_equal(v, d2[k]))
 
     @skipIf(not is_pandas, "pandas not installed")