Repository: bokeh/bokeh

Issue:
CDS creation from DataFrames should not use tolist()
Discovered fixing #5152: Currently Pandas DataFrames still convert all their columns to python lists using `.tolist` Not only is this slow, it means that no CDS creates this way will ever make use of the binary array protocol. One option is to use:
```
ds.to_dict('series')
```
which returns a "dict of series" however the series objects are references the actual same series objects in the dataframe. Not sure if sharing is advised. Copying to equivalent numpy array seems advised. 


Discussion:

Resolving patch:
diff --git a/bokeh/models/sources.py b/bokeh/models/sources.py
--- a/bokeh/models/sources.py
+++ b/bokeh/models/sources.py
@@ -117,19 +117,19 @@ def _data_from_df(df):
             df (DataFrame) : data to convert
 
         Returns:
-            dict(str, list)
+            dict[str, np.array]
 
         '''
-        index = df.index
-        new_data = {}
-        for colname in df:
-            new_data[colname] = df[colname].tolist()
+        _df = df.copy()
+        index = _df.index
+        new_data = _df.to_dict('series')
+
         if index.name:
-            new_data[index.name] = index.tolist()
+            new_data[index.name] = index.values
         elif index.names and not all([x is None for x in index.names]):
-            new_data["_".join(index.names)] = index.tolist()
+            new_data["_".join(index.names)] = index.values
         else:
-            new_data["index"] = index.tolist()
+            new_data["index"] = index.values
         return new_data
 
     @classmethod
@@ -141,7 +141,7 @@ def from_df(cls, data):
             data (DataFrame) : data to convert
 
         Returns:
-            dict[str, list]
+            dict[str, np.array]
 
         '''
         return cls._data_from_df(data)
diff --git a/bokeh/util/serialization.py b/bokeh/util/serialization.py
--- a/bokeh/util/serialization.py
+++ b/bokeh/util/serialization.py
@@ -150,6 +150,10 @@ def transform_array_to_list(array):
         transformed[np.isposinf(array)] = 'Infinity'
         transformed[np.isneginf(array)] = '-Infinity'
         return transformed.tolist()
+    elif (array.dtype.kind == 'O' and pd and pd.isnull(array).any()):
+        transformed = array.astype('object')
+        transformed[pd.isnull(array)] = 'NaN'
+        return transformed.tolist()
     return array.tolist()
 
 def transform_series(series, force_list=False):


Test patch:
diff --git a/bokeh/models/tests/test_sources.py b/bokeh/models/tests/test_sources.py
--- a/bokeh/models/tests/test_sources.py
+++ b/bokeh/models/tests/test_sources.py
@@ -39,7 +39,10 @@ def test_init_pandas_arg(self):
         ds = ColumnDataSource(df)
         self.assertTrue(set(df.columns).issubset(set(ds.column_names)))
         for key in data.keys():
-            self.assertEquals(list(df[key]), data[key])
+            self.assertIsInstance(ds.data[key], pd.Series)
+            self.assertEquals(list(df[key]), list(ds.data[key]))
+        self.assertIsInstance(ds.data['index'], np.ndarray)
+        self.assertEquals([0, 1], list(ds.data['index']))
         self.assertEqual(set(ds.column_names) - set(df.columns), set(["index"]))
 
     @skipIf(not is_pandas, "pandas not installed")
@@ -49,7 +52,10 @@ def test_init_pandas_data_kwarg(self):
         ds = ColumnDataSource(data=df)
         self.assertTrue(set(df.columns).issubset(set(ds.column_names)))
         for key in data.keys():
-            self.assertEquals(list(df[key]), data[key])
+            self.assertIsInstance(ds.data[key], pd.Series)
+            self.assertEquals(list(df[key]), list(ds.data[key]))
+        self.assertIsInstance(ds.data['index'], np.ndarray)
+        self.assertEquals([0, 1], list(ds.data['index']))
         self.assertEqual(set(ds.column_names) - set(df.columns), set(["index"]))
 
     def test_add_with_name(self):
diff --git a/bokeh/util/tests/test_serialization.py b/bokeh/util/tests/test_serialization.py
--- a/bokeh/util/tests/test_serialization.py
+++ b/bokeh/util/tests/test_serialization.py
@@ -2,6 +2,7 @@
 
 import base64
 
+import pytest
 import numpy as np
 import pandas as pd
 
@@ -88,6 +89,13 @@ def test_transform_array_to_list():
         out = bus.transform_array_to_list(a)
         assert isinstance(out, list)
 
+@pytest.mark.parametrize('values', [(['cat', 'dog']), ([1.2, 'apple'])])
+def test_transform_array_with_nans_to_list(values):
+    s = pd.Series([np.nan, values[0], values[1]])
+    out = bus.transform_array_to_list(s)
+    assert isinstance(out, list)
+    assert out == ['NaN', values[0], values[1]]
+
 def test_array_encoding_disabled_by_dtype():
 
     assert len(bus.BINARY_ARRAY_TYPES) > 0