Repository: bokeh/bokeh

Issue:
[BUG] Cannot serialize pd.NA
### Software versions

Python version      :  3.10.9 | packaged by conda-forge | (main, Feb  2 2023, 20:20:04) [GCC 11.3.0]
IPython version     :  8.10.0
Tornado version     :  6.2
Bokeh version       :  3.1.0.dev3+1.g0539444b.dirty
BokehJS static path :  /home/ben/Projects/bokeh/src/bokeh/server/static
node.js version     :  v16.17.1
npm version         :  8.15.0
Operating system    :  Linux-5.19.0-32-generic-x86_64-with-glibc2.35

### Browser name and version

_No response_

### Jupyter notebook / Jupyter Lab version

_No response_

### Expected behavior

Serialize `pd.NA` in same way as `None` or `NaN`

### Observed behavior

Serialization error

### Example code

```Python
import pandas as pd
import numpy as np

from bokeh.plotting import figure, show
from bokeh.models import ColumnDataSource, LabelSet


labels = ['Some', 'text', 'data', 'with', 'missing', None, None, 'values']

df = pd.DataFrame({
    'x': range(len(labels)),
    'y': range(len(labels)),
    'label': labels,
})

df['label'] = df['label'].astype(pd.StringDtype())

source = ColumnDataSource(df)

p = figure()

p.circle(source=source)

labels = LabelSet(
    text='label',
    x_offset=5,
    y_offset=5,
    source=source
)
p.add_layout(labels)

show(p)
```


### Stack traceback or browser console output

```
Traceback (most recent call last):
  File "/home/ben/Projects/bokeh/test_sparse.py", line 37, in <module>
    main()
  File "/home/ben/Projects/bokeh/test_sparse.py", line 33, in main
    show(p)
  File "/home/ben/Projects/bokeh/src/bokeh/io/showing.py", line 143, in show
    return _show_with_state(obj, state, browser, new, notebook_handle=notebook_handle)
  File "/home/ben/Projects/bokeh/src/bokeh/io/showing.py", line 194, in _show_with_state
    _show_file_with_state(obj, state, new, controller)
  File "/home/ben/Projects/bokeh/src/bokeh/io/showing.py", line 175, in _show_file_with_state
    filename = save(obj, state=state)
  File "/home/ben/Projects/bokeh/src/bokeh/io/saving.py", line 98, in save
    _save_helper(obj, filename, resources, title, template, theme)
  File "/home/ben/Projects/bokeh/src/bokeh/io/saving.py", line 164, in _save_helper
    html = file_html(obj, resources, title=title, template=template or FILE, theme=theme)
  File "/home/ben/Projects/bokeh/src/bokeh/embed/standalone.py", line 357, in file_html
    (docs_json, render_items) = standalone_docs_json_and_render_items(models_seq, suppress_callback_warning=suppress_callback_warning)
  File "/home/ben/Projects/bokeh/src/bokeh/embed/util.py", line 330, in standalone_docs_json_and_render_items
    docs_json[docid] = doc.to_json(deferred=False)
  File "/home/ben/Projects/bokeh/src/bokeh/document/document.py", line 737, in to_json
    roots = serializer.encode(self._roots)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 274, in _encode
    return self._encode_list(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 317, in _encode_list
    return [self.encode(item) for item in obj]
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 317, in <listcomp>
    return [self.encode(item) for item in obj]
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 258, in _encode
    return obj.to_serializable(self)
  File "/home/ben/Projects/bokeh/src/bokeh/model/model.py", line 525, in to_serializable
    super_rep = super().to_serializable(serializer)
  File "/home/ben/Projects/bokeh/src/bokeh/core/has_props.py", line 406, in to_serializable
    attributes = {key: serializer.encode(val) for key, val in properties.items()}
  File "/home/ben/Projects/bokeh/src/bokeh/core/has_props.py", line 406, in <dictcomp>
    attributes = {key: serializer.encode(val) for key, val in properties.items()}
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 274, in _encode
    return self._encode_list(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 317, in _encode_list
    return [self.encode(item) for item in obj]
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 317, in <listcomp>
    return [self.encode(item) for item in obj]
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 258, in _encode
    return obj.to_serializable(self)
  File "/home/ben/Projects/bokeh/src/bokeh/model/model.py", line 525, in to_serializable
    super_rep = super().to_serializable(serializer)
  File "/home/ben/Projects/bokeh/src/bokeh/core/has_props.py", line 406, in to_serializable
    attributes = {key: serializer.encode(val) for key, val in properties.items()}
  File "/home/ben/Projects/bokeh/src/bokeh/core/has_props.py", line 406, in <dictcomp>
    attributes = {key: serializer.encode(val) for key, val in properties.items()}
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 258, in _encode
    return obj.to_serializable(self)
  File "/home/ben/Projects/bokeh/src/bokeh/model/model.py", line 525, in to_serializable
    super_rep = super().to_serializable(serializer)
  File "/home/ben/Projects/bokeh/src/bokeh/core/has_props.py", line 406, in to_serializable
    attributes = {key: serializer.encode(val) for key, val in properties.items()}
  File "/home/ben/Projects/bokeh/src/bokeh/core/has_props.py", line 406, in <dictcomp>
    attributes = {key: serializer.encode(val) for key, val in properties.items()}
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 278, in _encode
    return self._encode_dict(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 334, in _encode_dict
    entries=[(self.encode(key), self.encode(val)) for key, val in obj.items()],
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 334, in <listcomp>
    entries=[(self.encode(key), self.encode(val)) for key, val in obj.items()],
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 290, in _encode
    return self._encode_other(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 458, in _encode_other
    return self._encode_ndarray(transform_series(obj))
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 418, in _encode_ndarray
    data = self._encode_list(array.flatten().tolist())
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 317, in _encode_list
    return [self.encode(item) for item in obj]
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 317, in <listcomp>
    return [self.encode(item) for item in obj]
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 249, in encode
    return self._encode(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 290, in _encode
    return self._encode_other(obj)
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 460, in _encode_other
    self.error(f"can't serialize {type(obj)}")
  File "/home/ben/Projects/bokeh/src/bokeh/core/serialization.py", line 463, in error
    raise SerializationError(message)
bokeh.core.serialization.SerializationError: can't serialize <class 'pandas._libs.missing.NAType'>
```

### Screenshots

_No response_

Discussion:

Patch:
diff --git a/src/bokeh/core/serialization.py b/src/bokeh/core/serialization.py
index 670047097..269ff0123 100644
--- a/src/bokeh/core/serialization.py
+++ b/src/bokeh/core/serialization.py
@@ -448,13 +448,14 @@ class Serializer:
         if np.issubdtype(type(obj), np.bool_):
             return self._encode_bool(bool(obj))
 
-        # avoid importing pandashere unless it is actually in use
+        # avoid importing pandas here unless it is actually in use
         module = type(obj).__module__
         if module is not None and module.startswith("pandas"):
             import pandas as pd
-            from pandas.core.arrays import PandasArray
-            if isinstance(obj, (pd.Series, pd.Index, PandasArray)):
+            if isinstance(obj, (pd.Series, pd.Index, pd.api.extensions.ExtensionArray)):
                 return self._encode_ndarray(transform_series(obj))
+            elif obj is pd.NA:
+                return None
 
         self.error(f"can't serialize {type(obj)}")
 
diff --git a/src/bokeh/util/serialization.py b/src/bokeh/util/serialization.py
index 50d45e78e..c86ff0220 100644
--- a/src/bokeh/util/serialization.py
+++ b/src/bokeh/util/serialization.py
@@ -45,7 +45,6 @@ from .strings import format_docstring
 if TYPE_CHECKING:
     import numpy.typing as npt
     import pandas as pd
-    from pandas.core.arrays import PandasArray
     from typing_extensions import TypeGuard
 
 #-----------------------------------------------------------------------------
@@ -339,7 +338,7 @@ def transform_array(array: npt.NDArray[Any]) -> npt.NDArray[Any]:
 
     return array
 
-def transform_series(series: pd.Series[Any] | pd.Index | PandasArray) -> npt.NDArray[Any]:
+def transform_series(series: pd.Series[Any] | pd.Index | pd.api.extensions.ExtensionArray) -> npt.NDArray[Any]:
     ''' Transforms a Pandas series into serialized form
 
     Args:
@@ -350,13 +349,12 @@ def transform_series(series: pd.Series[Any] | pd.Index | PandasArray) -> npt.NDA
 
     '''
     import pandas as pd
-    from pandas.core.arrays import PandasArray
 
     # not checking for pd here, this function should only be called if it
     # is already known that series is a Pandas Series type
     if isinstance(series, pd.PeriodIndex):
         vals = series.to_timestamp().values  # type: ignore
-    elif isinstance(series, PandasArray):
+    elif isinstance(series, pd.api.extensions.ExtensionArray):
         vals = series.to_numpy()
     else:
         vals = series.values


Test patch:
diff --git a/tests/unit/bokeh/core/test_serialization.py b/tests/unit/bokeh/core/test_serialization.py
index defb2d1ce..54a50b021 100644
--- a/tests/unit/bokeh/core/test_serialization.py
+++ b/tests/unit/bokeh/core/test_serialization.py
@@ -784,6 +784,10 @@ class TestSerializer:
         rep = encoder.encode(val)
         assert rep == -684115200000
 
+    def test_pd_NA(self) -> None:
+        encoder = Serializer()
+        assert encoder.encode(pd.NA) is None
+
 class TestDeserializer:
 
     def test_slice(self) -> None:
@@ -804,12 +808,6 @@ class TestDeserializer:
         rep4 = SliceRep(type="slice", start=None, stop=None, step=None)
         assert decoder.decode(rep4) == slice(None, None, None)
 
-    def test_pandas_string_array(self) -> None:
-        data = ['This is', 'some text', 'data.']
-        arr = pd.array(data, dtype='string')
-        serialized = Serializer().serialize(arr)
-        deserialized = Deserializer().deserialize(serialized)
-        assert (deserialized == np.array(data)).all()
 """
     def test_set_data_from_json_list(self) -> None:
         ds = bms.ColumnDataSource()
diff --git a/tests/unit/bokeh/util/test_util__serialization.py b/tests/unit/bokeh/util/test_util__serialization.py
index 22bc0c9ea..779633f53 100644
--- a/tests/unit/bokeh/util/test_util__serialization.py
+++ b/tests/unit/bokeh/util/test_util__serialization.py
@@ -171,6 +171,53 @@ def test_transform_series() -> None:
     out = bus.transform_series(df)
     assert isinstance(out, np.ndarray)
 
+    # boolean array
+    arr = pd.array([True, False])
+    assert isinstance(arr, pd.arrays.BooleanArray)
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
+    # string array
+    arr = pd.array(['hello', 'world'])
+    assert isinstance(arr, pd.arrays.StringArray)
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
+    # floating array
+    arr = pd.array([1.0, 42.0, np.nan])
+    assert isinstance(arr, pd.core.arrays.floating.FloatingArray)
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
+    # integer array
+    arr = pd.array([0, 1])
+    assert isinstance(arr, pd.core.arrays.integer.IntegerArray)
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
+    # sparse array
+    arr = pd.arrays.SparseArray([1.0, 2.0, np.nan, np.nan, 5.0])
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
+    # datetime array
+    arr = pd.array([pd.NaT, datetime.datetime.today(), pd.Timestamp.today()])
+    assert isinstance(arr, pd.arrays.DatetimeArray)
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
+    # timdelta array
+    arr = pd.array([datetime.timedelta(seconds=1), pd.Timedelta(0, unit='s')])
+    assert isinstance(arr, pd.arrays.TimedeltaArray)
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
+    # categorical array
+    arr = pd.array(pd.Series(['dog', 'cat', 'dog']).astype('category'))
+    assert isinstance(arr, pd.arrays.Categorical)
+    out = bus.transform_series(arr)
+    assert isinstance(out, np.ndarray)
+
 def test_array_encoding_disabled_by_dtype() -> None:
 
     assert len(bus.BINARY_ARRAY_TYPES) > 0