Repository: pandas-dev/pandas

Issue:
BUG: Pandas 2.0rc0 can't handle string column with PyArrow (large_string error)
### Pandas version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the [latest version](https://pandas.pydata.org/docs/whatsnew/index.html) of pandas.

- [ ] I have confirmed this bug exists on the [main branch](https://pandas.pydata.org/docs/dev/getting_started/install.html#installing-the-development-version-of-pandas) of pandas.


### Reproducible Example

```python
import polars as pl
from datetime import datetime

dataframe = pl.DataFrame({"integer": [1, 2, 3], 
                          "date": [
                              (datetime(2022, 1, 1)), 
                              (datetime(2022, 1, 2)), 
                              (datetime(2022, 1, 3))
                          ], 
                          "float":[4.0, 5.0, 6.0],
                          "strings": ["first", "second", "third"]
                        })

df_pd_pyarrow = dataframe.to_pandas(use_pyarrow_extension_array=True)


#ERROR *** NotImplementedError: large_string ***
```


### Issue Description

Hi guys!
Polars is another tool to work with dataframes.

When converting to Pandas, one dataframe in PyArrow (generated by Polars), which has one column with string datatype, I got this error:

`NotImplementedError: large_string`

This only occurs in Pandas 2.0rc0.
With Pandas 1.5.3 it works without error.

[This not appears to be a Polars' bug, as whats was tested here](https://github.com/pola-rs/polars/issues/7431)

### Expected Behavior

It's supposed to work, as its works with v 1.5.3

### Installed Versions

<details>

```
pandas.show_versions() 
INSTALLED VERSIONS
#------------------
commit           : 1a2e300170efc08cb509a0b4ff6248f8d55ae777
python           : 3.10.6.final.0
python-bits      : 64
OS               : Linux
OS-release       : 5.10.16.3-microsoft-standard-WSL2
Version          : #1 SMP Fri Apr 2 22:23:49 UTC 2021
machine          : x86_64
processor        : x86_64
byteorder        : little
LC_ALL           : None
LANG             : C.UTF-8
LOCALE           : en_US.UTF-8 
pandas           : 2.0.0rc0
numpy            : 1.24.2
pytz             : 2022.7.1
dateutil         : 2.8.2
setuptools       : 59.6.0
pip              : 22.0.2
Cython           : None
pytest           : None
hypothesis       : None
sphinx           : None
blosc            : None
feather          : None
xlsxwriter       : None
lxml.etree       : None
html5lib         : None
pymysql          : None
psycopg2         : None
jinja2           : 3.1.2
IPython          : 8.11.0
pandas_datareader: None
bs4              : 4.11.2
bottleneck       : None
brotli           : None
fastparquet      : None
fsspec           : None
gcsfs            : None
matplotlib       : None
numba            : None
numexpr          : None
odfpy            : None
openpyxl         : None
pandas_gbq       : None
pyarrow          : 11.0.0
pyreadstat       : None
pyxlsb           : None
s3fs             : None
scipy            : None
snappy           : None
sqlalchemy       : 2.0.5.post1
tables           : None
tabulate         : None
xarray           : None
xlrd             : None
zstandard        : None
tzdata           : None
qtpy             : None
pyqt5            : None

```
</details>



Discussion:
Thanks for the report. Could you post the full traceback?
> Thanks for the report. Could you post the full traceback?

Sure...

<details>

```python
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
Cell In[4], line 1
----> 1 df_pd_pyarrow = dataframe.to_pandas(use_pyarrow_extension_array=True)

File ~/.local/lib/python3.10/site-packages/polars/internals/dataframe/frame.py:2061, in DataFrame.to_pandas(self, use_pyarrow_extension_array, *args, **kwargs)
   2059 tbl = pa.Table.from_batches(record_batches)
   2060 if use_pyarrow_extension_array:
-> 2061     return tbl.to_pandas(
   2062         self_destruct=True,
   2063         split_blocks=True,
   2064         types_mapper=lambda pa_dtype: pd.ArrowDtype(pa_dtype),
   2065         **kwargs,
   2066     )
   2068 date_as_object = kwargs.pop("date_as_object", False)
   2069 return tbl.to_pandas(date_as_object=date_as_object, **kwargs)

File ~/.local/lib/python3.10/site-packages/pyarrow/array.pxi:830, in pyarrow.lib._PandasConvertible.to_pandas()

File ~/.local/lib/python3.10/site-packages/pyarrow/table.pxi:3990, in pyarrow.lib.Table._to_pandas()

File ~/.local/lib/python3.10/site-packages/pyarrow/pandas_compat.py:820, in table_to_blockmanager(options, table, categories, ignore_metadata, types_mapper)
    818 _check_data_column_metadata_consistency(all_columns)
    819 columns = _deserialize_column_index(table, all_columns, column_indexes)
--> 820 blocks = _table_to_blocks(options, table, categories, ext_columns_dtypes)
    822 axes = [columns, index]
    823 return BlockManager(blocks, axes)

File ~/.local/lib/python3.10/site-packages/pyarrow/pandas_compat.py:1171, in _table_to_blocks(options, block_table, categories, extension_columns)
   1168 columns = block_table.column_names
   1169 result = pa.lib.table_to_blocks(options, block_table, categories,
   1170                                 list(extension_columns.keys()))
-> 1171 return [_reconstruct_block(item, columns, extension_columns)
   1172         for item in result]

File ~/.local/lib/python3.10/site-packages/pyarrow/pandas_compat.py:1171, in <listcomp>(.0)
   1168 columns = block_table.column_names
   1169 result = pa.lib.table_to_blocks(options, block_table, categories,
   1170                                 list(extension_columns.keys()))
-> 1171 return [_reconstruct_block(item, columns, extension_columns)
   1172         for item in result]

File ~/.local/lib/python3.10/site-packages/pyarrow/pandas_compat.py:781, in _reconstruct_block(item, columns, extension_columns)
    778         raise ValueError("This column does not support to be converted "
    779                          "to a pandas ExtensionArray")
    780     pd_ext_arr = pandas_dtype.__from_arrow__(arr)
--> 781     block = _int.make_block(pd_ext_arr, placement=placement)
    782 else:
    783     block = _int.make_block(block_arr, placement=placement)

File ~/.local/lib/python3.10/site-packages/pandas/core/internals/api.py:73, in make_block(values, placement, klass, ndim, dtype)
     70     placement = BlockPlacement(placement)
     72 ndim = maybe_infer_ndim(values, placement, ndim)
---> 73 if is_datetime64tz_dtype(values.dtype) or is_period_dtype(values.dtype):
     74     # GH#41168 ensure we can pass 1D dt64tz values
     75     # More generally, any EA dtype that isn't is_1d_only_ea_dtype
     76     values = extract_array(values, extract_numpy=True)
     77     values = ensure_block_shape(values, ndim)

File ~/.local/lib/python3.10/site-packages/pandas/core/dtypes/common.py:414, in is_period_dtype(arr_or_dtype)
    386 """
    387 Check whether an array-like or dtype is of the Period dtype.
    388 
   (...)
    410 True
    411 """
    412 if isinstance(arr_or_dtype, ExtensionDtype):
    413     # GH#33400 fastpath for dtype object
--> 414     return arr_or_dtype.type is Period
    416 if arr_or_dtype is None:
    417     return False

File ~/.local/lib/python3.10/site-packages/pandas/core/arrays/arrow/dtype.py:135, in ArrowDtype.type(self)
    133     return type(pa_type)
    134 else:
--> 135     raise NotImplementedError(pa_type)

NotImplementedError: large_string
```

</details>


Patch:
diff --git a/pandas/core/arrays/arrow/dtype.py b/pandas/core/arrays/arrow/dtype.py
index a262a45feb9c5..2ba0711de98f9 100644
--- a/pandas/core/arrays/arrow/dtype.py
+++ b/pandas/core/arrays/arrow/dtype.py
@@ -27,6 +27,7 @@
     StorageExtensionDtype,
     register_extension_dtype,
 )
+from pandas.core.dtypes.dtypes import CategoricalDtypeType
 
 if not pa_version_under7p0:
     import pyarrow as pa
@@ -106,7 +107,7 @@ def type(self):
             return int
         elif pa.types.is_floating(pa_type):
             return float
-        elif pa.types.is_string(pa_type):
+        elif pa.types.is_string(pa_type) or pa.types.is_large_string(pa_type):
             return str
         elif (
             pa.types.is_binary(pa_type)
@@ -132,6 +133,14 @@ def type(self):
             return time
         elif pa.types.is_decimal(pa_type):
             return Decimal
+        elif pa.types.is_dictionary(pa_type):
+            # TODO: Potentially change this & CategoricalDtype.type to
+            #  something more representative of the scalar
+            return CategoricalDtypeType
+        elif pa.types.is_list(pa_type) or pa.types.is_large_list(pa_type):
+            return list
+        elif pa.types.is_map(pa_type):
+            return dict
         elif pa.types.is_null(pa_type):
             # TODO: None? pd.NA? pa.null?
             return type(pa_type)


Test patch:
diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py
index 712fccae83cfe..22920e077123e 100644
--- a/pandas/tests/extension/test_arrow.py
+++ b/pandas/tests/extension/test_arrow.py
@@ -39,6 +39,7 @@
 from pandas.errors import PerformanceWarning
 
 from pandas.core.dtypes.common import is_any_int_dtype
+from pandas.core.dtypes.dtypes import CategoricalDtypeType
 
 import pandas as pd
 import pandas._testing as tm
@@ -1530,9 +1531,23 @@ def test_mode_dropna_false_mode_na(data):
     tm.assert_series_equal(result, expected)
 
 
-@pytest.mark.parametrize("arrow_dtype", [pa.binary(), pa.binary(16), pa.large_binary()])
-def test_arrow_dtype_type(arrow_dtype):
-    assert ArrowDtype(arrow_dtype).type == bytes
+@pytest.mark.parametrize(
+    "arrow_dtype, expected_type",
+    [
+        [pa.binary(), bytes],
+        [pa.binary(16), bytes],
+        [pa.large_binary(), bytes],
+        [pa.large_string(), str],
+        [pa.list_(pa.int64()), list],
+        [pa.large_list(pa.int64()), list],
+        [pa.map_(pa.string(), pa.int64()), dict],
+        [pa.dictionary(pa.int64(), pa.int64()), CategoricalDtypeType],
+    ],
+)
+def test_arrow_dtype_type(arrow_dtype, expected_type):
+    # GH 51845
+    # TODO: Redundant with test_getitem_scalar once arrow_dtype exists in data fixture
+    assert ArrowDtype(arrow_dtype).type == expected_type
 
 
 def test_is_bool_dtype():
@@ -1925,7 +1940,7 @@ def test_str_get(i, exp):
 
 @pytest.mark.xfail(
     reason="TODO: StringMethods._validate should support Arrow list types",
-    raises=NotImplementedError,
+    raises=AttributeError,
 )
 def test_str_join():
     ser = pd.Series(ArrowExtensionArray(pa.array([list("abc"), list("123"), None])))