Repository: duckdb/duckdb
File: tools/pythonpkg/tests/fast/pandas/test_pandas_arrow.py (Python)

Code before:
            }
        )
        pyarrow_df = df.convert_dtypes(dtype_backend='pyarrow')
        con = duckdb.connect()
        with pytest.raises(duckdb.InvalidInputException, match='Conversion failed for column objects with type object'):
            res = con.sql('select * from pyarrow_df').fetchall()

        numpy_df = pd.DataFrame(
            {'a': np.ndarray((2,), buffer=np.array([1, 2, 3]), offset=np.int_().itemsize, dtype=int)}
        ).convert_dtypes(dtype_backend='numpy_nullable')
        arrow_df = pd.DataFrame(
            {
                'a': pd.Series(
                    [
                        datetime.datetime(1990, 10, 21),
                        datetime.datetime(2023, 1, 11),
                        datetime.datetime(2001, 2, 5),
                        datetime.datetime(1990, 10, 21),
                    ]
                )
            }
        ).convert_dtypes(dtype_backend='pyarrow')
        python_df = pd.DataFrame({'a': pd.Series(['test', [5, 4, 3], {'a': 42}])}).convert_dtypes()

        df = pd.concat([numpy_df['a'], arrow_df['a'], python_df['a']], axis=1, keys=['numpy', 'arrow', 'python'])
        assert isinstance(df.dtypes['numpy'], pd.Int64Dtype)
        assert isinstance(df.dtypes['arrow'], pd.ArrowDtype)
        assert isinstance(df.dtypes['python'], np.dtype('O').__class__)

        with pytest.raises(duckdb.InvalidInputException, match='Conversion failed for column python with type object'):
            res = con.sql('select * from df').fetchall()

    def test_empty_df(self):
        df = pd.DataFrame(
            {
                'string': pd.Series(data=[], dtype='string'),
                'object': pd.Series(data=[], dtype='object'),
                'Int64': pd.Series(data=[], dtype='Int64'),
                'Float64': pd.Series(data=[], dtype='Float64'),
                'bool': pd.Series(data=[], dtype='bool'),
                'datetime64[ns]': pd.Series(data=[], dtype='datetime64[ns]'),
                'datetime64[ms]': pd.Series(data=[], dtype='datetime64[ms]'),
                'datetime64[us]': pd.Series(data=[], dtype='datetime64[us]'),
                'datetime64[s]': pd.Series(data=[], dtype='datetime64[s]'),
                'category': pd.Series(data=[], dtype='category'),
                'timedelta64[ns]': pd.Series(data=[], dtype='timedelta64[ns]'),
            }
        )
        pyarrow_df = df.convert_dtypes(dtype_backend='pyarrow')

        con = duckdb.connect()

Review comment (suggestion):
I'm not entirely sure about this, I have had issues in the past where on Linux and Mac, Numpy does not agree on long/int so ideally I would prefer to be able to do `pd.IntegerDtype` but that import doesn't work

Code after:
                'integers': np.ndarray((4,), buffer=np.array([1, 2, 3, 4, 5]), offset=np.int_().itemsize, dtype=int),
            }
        )
        pyarrow_df = df.convert_dtypes(dtype_backend='pyarrow')
        con = duckdb.connect()
        with pytest.raises(duckdb.InvalidInputException, match='Conversion failed for column objects with type object'):
            res = con.sql('select * from pyarrow_df').fetchall()

        numpy_df = pd.DataFrame(
            {'a': np.ndarray((2,), buffer=np.array([1, 2, 3]), offset=np.int_().itemsize, dtype=int)}
        ).convert_dtypes(dtype_backend='numpy_nullable')
        arrow_df = pd.DataFrame(
            {
                'a': pd.Series(
                    [
                        datetime.datetime(1990, 10, 21),
                        datetime.datetime(2023, 1, 11),
                        datetime.datetime(2001, 2, 5),
                        datetime.datetime(1990, 10, 21),
                    ]
                )
            }
        ).convert_dtypes(dtype_backend='pyarrow')
        python_df = pd.DataFrame({'a': pd.Series(['test', [5, 4, 3], {'a': 42}])}).convert_dtypes()

        df = pd.concat([numpy_df['a'], arrow_df['a'], python_df['a']], axis=1, keys=['numpy', 'arrow', 'python'])
        assert is_integer_dtype(df.dtypes['numpy'])
        assert isinstance(df.dtypes['arrow'], pd.ArrowDtype)
        assert isinstance(df.dtypes['python'], np.dtype('O').__class__)

        with pytest.raises(duckdb.InvalidInputException, match='Conversion failed for column python with type object'):
            res = con.sql('select * from df').fetchall()

    def test_empty_df(self):
        df = pd.DataFrame(
            {
                'string': pd.Series(data=[], dtype='string'),
                'object': pd.Series(data=[], dtype='object'),
                'Int64': pd.Series(data=[], dtype='Int64'),
                'Float64': pd.Series(data=[], dtype='Float64'),
                'bool': pd.Series(data=[], dtype='bool'),
                'datetime64[ns]': pd.Series(data=[], dtype='datetime64[ns]'),
                'datetime64[ms]': pd.Series(data=[], dtype='datetime64[ms]'),
                'datetime64[us]': pd.Series(data=[], dtype='datetime64[us]'),
                'datetime64[s]': pd.Series(data=[], dtype='datetime64[s]'),
                'category': pd.Series(data=[], dtype='category'),
                'timedelta64[ns]': pd.Series(data=[], dtype='timedelta64[ns]'),
            }
        )
        pyarrow_df = df.convert_dtypes(dtype_backend='pyarrow')