Repository: feast-dev/feast

Issue:
Pandas 2.0 support
**Is your feature request related to a problem? Please describe.**

I'm trying to install feast with pandas 2.0.3, but feast is pinned to pandas `<2`: https://github.com/feast-dev/feast/blob/c75a01fce2d52cd18479ace748b8eb2e6c81c988/setup.py#L55

**Describe the solution you'd like**

Allow pandas >2. I'm not sure how much effort it will be, my experience upgrading has been simple, and provided there's a good test coverage of the pandas features being used by feast, it should be fine.

**Describe alternatives you've considered**

I could rollback my codebase to an older version of pandas but I'm using some of the new feature and rely on recent bug fixes.

**Additional context**

```
python3.9 -m venv --clear tmp
source tmp/bin/activate/
pip install pandas==2.0.3 feast==0.31.1

ERROR: Cannot install feast==0.31.1 and pandas==2.0.3 because these package versions have conflicting dependencies.

The conflict is caused by:
    The user requested pandas==2.0.3
    feast 0.31.1 depends on pandas<2 and >=1.4.3


```


Discussion:
I reopened my PR here

#3664 
java_pr failed somehow 😢 
Is this considered a priority for Feast? We're attempting to use Feast but resolving the dependencies and downgrading Pandas so far to accommodate this library is problematic.
Hello, 

The issue that blocked the PR https://github.com/feast-dev/feast/pull/3664 to extend Pandas version support has been solved. Could you please reconsider this request? Thanks.
Great to know, thanks @ddl-joyce-zhao 
I'm going to handle this soon

Resolving patch:
diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py
--- a/sdk/python/feast/infra/offline_stores/file.py
+++ b/sdk/python/feast/infra/offline_stores/file.py
@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import Any, Callable, List, Literal, Optional, Tuple, Union
 
+import dask
 import dask.dataframe as dd
 import pandas as pd
 import pyarrow
@@ -42,6 +43,11 @@
     _run_dask_field_mapping,
 )
 
+# FileRetrievalJob will cast string objects to string[pyarrow] from dask version 2023.7.1
+# This is not the desired behavior for our use case, so we set the convert-string option to False
+# See (https://github.com/dask/dask/issues/10881#issuecomment-1923327936)
+dask.config.set({"dataframe.convert-string": False})
+
 
 class FileOfflineStoreConfig(FeastConfigBaseModel):
     """Offline store config for local (file-based) store"""
@@ -366,8 +372,6 @@ def evaluate_offline_job():
                 source_df[DUMMY_ENTITY_ID] = DUMMY_ENTITY_VAL
                 columns_to_extract.add(DUMMY_ENTITY_ID)
 
-            source_df = source_df.persist()
-
             return source_df[list(columns_to_extract)].persist()
 
         # When materializing a single feature view, we don't need full feature names. On demand transforms aren't materialized
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,6 @@
     "click>=7.0.0,<9.0.0",
     "colorama>=0.3.9,<1",
     "dill~=0.3.0",
-    "fastavro>=1.1.0,<2",
     "grpcio>=1.56.2,<2",
     "grpcio-tools>=1.56.2,<2",
     "grpcio-reflection>=1.56.2,<2",
@@ -54,9 +53,7 @@
     "jsonschema",
     "mmh3",
     "numpy>=1.22,<1.25",
-    "pandas>=1.4.3,<2",
-    # For some reason pandavro higher than 1.5.* only support pandas less than 1.3.
-    "pandavro~=1.5.0",
+    "pandas>=1.4.3,<3",
     # Higher than 4.23.4 seems to cause a seg fault
     "protobuf<4.23.4,>3.20",
     "proto-plus>=1.20.0,<2",
@@ -190,6 +187,7 @@
         "types-setuptools",
         "types-tabulate",
         "virtualenv<20.24.2",
+        "pandas>=1.4.3,<2; python_version < '3.9'",
     ]
     + GCP_REQUIRED
     + REDIS_REQUIRED


Test patch:
diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py
--- a/sdk/python/tests/integration/e2e/test_validation.py
+++ b/sdk/python/tests/integration/e2e/test_validation.py
@@ -167,7 +167,7 @@ def test_logged_features_validation(environment, universal_data_sources):
                         {
                             "customer_id": 2000 + i,
                             "driver_id": 6000 + i,
-                            "event_timestamp": datetime.datetime.now(),
+                            "event_timestamp": make_tzaware(datetime.datetime.now()),
                         }
                     ]
                 ),
diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
--- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
+++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
@@ -340,6 +340,11 @@ def test_historical_features_with_entities_from_query(
 
     table_from_sql_entities = job_from_sql.to_arrow().to_pandas()
     for col in table_from_sql_entities.columns:
+        # check if col dtype is timezone naive
+        if pd.api.types.is_datetime64_dtype(table_from_sql_entities[col]):
+            table_from_sql_entities[col] = table_from_sql_entities[col].dt.tz_localize(
+                "UTC"
+            )
         expected_df_query[col] = expected_df_query[col].astype(
             table_from_sql_entities[col].dtype
         )