Repository: feast-dev/feast

Issue:
On demand feature view not supporting objects
## Expected Behavior 
```
@on_demand_feature_view(
   inputs={
       'driver_hourly_stats': driver_hourly_stats_view
   },
   features=[
     Feature(name='conv_rate_plus_val1', dtype=ValueType.DOUBLE),
     Feature(name='conv_rate_plus_val2', dtype=ValueType.DOUBLE),
     Feature(name='conv_rate_plus_val3', dtype=ValueType.STRING)
   ]
)
def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame:
    df = pd.DataFrame()
    df['conv_rate_plus_val1'] = (inputs['conv_rate'] + 1)
    df['conv_rate_plus_val2'] = (inputs['conv_rate'] + 1)
    df['conv_rate_plus_val3'] = str(inputs['conv_rate'])
    return df
```
Should process fine when applying.

## Current Behavior
```
Traceback (most recent call last):
  File "/Users/user/anaconda3/envs/feast/bin/feast", line 8, in <module>
    sys.exit(cli())
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/click/decorators.py", line 21, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/cli.py", line 390, in apply_total_command
    apply_total(repo_config, repo, skip_source_validation)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/usage.py", line 269, in wrapper
    return func(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/repo_operations.py", line 260, in apply_total
    diff = store.apply(all_to_apply, objects_to_delete=all_to_delete, partial=False)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/usage.py", line 280, in wrapper
    raise exc.with_traceback(traceback)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/usage.py", line 269, in wrapper
    return func(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/feature_store.py", line 579, in apply
    odfv.infer_features()
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/on_demand_feature_view.py", line 226, in infer_features
    name=f, dtype=python_type_to_feast_value_type(f, type_name=str(dt))
  File "/Users/user/anaconda3/envs/feast/lib/python3.10/site-packages/feast/type_map.py", line 162, in python_type_to_feast_value_type
    assert value
AssertionError
```
## Steps to reproduce
Get the on demand feature view demo from: https://github.com/feast-dev/feast-demo/blob/main/feature_repo/features.py
Add the 2 lines as noted in the snippet above for the new STRING feature: `conv_rate_plus_val3`
### Specifications

- Version: 0.17.0
- Platform: Mac
- Subsystem: 12.0.1

## Possible Solution
I might be doing something wrong or expecting something which shouldn't be supported. Would love to hear feedback.
Incase this turns out to be a bug, this might be a possible solution -- 
The type map defined at: 
https://github.com/feast-dev/feast/blob/9dc9e60aa6a5d6a85f012307f1910f5233a251c6/sdk/python/feast/type_map.py#L94
Should possibly include an entry for pandas `obj` dtype?


Discussion:
Hi @roy651. Thanks for reporting this. I understand that exception information is very confusing. **We should add some details to the assertion or better raise proper exceptions.**

That being said, this line is not correct
```
df['conv_rate_plus_val3'] = str(inputs['conv_rate'])
```
If you need to convert column to string the correct way would be:
```
df['conv_rate_plus_val3'] = inputs['conv_rate'].astype(str)
```
Otherwise, you create a column with dtype object, which is intentionally not supported by Feast.


Thanks @pyalex for the guidance to use `astype` instead of the plain casting, however, in practice it yields the same result.
Using this code:
```
@on_demand_feature_view(
   inputs={
       'driver_hourly_stats': driver_hourly_stats_view
   },
   features=[
     Feature(name='conv_rate_plus_val1', dtype=ValueType.DOUBLE),
     Feature(name='conv_rate_plus_val2', dtype=ValueType.DOUBLE),
     Feature(name='conv_rate_plus_val3', dtype=ValueType.STRING)
   ]
)
def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame:
    df = pd.DataFrame()
    df['conv_rate_plus_val1'] = (inputs['conv_rate'] + 1)
    df['conv_rate_plus_val2'] = (inputs['conv_rate'] + 1)
    df['conv_rate_plus_val3'] = inputs['conv_rate'].astype("string")
    print(f"###df.dtypes:\n{df.dtypes}")
    return df
```
Crashes similarly with:
```
###df.dtypes:
conv_rate_plus_val1    float64
conv_rate_plus_val2    float64
conv_rate_plus_val3     object
dtype: object
>>>type_name:object
Traceback (most recent call last):
  File "/Users/user/anaconda3/envs/feast_wo/bin/feast", line 8, in <module>
    sys.exit(cli())
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/click/decorators.py", line 21, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/cli.py", line 390, in apply_total_command
    apply_total(repo_config, repo, skip_source_validation)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/usage.py", line 269, in wrapper
    return func(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/repo_operations.py", line 260, in apply_total
    diff = store.apply(all_to_apply, objects_to_delete=all_to_delete, partial=False)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/usage.py", line 280, in wrapper
    raise exc.with_traceback(traceback)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/usage.py", line 269, in wrapper
    return func(*args, **kwargs)
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/feature_store.py", line 579, in apply
    odfv.infer_features()
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/on_demand_feature_view.py", line 226, in infer_features
    name=f, dtype=python_type_to_feast_value_type(f, type_name=str(dt))
  File "/Users/user/anaconda3/envs/feast_wo/lib/python3.10/site-packages/feast/type_map.py", line 163, in python_type_to_feast_value_type
    assert value
AssertionError
```
Note that I've added a couple of debug prints, the last of which comes from within `type_map.py`.

A quick search reveals that Pandas relies upon Numpy for typing and, in short, because Numpy treats strings as arrays [they become objects](https://stackoverflow.com/a/21020411/3767429). There is an alternative approach, starting from Pandas 1.0.0, but it supports the use of `string` type and not `str` as explained [in the Pandas documentation](https://pandas.pydata.org/pandas-docs/stable/user_guide/text.html).

I saw that the `type_map.py` already has a newer version (than 0.17.0) with a different flow for the type check but I believe it will fail in a similar manner as the type map still doesn't contain `string` or `obj`.

@pyalex Just to clarify the point - This is obviously just an example, but as it stands, I couldn't get any strings to work in the transformed ODFV. 
Hence the (previous) title which I gave the issue.
@roy651, you're right. This is my mistake. It always will be dtype `object` unless we use special `pandas.StringType`. But we currently are not converting either, because even `pandas.StringType` will have the name `string`, which is not in [the list](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/type_map.py#L97)

I will increase priority and return the bug label. We're gonna fix it ASAP.

Resolving patch:
diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py
--- a/sdk/python/feast/type_map.py
+++ b/sdk/python/feast/type_map.py
@@ -97,6 +97,7 @@ def python_type_to_feast_value_type(
     type_map = {
         "int": ValueType.INT64,
         "str": ValueType.STRING,
+        "string": ValueType.STRING,  # pandas.StringDtype
         "float": ValueType.DOUBLE,
         "bytes": ValueType.BYTES,
         "float64": ValueType.DOUBLE,
@@ -119,48 +120,50 @@ def python_type_to_feast_value_type(
     if type_name in type_map:
         return type_map[type_name]
 
-    if type_name == "ndarray" or isinstance(value, list):
-        if recurse:
-
-            # Convert to list type
-            list_items = pd.core.series.Series(value)
-
-            # This is the final type which we infer from the list
-            common_item_value_type = None
-            for item in list_items:
-                if isinstance(item, ProtoValue):
-                    current_item_value_type: ValueType = _proto_value_to_value_type(
-                        item
-                    )
-                else:
-                    # Get the type from the current item, only one level deep
-                    current_item_value_type = python_type_to_feast_value_type(
-                        name=name, value=item, recurse=False
-                    )
-                # Validate whether the type stays consistent
-                if (
-                    common_item_value_type
-                    and not common_item_value_type == current_item_value_type
-                ):
-                    raise ValueError(
-                        f"List value type for field {name} is inconsistent. "
-                        f"{common_item_value_type} different from "
-                        f"{current_item_value_type}."
-                    )
-                common_item_value_type = current_item_value_type
-            if common_item_value_type is None:
-                return ValueType.UNKNOWN
-            return ValueType[common_item_value_type.name + "_LIST"]
-        else:
-            assert value
+    if isinstance(value, np.ndarray) and str(value.dtype) in type_map:
+        item_type = type_map[str(value.dtype)]
+        return ValueType[item_type.name + "_LIST"]
+
+    if isinstance(value, (list, np.ndarray)):
+        # if the value's type is "ndarray" and we couldn't infer from "value.dtype"
+        # this is most probably array of "object",
+        # so we need to iterate over objects and try to infer type of each item
+        if not recurse:
             raise ValueError(
-                f"Value type for field {name} is {value.dtype.__str__()} but "
+                f"Value type for field {name} is {type(value)} but "
                 f"recursion is not allowed. Array types can only be one level "
                 f"deep."
             )
 
-    assert value
-    return type_map[value.dtype.__str__()]
+        # This is the final type which we infer from the list
+        common_item_value_type = None
+        for item in value:
+            if isinstance(item, ProtoValue):
+                current_item_value_type: ValueType = _proto_value_to_value_type(item)
+            else:
+                # Get the type from the current item, only one level deep
+                current_item_value_type = python_type_to_feast_value_type(
+                    name=name, value=item, recurse=False
+                )
+            # Validate whether the type stays consistent
+            if (
+                common_item_value_type
+                and not common_item_value_type == current_item_value_type
+            ):
+                raise ValueError(
+                    f"List value type for field {name} is inconsistent. "
+                    f"{common_item_value_type} different from "
+                    f"{current_item_value_type}."
+                )
+            common_item_value_type = current_item_value_type
+        if common_item_value_type is None:
+            return ValueType.UNKNOWN
+        return ValueType[common_item_value_type.name + "_LIST"]
+
+    raise ValueError(
+        f"Value with native type {type_name} "
+        f"cannot be converted into Feast value type"
+    )
 
 
 def python_values_to_feast_value_type(


Test patch:
diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py
--- a/sdk/python/tests/integration/registration/test_inference.py
+++ b/sdk/python/tests/integration/registration/test_inference.py
@@ -3,7 +3,7 @@
 
 from feast import Entity, Feature, RepoConfig, ValueType
 from feast.data_source import RequestDataSource
-from feast.errors import RegistryInferenceFailure
+from feast.errors import RegistryInferenceFailure, SpecifiedFeaturesNotPresentError
 from feast.feature_view import FeatureView
 from feast.inference import (
     update_data_sources_with_inferred_event_timestamp_col,
@@ -86,7 +86,7 @@ def test_update_data_sources_with_inferred_event_timestamp_col(simple_dataset_1)
             )
 
 
-def test_modify_feature_views_success():
+def test_on_demand_features_type_inference():
     # Create Feature Views
     date_request = RequestDataSource(
         name="date_request", schema={"some_date": ValueType.UNIX_TIMESTAMP}
@@ -94,11 +94,46 @@ def test_modify_feature_views_success():
 
     @on_demand_feature_view(
         inputs={"date_request": date_request},
-        features=[Feature("output", ValueType.UNIX_TIMESTAMP)],
+        features=[
+            Feature("output", ValueType.UNIX_TIMESTAMP),
+            Feature("string_output", ValueType.STRING),
+        ],
     )
     def test_view(features_df: pd.DataFrame) -> pd.DataFrame:
         data = pd.DataFrame()
         data["output"] = features_df["some_date"]
+        data["string_output"] = features_df["some_date"].astype(pd.StringDtype())
         return data
 
     test_view.infer_features()
+
+    @on_demand_feature_view(
+        inputs={"date_request": date_request},
+        features=[
+            Feature("output", ValueType.UNIX_TIMESTAMP),
+            Feature("object_output", ValueType.STRING),
+        ],
+    )
+    def invalid_test_view(features_df: pd.DataFrame) -> pd.DataFrame:
+        data = pd.DataFrame()
+        data["output"] = features_df["some_date"]
+        data["object_output"] = features_df["some_date"].astype(str)
+        return data
+
+    with pytest.raises(ValueError, match="Value with native type object"):
+        invalid_test_view.infer_features()
+
+    @on_demand_feature_view(
+        inputs={"date_request": date_request},
+        features=[
+            Feature("output", ValueType.UNIX_TIMESTAMP),
+            Feature("missing", ValueType.STRING),
+        ],
+    )
+    def test_view_with_missing_feature(features_df: pd.DataFrame) -> pd.DataFrame:
+        data = pd.DataFrame()
+        data["output"] = features_df["some_date"]
+        return data
+
+    with pytest.raises(SpecifiedFeaturesNotPresentError):
+        test_view_with_missing_feature.infer_features()