Repository: feast-dev/feast

Issue:
Error during feast materialize-incremental: AttributeError: 'NoneType' object has no attribute 'dtype'
## Expected Behavior 

It should be possible to materialize TableView if some values for some rows are None

## Current Behavior

In the current version error appears when any value in historical store is None.
On command:
```
feast materialize-incremental 2021-05-21T23:46:43
```

I have error:
```
Materializing feature view app_features from 2021-04-19 00:03:27.010110+00:00 to 2021-04-21 21:55:52+00:00Traceback (most recent call last):

  File "/opt/conda/bin/feast", line 8, in <module>

    sys.exit(cli())

  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 829, in __call__

    return self.main(*args, **kwargs)

  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 782, in main

    rv = self.invoke(ctx)

  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 1259, in invoke

    return _process_result(sub_ctx.command.invoke(sub_ctx))

  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 1066, in invoke

    return ctx.invoke(self.callback, **ctx.params)

  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 610, in invoke

    return callback(*args, **kwargs)

  File "/opt/conda/lib/python3.8/site-packages/feast/cli.py", line 247, in materialize_incremental_command

    store.materialize_incremental(

  File "/opt/conda/lib/python3.8/site-packages/feast/feature_store.py", line 337, in materialize_incremental

    provider.materialize_single_feature_view(

  File "/opt/conda/lib/python3.8/site-packages/feast/infra/local.py", line 188, in materialize_single_feature_view

    rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys)

  File "/opt/conda/lib/python3.8/site-packages/feast/infra/provider.py", line 260, in _convert_arrow_to_proto

    value = python_value_to_proto_value(row[idx])

  File "/opt/conda/lib/python3.8/site-packages/feast/type_map.py", line 317, in python_value_to_proto_value

    value_type = python_type_to_feast_value_type("", value)

  File "/opt/conda/lib/python3.8/site-packages/feast/type_map.py", line 160, in python_type_to_feast_value_type

    return type_map[value.dtype.__str__()]

AttributeError: 'NoneType' object has no attribute 'dtype'
```

## Steps to reproduce

The error will appear for any input which contains None value in any row.

### Specifications

- Version: Feast SDK Version: "feast 0.10.0"
- Platform: Python
- Subsystem: Linux

## Possible Solution

We can use feature ValueType instead of parsing type from value if value is None




Discussion:

Resolving patch:
diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py
--- a/sdk/python/feast/infra/provider.py
+++ b/sdk/python/feast/infra/provider.py
@@ -257,7 +257,7 @@ def _coerce_datetime(ts):
         feature_dict = {}
         for feature in feature_view.features:
             idx = table.column_names.index(feature.name)
-            value = python_value_to_proto_value(row[idx])
+            value = python_value_to_proto_value(row[idx], feature.dtype)
             feature_dict[feature.name] = value
         event_timestamp_idx = table.column_names.index(
             feature_view.input.event_timestamp_column
diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py
--- a/sdk/python/feast/type_map.py
+++ b/sdk/python/feast/type_map.py
@@ -313,8 +313,10 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
     raise Exception(f"Unsupported data type: ${str(type(value))}")
 
 
-def python_value_to_proto_value(value: Any) -> ProtoValue:
-    value_type = python_type_to_feast_value_type("", value)
+def python_value_to_proto_value(
+    value: Any, feature_type: ValueType = None
+) -> ProtoValue:
+    value_type = python_type_to_feast_value_type("", value) if value else feature_type
     return _python_value_to_proto_value(value_type, value)
 
 


Test patch:
diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py
--- a/sdk/python/tests/test_offline_online_store_consistency.py
+++ b/sdk/python/tests/test_offline_online_store_consistency.py
@@ -4,7 +4,7 @@
 import uuid
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Iterator, Tuple, Union
+from typing import Iterator, Optional, Tuple, Union
 
 import pandas as pd
 import pytest
@@ -26,7 +26,7 @@ def create_dataset() -> pd.DataFrame:
     ts = pd.Timestamp(now).round("ms")
     data = {
         "id": [1, 2, 1, 3, 3],
-        "value": [0.1, 0.2, 0.3, 4, 5],
+        "value": [0.1, None, 0.3, 4, 5],
         "ts_1": [
             ts - timedelta(hours=4),
             ts,
@@ -147,13 +147,17 @@ def check_offline_and_online_features(
     fv: FeatureView,
     driver_id: int,
     event_timestamp: datetime,
-    expected_value: float,
+    expected_value: Optional[float],
 ) -> None:
     # Check online store
     response_dict = fs.get_online_features(
         [f"{fv.name}:value"], [{"driver": driver_id}]
     ).to_dict()
-    assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6
+
+    if expected_value:
+        assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6
+    else:
+        assert response_dict[f"{fv.name}__value"][0] is None
 
     # Check offline store
     df = fs.get_historical_features(
@@ -163,7 +167,11 @@ def check_offline_and_online_features(
         feature_refs=[f"{fv.name}:value"],
     ).to_df()
 
-    assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6
+    if expected_value:
+        assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6
+    else:
+        df = df.where(pd.notnull(df), None)
+        assert df.to_dict()[f"{fv.name}__value"][0] is None
 
 
 def run_offline_online_store_consistency_test(
@@ -181,6 +189,10 @@ def run_offline_online_store_consistency_test(
         fs=fs, fv=fv, driver_id=1, event_timestamp=end_date, expected_value=0.3
     )
 
+    check_offline_and_online_features(
+        fs=fs, fv=fv, driver_id=2, event_timestamp=end_date, expected_value=None
+    )
+
     # check prior value for materialize_incremental()
     check_offline_and_online_features(
         fs=fs, fv=fv, driver_id=3, event_timestamp=end_date, expected_value=4