Repository: feast-dev/feast

Issue:
Unable to `feast apply` when using SQL Registry
## Expected Behavior 
Using a SQL registry, we should be able to run `feast apply` more than once for a given project, so that feast can identify the diff between the existing registry and project.

## Current Behavior
When using a SQL registry, running `feast apply` for the first time works. However, when running it again, it returns the following error:
```
Traceback (most recent call last):
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/bin/feast", line 8, in <module>
    sys.exit(cli())
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/click/core.py", line 1137, in __call__
    return self.main(*args, **kwargs)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/click/core.py", line 1062, in main
    rv = self.invoke(ctx)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/click/core.py", line 1668, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/click/core.py", line 763, in invoke
    return __callback(*args, **kwargs)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/click/decorators.py", line 26, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/cli.py", line 492, in apply_total_command
    apply_total(repo_config, repo, skip_source_validation)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/usage.py", line 274, in wrapper
    return func(*args, **kwargs)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/repo_operations.py", line 305, in apply_total
    apply_total_with_repo_instance(
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/repo_operations.py", line 265, in apply_total_with_repo_instance
    registry_diff, infra_diff, new_infra = store.plan(repo)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/usage.py", line 285, in wrapper
    raise exc.with_traceback(traceback)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/usage.py", line 274, in wrapper
    return func(*args, **kwargs)
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/feature_store.py", line 708, in plan
    current_infra_proto = self._registry.proto().infra.__deepcopy__()
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/infra/registry_stores/sql.py", line 666, in proto
    r.infra.CopyFrom(self.get_infra(project).to_proto())
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/infra/registry_stores/sql.py", line 558, in get_infra
    return self._get_object(
  File "/Users/charleslariviere/.local/share/virtualenvs/data-feature-store-QM-Cj6mZ/lib/python3.8/site-packages/feast/infra/registry_stores/sql.py", line 774, in _get_object
    raise not_found_exception(name, project)
TypeError: 'NoneType' object is not callable
```

This error is raised because of what looks like a bug in `SqlRegistry.get_infra()`. Indeed, `SqlRegistry.get_infra()` calls `SqlRegistry._get_object()` but passing `None` for the `not_found_exception` argument.
[feast/infra/registry_stores/sql.py
](https://github.com/feast-dev/feast/blob/31be8e8965444ce52c16542bf3983093514d0abb/sdk/python/feast/infra/registry_stores/sql.py#L562)
```python
    ...
    def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
        return self._get_object(
            managed_infra,
            "infra_obj",
            project,
            InfraProto,
            Infra,
            "infra_name",
            "infra_proto",
            >>>> None, <<<<
        )
    ...
    def _get_object(
        self,
        table,
        name,
        project,
        proto_class,
        python_class,
        id_field_name,
        proto_field_name,
        >>> not_found_exception <<<,
    ):
        self._maybe_init_project_metadata(project)

        with self.engine.connect() as conn:
            stmt = select(table).where(
                getattr(table.c, id_field_name) == name, table.c.project_id == project
            )
            row = conn.execute(stmt).first()
            if row:
                _proto = proto_class.FromString(row[proto_field_name])
                return python_class.from_proto(_proto)
        >>> raise not_found_exception(name, project) <<<
```
Since no rows are returned for the `managed_infra` table, `_get_object` attempts to raise an error with the `not_found_exception` callable provided, but since `None` was provided, it is unable to raise the error correctly.

This seems to be hiding the actual error, which is that no rows are returned from the `managed_infra` table, which I can confirm is indeed empty in the SQL backend. I've provided my `feature_store.yaml` configuration below -- is the expectation that something should be created in the `managed_infra` table?

## Steps to reproduce
`feature_store.yaml`
```yaml
project: feature_store
provider: local
registry:
    registry_type: sql
    path: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}
offline_store:
    type: snowflake.offline
    account: ${SNOWFLAKE_ACCOUNT}
    user: ${SNOWFLAKE_USER}
    password: ${SNOWFLAKE_PASSWORD}
    role: ${SNOWFLAKE_ROLE}
    warehouse: ${SNOWFLAKE_WAREHOUSE}
    database: ${SNOWFLAKE_DATABASE}
    schema: ${SNOWFLAKE_SCHEMA}
online_store:
    type: redis
    redis_type: redis
    connection_string: "${REDIS_HOST}:${REDIS_PORT},password=${REDIS_PASSWORD}"
```

Run the following commands:
```
feast apply # works
feast apply # raises the error above
```

### Specifications

- Version: 0.22.1
- Platform: macOS
- Subsystem: 12.4

## Possible Solution




Discussion:

Resolving patch:
diff --git a/sdk/python/feast/infra/registry_stores/sql.py b/sdk/python/feast/infra/registry_stores/sql.py
--- a/sdk/python/feast/infra/registry_stores/sql.py
+++ b/sdk/python/feast/infra/registry_stores/sql.py
@@ -2,7 +2,7 @@
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
-from typing import Any, List, Optional, Set, Union
+from typing import Any, Callable, List, Optional, Set, Union
 
 from sqlalchemy import (  # type: ignore
     BigInteger,
@@ -560,7 +560,7 @@ def update_infra(self, infra: Infra, project: str, commit: bool = True):
         )
 
     def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
-        return self._get_object(
+        infra_object = self._get_object(
             managed_infra,
             "infra_obj",
             project,
@@ -570,6 +570,8 @@ def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
             "infra_proto",
             None,
         )
+        infra_object = infra_object or InfraProto()
+        return Infra.from_proto(infra_object)
 
     def apply_user_metadata(
         self,
@@ -683,11 +685,18 @@ def commit(self):
         pass
 
     def _apply_object(
-        self, table, project: str, id_field_name, obj, proto_field_name, name=None
+        self,
+        table: Table,
+        project: str,
+        id_field_name,
+        obj,
+        proto_field_name,
+        name=None,
     ):
         self._maybe_init_project_metadata(project)
 
-        name = name or obj.name
+        name = name or obj.name if hasattr(obj, "name") else None
+        assert name, f"name needs to be provided for {obj}"
         with self.engine.connect() as conn:
             update_datetime = datetime.utcnow()
             update_time = int(update_datetime.timestamp())
@@ -749,7 +758,14 @@ def _maybe_init_project_metadata(self, project):
                 conn.execute(insert_stmt)
                 usage.set_current_project_uuid(new_project_uuid)
 
-    def _delete_object(self, table, name, project, id_field_name, not_found_exception):
+    def _delete_object(
+        self,
+        table: Table,
+        name: str,
+        project: str,
+        id_field_name: str,
+        not_found_exception: Optional[Callable],
+    ):
         with self.engine.connect() as conn:
             stmt = delete(table).where(
                 getattr(table.c, id_field_name) == name, table.c.project_id == project
@@ -763,14 +779,14 @@ def _delete_object(self, table, name, project, id_field_name, not_found_exceptio
 
     def _get_object(
         self,
-        table,
-        name,
-        project,
-        proto_class,
-        python_class,
-        id_field_name,
-        proto_field_name,
-        not_found_exception,
+        table: Table,
+        name: str,
+        project: str,
+        proto_class: Any,
+        python_class: Any,
+        id_field_name: str,
+        proto_field_name: str,
+        not_found_exception: Optional[Callable],
     ):
         self._maybe_init_project_metadata(project)
 
@@ -782,10 +798,18 @@ def _get_object(
             if row:
                 _proto = proto_class.FromString(row[proto_field_name])
                 return python_class.from_proto(_proto)
-        raise not_found_exception(name, project)
+        if not_found_exception:
+            raise not_found_exception(name, project)
+        else:
+            return None
 
     def _list_objects(
-        self, table, project, proto_class, python_class, proto_field_name
+        self,
+        table: Table,
+        project: str,
+        proto_class: Any,
+        python_class: Any,
+        proto_field_name: str,
     ):
         self._maybe_init_project_metadata(project)
         with self.engine.connect() as conn:


Test patch: