Repository: feast-dev/feast

Issue:
Implement lazy loading for offline and online stores
The offline and online stores are eagerly loaded in `FeatureStore`, leading unnecessary dependencies to be pulled in. For example, as #2560 reported, the AWS Lambda feature server does not work with a Snowflake offline store, even though the offline store is not strictly required for serving online features.

The offline and online stores should be lazily loaded instead. This will allow the Snowflake dependencies to removed from the AWS Lambda feature server Dockerfile. Note that #2560 was fixed by #2565, but this issue tracks a longer-term solution.


Discussion:

Resolving patch:
diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py
--- a/sdk/python/feast/infra/passthrough_provider.py
+++ b/sdk/python/feast/infra/passthrough_provider.py
@@ -39,12 +39,24 @@ def __init__(self, config: RepoConfig):
         super().__init__(config)
 
         self.repo_config = config
-        self.offline_store = get_offline_store_from_config(config.offline_store)
-        self.online_store = (
-            get_online_store_from_config(config.online_store)
-            if config.online_store
-            else None
-        )
+        self._offline_store = None
+        self._online_store = None
+
+    @property
+    def online_store(self):
+        if not self._online_store and self.repo_config.online_store:
+            self._online_store = get_online_store_from_config(
+                self.repo_config.online_store
+            )
+        return self._online_store
+
+    @property
+    def offline_store(self):
+        if not self._offline_store:
+            self._offline_store = get_offline_store_from_config(
+                self.repo_config.offline_store
+            )
+        return self._offline_store
 
     def update_infra(
         self,
diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py
--- a/sdk/python/feast/repo_config.py
+++ b/sdk/python/feast/repo_config.py
@@ -6,6 +6,7 @@
 import yaml
 from pydantic import (
     BaseModel,
+    Field,
     StrictInt,
     StrictStr,
     ValidationError,
@@ -107,10 +108,10 @@ class RepoConfig(FeastBaseModel):
     provider: StrictStr
     """ str: local or gcp or aws """
 
-    online_store: Any
+    _online_config: Any = Field(alias="online_store")
     """ OnlineStoreConfig: Online store configuration (optional depending on provider) """
 
-    offline_store: Any
+    _offline_config: Any = Field(alias="offline_store")
     """ OfflineStoreConfig: Offline store configuration (optional depending on provider) """
 
     feature_server: Optional[Any]
@@ -126,19 +127,27 @@ class RepoConfig(FeastBaseModel):
     def __init__(self, **data: Any):
         super().__init__(**data)
 
-        if isinstance(self.online_store, Dict):
-            self.online_store = get_online_config_from_type(self.online_store["type"])(
-                **self.online_store
-            )
-        elif isinstance(self.online_store, str):
-            self.online_store = get_online_config_from_type(self.online_store)()
-
-        if isinstance(self.offline_store, Dict):
-            self.offline_store = get_offline_config_from_type(
-                self.offline_store["type"]
-            )(**self.offline_store)
-        elif isinstance(self.offline_store, str):
-            self.offline_store = get_offline_config_from_type(self.offline_store)()
+        self._offline_store = None
+        if "offline_store" in data:
+            self._offline_config = data["offline_store"]
+        else:
+            if data["provider"] == "local":
+                self._offline_config = "file"
+            elif data["provider"] == "gcp":
+                self._offline_config = "bigquery"
+            elif data["provider"] == "aws":
+                self._offline_config = "redshift"
+
+        self._online_store = None
+        if "online_store" in data:
+            self._online_config = data["online_store"]
+        else:
+            if data["provider"] == "local":
+                self._online_config = "sqlite"
+            elif data["provider"] == "gcp":
+                self._online_config = "datastore"
+            elif data["provider"] == "aws":
+                self._online_config = "dynamodb"
 
         if isinstance(self.feature_server, Dict):
             self.feature_server = get_feature_server_config_from_type(
@@ -151,6 +160,35 @@ def get_registry_config(self):
         else:
             return self.registry
 
+    @property
+    def offline_store(self):
+        if not self._offline_store:
+            if isinstance(self._offline_config, Dict):
+                self._offline_store = get_offline_config_from_type(
+                    self._offline_config["type"]
+                )(**self._offline_config)
+            elif isinstance(self._offline_config, str):
+                self._offline_store = get_offline_config_from_type(
+                    self._offline_config
+                )()
+            elif self._offline_config:
+                self._offline_store = self._offline_config
+        return self._offline_store
+
+    @property
+    def online_store(self):
+        if not self._online_store:
+            if isinstance(self._online_config, Dict):
+                self._online_store = get_online_config_from_type(
+                    self._online_config["type"]
+                )(**self._online_config)
+            elif isinstance(self._online_config, str):
+                self._online_store = get_online_config_from_type(self._online_config)()
+            elif self._online_config:
+                self._online_store = self._online_config
+
+        return self._online_store
+
     @root_validator(pre=True)
     @log_exceptions
     def _validate_online_store_config(cls, values):
@@ -304,6 +342,9 @@ def write_to_path(self, repo_path: Path):
                 sort_keys=False,
             )
 
+    class Config:
+        allow_population_by_field_name = True
+
 
 class FeastConfigError(Exception):
     def __init__(self, error_message, config_path):


Test patch:
diff --git a/sdk/python/tests/integration/feature_repos/integration_test_repo_config.py b/sdk/python/tests/integration/feature_repos/integration_test_repo_config.py
--- a/sdk/python/tests/integration/feature_repos/integration_test_repo_config.py
+++ b/sdk/python/tests/integration/feature_repos/integration_test_repo_config.py
@@ -19,7 +19,7 @@ class IntegrationTestRepoConfig:
     """
 
     provider: str = "local"
-    online_store: Union[str, Dict] = "sqlite"
+    online_store: Optional[Union[str, Dict]] = "sqlite"
 
     offline_store_creator: Type[DataSourceCreator] = FileDataSourceCreator
     online_store_creator: Optional[Type[OnlineStoreCreator]] = None
@@ -38,8 +38,10 @@ def __repr__(self) -> str:
                     online_store_type = self.online_store.get("redis_type", "redis")
                 else:
                     online_store_type = self.online_store["type"]
-            else:
+            elif self.online_store:
                 online_store_type = self.online_store.__name__
+            else:
+                online_store_type = "none"
         else:
             online_store_type = self.online_store_creator.__name__
 
diff --git a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py
--- a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py
+++ b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py
@@ -39,6 +39,7 @@ def repo_config():
         project=PROJECT,
         provider=PROVIDER,
         online_store=DynamoDBOnlineStoreConfig(region=REGION),
+        # online_store={"type": "dynamodb", "region": REGION},
         offline_store=FileOfflineStoreConfig(),
     )