Repository: ourownstory/neural_prophet

Issue:
regularize covariates



Discussion:
@ourownstory this is the issue we were talking about right? feel free to assign to me then

Resolving patch:
diff --git a/neuralprophet/forecaster.py b/neuralprophet/forecaster.py
--- a/neuralprophet/forecaster.py
+++ b/neuralprophet/forecaster.py
@@ -2293,7 +2293,12 @@ def _add_batch_regularizations(self, loss, e, iter_progress):
                 reg_events_loss = utils.reg_func_events(self.config_events, self.config_country_holidays, self.model)
                 reg_loss += reg_events_loss
 
-            # Regularize regressors: sparsify regressor features coefficients
+            # Regularize lagged regressors: sparsify covariate features coefficients
+            if self.config_covar is not None:
+                reg_covariate_loss = utils.reg_func_covariates(self.config_covar, self.model)
+                reg_loss += reg_covariate_loss
+
+            # Regularize future regressors: sparsify regressor features coefficients
             if self.config_regressors is not None:
                 reg_regressor_loss = utils.reg_func_regressors(self.config_regressors, self.model)
                 reg_loss += reg_regressor_loss
diff --git a/neuralprophet/utils.py b/neuralprophet/utils.py
--- a/neuralprophet/utils.py
+++ b/neuralprophet/utils.py
@@ -6,6 +6,7 @@
 import torch
 from collections import OrderedDict
 from neuralprophet import hdays as hdays_part2
+from neuralprophet import utils_torch
 import holidays as pyholidays
 import warnings
 import logging
@@ -133,6 +134,33 @@ def reg_func_events(config_events, config_country_holidays, model):
     return reg_events_loss
 
 
+def reg_func_covariates(config_covariates, model):
+    """
+    Regularization of lagged covariates to induce sparsity
+
+    Parameters
+    ----------
+        config_covariates : configure.Covar
+            Configurations for user specified lagged covariates
+        model : TimeNet
+            TimeNet model object
+
+    Returns
+    -------
+        scalar
+            Regularization loss
+    """
+    reg_covariate_loss = 0.0
+    for covariate, configs in config_covariates.items():
+        reg_lambda = configs.reg_lambda
+        if reg_lambda is not None:
+            weights = model.get_covar_weights(covariate)
+            loss = torch.mean(utils_torch.penalize_nonzero(weights)).squeeze()
+            reg_covariate_loss += reg_lambda * loss
+
+    return reg_covariate_loss
+
+
 def reg_func_regressors(config_regressors, model):
     """
     Regularization of regressors coefficients to induce sparsity


Test patch:
diff --git a/tests/test_regularization.py b/tests/test_regularization.py
--- a/tests/test_regularization.py
+++ b/tests/test_regularization.py
@@ -6,7 +6,7 @@
 import pandas as pd
 import pytest
 import torch
-from utils.dataset_generators import generate_event_dataset, generate_holiday_dataset
+from utils.dataset_generators import generate_event_dataset, generate_holiday_dataset, generate_lagged_regressor_dataset
 
 from neuralprophet import NeuralProphet, df_utils
 from neuralprophet.utils import reg_func_abs
@@ -116,3 +116,48 @@ def test_regularization_events():
     # print(to_preserve)
     assert np.mean(to_reduce) < 0.1
     assert np.mean(to_preserve) > 0.5
+
+
+def test_regularization_lagged_regressor():
+    """
+    Test case for  regularization feature of lagged regressors. Utlizes a
+    synthetic dataset with 4 noise-based lagged regressors (a, b, c, d).
+    The first and last lagged regressors (a, d) are expected to have a weight
+    close to 1. The middle lagged regressors (b, c) meanwhile are expected to
+    have a weight close to 0, due to the regularization. All other model
+    components are turned off to avoid side effects.
+    """
+    df, lagged_regressors = generate_lagged_regressor_dataset(periods=100)
+    df = df_utils.check_dataframe(df, check_y=False)
+
+    m = NeuralProphet(
+        epochs=30,
+        batch_size=8,
+        learning_rate=0.1,
+        yearly_seasonality=False,
+        weekly_seasonality=False,
+        daily_seasonality=False,
+        growth="off",
+        normalize="off",
+    )
+    m = m.add_lagged_regressor(
+        n_lags=3,
+        names=[lagged_regressor for lagged_regressor, _ in lagged_regressors],
+        regularization=0.1,
+    )
+    m.fit(df, freq="D")
+
+    lagged_regressors_config = dict(lagged_regressors)
+
+    for name in m.config_covar.keys():
+        weights = m.model.get_covar_weights(name).detach().numpy()
+        weight_average = np.average(weights)
+
+        lagged_regressor_weight = lagged_regressors_config[name]
+
+        if lagged_regressor_weight > 0.9:
+            assert weight_average > 0.6
+        else:
+            assert weight_average < 0.1
+
+        print(name, weight_average, lagged_regressors_config[name])
diff --git a/tests/utils/dataset_generators.py b/tests/utils/dataset_generators.py
--- a/tests/utils/dataset_generators.py
+++ b/tests/utils/dataset_generators.py
@@ -1,4 +1,5 @@
 import pandas as pd
+import numpy as np
 
 from neuralprophet.time_dataset import make_country_specific_holidays_df
 
@@ -34,3 +35,35 @@ def generate_event_dataset(
         df.loc[event, "y"] = y_events_override.get(event, y_event)
 
     return df, events
+
+
+def generate_lagged_regressor_dataset(periods=31):
+    """
+    Generate dataset for tests on lagged regressor.
+    Columns are: ds, lagged_regressors (one entry each), y
+    Each lagged regressor is random noise (range 0 to 1).
+    y is a weighted sum of the the previous 3 lagged regressors.
+    """
+    lagged_regressors = [("a", 1), ("b", 0.1), ("c", 0.1), ("d", 1)]
+
+    dates = pd.date_range("2022-01-01", periods=periods, freq="D")
+
+    df = pd.DataFrame({"ds": dates}, index=dates)
+
+    for lagged_regressor, _ in lagged_regressors:
+        df[lagged_regressor] = np.random.random(periods)
+
+    df["weighted_sum"] = sum(
+        df[lagged_regressor] * lagged_regressor_scale for lagged_regressor, lagged_regressor_scale in lagged_regressors
+    )
+    df["y"] = 0
+
+    overlap = 3
+
+    for pos, (index, data) in enumerate(df.iterrows()):
+        if pos >= overlap:
+            df.loc[index, "y"] = sum([df.iloc[pos - lag - 1]["weighted_sum"] for lag in range(overlap)])
+
+    df = df.drop(columns=["weighted_sum"])
+
+    return df, lagged_regressors