Repository: mmschlk/iXAI (language: python)

Issue:
Incremental PFI (regression)
Implemented Data Storage mechanisms - Batch Storage, Interval Storage, Sequence Storage, Uniform Reservoir Storage, Geometric Reservoir Storage
Implemented Imputation - Default value Imputation, Marginal Imputation (Joint and Product)
Implemented Incremental PFI for regression
Paramter n_samples in DefaultImputer's impute function is doing nothing 
The parameter n_samples in DefaultImputer's impute function is doing nothing. It needs to control for the number of calls.


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 7b3b55a..7473f29 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 workdirs/
 .idea/
 data/datasets/
+/experiments/iSAGE_paper/result_data
 
 # IDE and OS cache
 .idea
@@ -63,7 +64,7 @@ coverage.xml
 
 
 # Sphinx documentation
-docs/build/
+docs/_build/
 
 # PyBuilder
 target/
@@ -88,6 +89,7 @@ __pypackages__/
 # Environments
 .env
 .venv
+.venv_test_init_release
 env/
 venv/
 ENV/
@@ -95,7 +97,6 @@ env.bak/
 venv.bak/
 venv*
 
-
 # mkdocs documentation
 /site
 
diff --git a/agrawal_accuracy_loss.py b/agrawal_accuracy_loss.py
new file mode 100644
index 0000000..1ab69e4
--- /dev/null
+++ b/agrawal_accuracy_loss.py
@@ -0,0 +1,93 @@
+import river.metrics
+from river.utils import Rolling
+from river.ensemble import AdaptiveRandomForestClassifier
+
+from river.datasets.synth import Agrawal
+
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IncrementalSage, IntervalSage
+from increment_explain.imputer import MarginalImputer
+from increment_explain.storage import GeometricReservoirStorage
+from increment_explain.utils.wrappers.river import RiverToPredictionFunction
+
+N_SAMPLES = 10_000
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    stream = Agrawal(classification_function=1, seed=42)
+    feature_names = list([x_0 for x_0, _ in stream.take(1)][0].keys())
+
+    loss_metric = river.metrics.Accuracy()
+    training_metric = Rolling(river.metrics.Accuracy(), window_size=1000)
+
+    model = AdaptiveRandomForestClassifier(n_models=15, max_depth=10, leaf_prediction='mc')
+    model_function = RiverToPredictionFunction(model.predict_one)
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    storage = GeometricReservoirStorage(
+        size=200,
+        store_targets=False
+    )
+
+    imputer = MarginalImputer(
+        model_function=model_function,
+        storage_object=storage,
+        sampling_strategy="joint"
+    )
+
+    incremental_sage = IncrementalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=1,
+        loss_bigger_is_better=True
+    )
+
+    interval_sage = IntervalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        feature_names=feature_names,
+        interval_length=2000,
+        n_inner_samples=1
+    )
+
+    incremental_pfi = IncrementalPFI(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=1
+    )
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        # predicting
+        y_i_pred = model.predict_one(x_i)
+        training_metric.update(y_true=y_i, y_pred=y_i_pred)
+
+        # sage inc
+        _ = incremental_sage.explain_one(x_i, y_i)
+        _ = interval_sage.explain_one(x_i, y_i)
+        _ = incremental_pfi.explain_one(x_i, y_i, update_storage=False)
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if n % 1000 == 0:
+            print(f"{n}: perf                {training_metric.get()}\n"
+                  f"{n}: x_i                 {x_i}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: int-sage            {interval_sage.importance_values}\n"
+                  f"{n}: pfi-sage            {incremental_pfi.importance_values}\n"
+                  f"{n}: diff                {incremental_sage.marginal_loss - incremental_sage.model_loss}\n"
+                  f"{n}: marginal-loss       {incremental_sage.marginal_loss}\n"
+                  f"{n}: model-loss          {incremental_sage.model_loss}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n")
+
+        if n >= N_SAMPLES:
+            break
diff --git a/agrawal_cross_entropy_loss.py b/agrawal_cross_entropy_loss.py
new file mode 100644
index 0000000..f53d84a
--- /dev/null
+++ b/agrawal_cross_entropy_loss.py
@@ -0,0 +1,92 @@
+import river.metrics
+from river.utils import Rolling
+from river.ensemble import AdaptiveRandomForestClassifier
+
+from river.datasets.synth import Agrawal
+
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IncrementalSage, IntervalSage
+from increment_explain.imputer import MarginalImputer
+from increment_explain.storage import GeometricReservoirStorage
+from increment_explain.utils.wrappers.river import RiverToPredictionFunction
+
+N_SAMPLES = 10_000
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    stream = Agrawal(classification_function=1, seed=42)
+    feature_names = list([x_0 for x_0, _ in stream.take(1)][0].keys())
+
+    loss_metric = river.metrics.CrossEntropy()
+    training_metric = Rolling(river.metrics.CrossEntropy(), window_size=1000)
+
+    model = AdaptiveRandomForestClassifier(n_models=15, max_depth=10, leaf_prediction='mc')
+    model_function = RiverToPredictionFunction(model.predict_proba_one)
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    storage = GeometricReservoirStorage(
+        size=200,
+        store_targets=False
+    )
+
+    imputer = MarginalImputer(
+        model_function=model_function,
+        storage_object=storage,
+        sampling_strategy="joint"
+    )
+
+    incremental_sage = IncrementalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=1
+    )
+
+    interval_sage = IntervalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        feature_names=feature_names,
+        interval_length=2000,
+        n_inner_samples=1
+    )
+
+    incremental_pfi = IncrementalPFI(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=1
+    )
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        # predicting
+        y_i_pred = model.predict_proba_one(x_i)
+        training_metric.update(y_true=y_i, y_pred=y_i_pred)
+
+        # sage inc
+        _ = incremental_sage.explain_one(x_i, y_i)
+        _ = interval_sage.explain_one(x_i, y_i)
+        _ = incremental_pfi.explain_one(x_i, y_i, update_storage=False)
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if n % 1000 == 0:
+            print(f"{n}: perf                {training_metric.get()}\n"
+                  f"{n}: x_i                 {x_i}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: int-sage            {interval_sage.importance_values}\n"
+                  f"{n}: pfi-sage            {incremental_pfi.importance_values}\n"
+                  f"{n}: diff                {incremental_sage.marginal_loss - incremental_sage.model_loss}\n"
+                  f"{n}: marginal-loss       {incremental_sage.marginal_loss}\n"
+                  f"{n}: model-loss          {incremental_sage.model_loss}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n")
+
+        if n >= N_SAMPLES:
+            break
diff --git a/data/batch/__init__.py b/data/batch/__init__.py
new file mode 100644
index 0000000..b4f363a
--- /dev/null
+++ b/data/batch/__init__.py
@@ -0,0 +1,9 @@
+from .adult import Adult
+from .bank import BankMarketing
+from .bike import BikeSharing
+
+__all__ = [
+    "Adult",
+    "BankMarketing",
+    "BikeSharing"
+]
diff --git a/data/batch/_base.py b/data/batch/_base.py
new file mode 100644
index 0000000..4e16bc4
--- /dev/null
+++ b/data/batch/_base.py
@@ -0,0 +1,115 @@
+from abc import ABCMeta, abstractmethod
+from typing import Optional, Union
+import openml
+from sklearn.utils import shuffle
+from river.stream import iter_pandas
+from river.datasets import base
+from sklearn.preprocessing import LabelEncoder
+import numpy as np
+
+from data.stream._base import BatchStream, ConceptDriftStream
+
+
+__all__ = [
+    "get_open_ml_dataset",
+    "BaseBatchDataset",
+]
+
+
+def get_open_ml_dataset(open_ml_id, version=1):
+    dataset = openml.datasets.get_dataset(open_ml_id, version=version, download_data=True)
+    class_label = dataset.default_target_attribute
+    x_data = dataset.get_data()[0]
+    return x_data, class_label
+
+
+class BaseBatchDataset(metaclass=ABCMeta):
+    """Base class for creating data sets.
+
+    Warning: This class should not be used directly.
+    Use derived classes instead.
+    """
+
+    @abstractmethod
+    def __init__(
+            self,
+            *,
+            dataset,
+            class_label,
+            classification: bool,
+            random_seed: Optional[int] = None,
+            drop_na: bool = True,
+            shuffle_dataset: bool = True,
+            n_samples: Optional[int] = None
+    ):
+        if drop_na:
+            dataset.dropna(inplace=True)
+        if shuffle_dataset:
+            dataset = shuffle(dataset, random_state=random_seed)
+        if n_samples is not None:
+            dataset = dataset[0:n_samples]
+        self.x_data = dataset
+        self.y_data = dataset.pop(class_label)
+        self.feature_names = list(self.x_data.columns)
+        self.n_features = len(self.feature_names)
+        self.n_samples = len(self.x_data)
+        self.classification = classification
+        if self.classification:
+            self.task = base.BINARY_CLF
+            self._label_encoder = LabelEncoder()
+            self._transform_label_column()
+        else:
+            self.task = base.REG
+        self.n_outputs = 1
+
+    @property
+    def stream(self, start_pos: int = 0, end_pos: Optional[int] = None):
+        end_pos = self.n_samples + 1 if end_pos is None else end_pos
+        start_pos = start_pos * self.n_samples if type(start_pos) == float else start_pos
+        end_pos = end_pos * self.n_samples if type(end_pos) == float else end_pos
+        if (end_pos - start_pos - 1) != self.n_samples:
+            self.n_samples = end_pos - start_pos - 1
+        return BatchStream(
+            stream=iter_pandas(X=self.x_data[start_pos:end_pos], y=self.y_data[start_pos:end_pos]),
+            task=self.task, n_features=self.n_features, n_outputs=self.n_outputs
+        )
+
+    def to_concept_drift_stream(
+            self,
+            feature_remapping: dict[str, str],
+            position: Union[int, float] = 0.5,
+            width: Union[int, float] = 0.05
+    ):
+        feature_remapping.update({v: k for k, v in feature_remapping.items()})
+        n_stream_1 = int(np.floor(position * self.n_samples)) if position < 1 else int(position)
+        width = int(width * self.n_samples) if width < 1 else int(width)
+
+        x_data_1, y_data_1 = self.x_data[:n_stream_1], self.y_data[:n_stream_1]
+        x_data_2, y_data_2 = self.x_data[n_stream_1:], self.y_data[n_stream_1:]
+        x_data_2 = x_data_2.rename(columns=feature_remapping, inplace=False)
+        stream_1 = BatchStream(
+            stream=iter_pandas(X=x_data_1, y=y_data_1),
+            task=self.task, n_features=self.n_features, n_outputs=self.n_outputs
+        )
+        stream_2 = BatchStream(
+            stream=iter_pandas(X=x_data_2, y=y_data_2),
+            task=self.task, n_features=self.n_features, n_outputs=self.n_outputs
+        )
+
+        concept_drift_stream = ConceptDriftStream(stream=stream_1, drift_stream=stream_2, width=width)
+        return concept_drift_stream
+
+    def transform_features(self, feature_names, transformer):
+        for feature in feature_names:
+            self.x_data[feature] = transformer.fit_transform(self.x_data[feature].values.reshape(-1, 1))
+
+    def _transform_label_column(self):
+        self.y_data[:] = self._label_encoder.fit_transform(self.y_data)
+
+    @property
+    def label_encoding(self):
+        return self._label_encoder.get_params()
+
+
+if __name__ == "__main__":
+    pass
diff --git a/data/batch/adult.py b/data/batch/adult.py
new file mode 100644
index 0000000..ff80ffd
--- /dev/null
+++ b/data/batch/adult.py
@@ -0,0 +1,58 @@
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OrdinalEncoder, StandardScaler
+
+from data.batch._base import BaseBatchDataset, get_open_ml_dataset
+
+
+class Adult(BaseBatchDataset):
+
+    def __init__(
+            self,
+            version=2,
+            random_seed=None,
+            shuffle_dataset=False,
+            n_samples=None
+    ):
+        assert version in [1, 2], "OpenML census dataset version must be '1' or '2'."
+        dataset, class_label = get_open_ml_dataset("adult", version=version)
+        self.num_feature_names = ['age', 'capital-gain', 'capital-loss', 'hours-per-week', 'fnlwgt']
+        self.cat_feature_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race', 'sex', 'native-country', 'education-num']
+        dataset[self.num_feature_names] = dataset[self.num_feature_names].apply(pd.to_numeric)
+        num_pipeline = Pipeline([
+            ('imputer', SimpleImputer(strategy='median')),
+            ('std_scaler', StandardScaler())
+        ])
+        cat_pipeline = Pipeline([
+            ('ordinal_encoder', OrdinalEncoder()),
+        ])
+        column_transformer = ColumnTransformer([
+            ('numerical', num_pipeline, self.num_feature_names),
+            ('categorical', cat_pipeline, self.cat_feature_names),
+        ], remainder='passthrough')
+        col_names = self.num_feature_names + self.cat_feature_names
+        col_names += [feature for feature in dataset.columns if feature not in col_names]
+        dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
+        dataset.dropna(inplace=True)
+        super(Adult, self).__init__(
+            dataset=dataset,
+            class_label=class_label,
+            classification=True,
+            random_seed=random_seed,
+            shuffle_dataset=shuffle_dataset,
+            n_samples=n_samples
+        )
+
+
+if __name__ == "__main__":
+    test_dataset = Adult(random_seed=42, shuffle_dataset=False)
+    print(f"n_samples:     {test_dataset.n_samples}")
+    print(f"n_features:    {test_dataset.n_features}")
+    print(f"feature_names: {test_dataset.feature_names}")
+    stream = test_dataset.stream
+    for i, sample in enumerate(stream):
+        print(i, sample)
+        if i > 3:
+            break
\ No newline at end of file
diff --git a/data/batch/bank.py b/data/batch/bank.py
new file mode 100644
index 0000000..3f961de
--- /dev/null
+++ b/data/batch/bank.py
@@ -0,0 +1,77 @@
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OrdinalEncoder, StandardScaler
+
+from data.batch._base import BaseBatchDataset, get_open_ml_dataset
+
+
+OPEN_ML_BANK_MARKETING_RENAME_MAPPER = {
+    'V1': 'age',
+    'V2': 'job',
+    'V3': 'marital',
+    'V4': 'education',
+    'V5': 'default',
+    'V6': 'balance',
+    'V7': 'housing',
+    'V8': 'loan',
+    'V9': 'contact',
+    'V10': 'day',
+    'V11': 'month',
+    'V12': 'duration',
+    'V13': 'campaign',
+    'V14': 'pdays',
+    'V15': 'previous',
+    'V16': 'poutcome'
+}
+
+
+class BankMarketing(BaseBatchDataset):
+
+    def __init__(
+            self,
+            random_seed=None,
+            shuffle_dataset=False,
+            n_samples=None
+    ):
+        dataset, class_label = get_open_ml_dataset("bank-marketing", version=1)
+        dataset = dataset.rename(columns=OPEN_ML_BANK_MARKETING_RENAME_MAPPER)
+        self.num_feature_names = ['age', 'balance', 'duration', 'pdays', 'previous']
+        self.cat_feature_names = ['job', 'marital', 'education', 'default', 'housing', 'loan', 'contact', 'day', 'month', 'campaign', 'poutcome']
+        dataset[self.num_feature_names] = dataset[self.num_feature_names].apply(pd.to_numeric)
+        num_pipeline = Pipeline([
+            ('imputer', SimpleImputer(strategy='median')),
+            ('std_scaler', StandardScaler())
+        ])
+        cat_pipeline = Pipeline([
+            ('ordinal_encoder', OrdinalEncoder()),
+        ])
+        column_transformer = ColumnTransformer([
+            ('numerical', num_pipeline, self.num_feature_names),
+            ('categorical', cat_pipeline, self.cat_feature_names),
+        ], remainder='passthrough')
+        col_names = self.num_feature_names + self.cat_feature_names
+        col_names += [feature for feature in dataset.columns if feature not in col_names]
+        dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
+        dataset.dropna(inplace=True)
+        super(BankMarketing, self).__init__(
+            dataset=dataset,
+            class_label=class_label,
+            classification=True,
+            random_seed=random_seed,
+            shuffle_dataset=shuffle_dataset,
+            n_samples=n_samples
+        )
+
+
+if __name__ == "__main__":
+    test_dataset = BankMarketing(random_seed=42, shuffle_dataset=False)
+    print(f"n_samples:     {test_dataset.n_samples}")
+    print(f"n_features:    {test_dataset.n_features}")
+    print(f"feature_names: {test_dataset.feature_names}")
+    stream = test_dataset.stream
+    for i, sample in enumerate(stream):
+        print(i, sample)
+        if i > 3:
+            break
\ No newline at end of file
diff --git a/data/batch/bike.py b/data/batch/bike.py
new file mode 100644
index 0000000..47b118b
--- /dev/null
+++ b/data/batch/bike.py
@@ -0,0 +1,54 @@
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OrdinalEncoder, RobustScaler
+
+from data.batch._base import BaseBatchDataset, get_open_ml_dataset
+
+
+class BikeSharing(BaseBatchDataset):
+
+    def __init__(
+            self,
+            random_seed=None,
+            shuffle_dataset=False,
+            n_samples=None
+    ):
+        dataset, class_label = get_open_ml_dataset(42713, version=1)
+        self.num_feature_names = ['hour', 'temp', 'feel_temp', 'humidity', 'windspeed']
+        self.cat_feature_names = ['season', 'year', 'month', 'holiday', 'weekday', 'workingday', 'weather']
+        dataset[self.num_feature_names] = dataset[self.num_feature_names].apply(pd.to_numeric)
+        num_pipeline = Pipeline([
+            ('scaler', RobustScaler())
+        ])
+        cat_pipeline = Pipeline([
+            ('ordinal_encoder', OrdinalEncoder()),
+        ])
+        column_transformer = ColumnTransformer([
+            ('numerical', num_pipeline, self.num_feature_names),
+            ('categorical', cat_pipeline, self.cat_feature_names),
+        ], remainder='passthrough')
+        col_names = self.num_feature_names + self.cat_feature_names
+        col_names += [feature for feature in dataset.columns if feature not in col_names]
+        dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
+        dataset.dropna(inplace=True)
+        super(BikeSharing, self).__init__(
+            dataset=dataset,
+            class_label=class_label,
+            classification=False,
+            random_seed=random_seed,
+            shuffle_dataset=shuffle_dataset,
+            n_samples=n_samples
+        )
+
+
+if __name__ == "__main__":
+    test_dataset = BikeSharing(random_seed=42, shuffle_dataset=False)
+    print(f"n_samples:     {test_dataset.n_samples}")
+    print(f"n_features:    {test_dataset.n_features}")
+    print(f"feature_names: {test_dataset.feature_names}")
+    stream = test_dataset.stream
+    for i, sample in enumerate(stream):
+        print(i, sample)
+        if i > 3:
+            break
diff --git a/data/data_sets.py b/data/data_sets.py
deleted file mode 100644
index 5cfe8ba..0000000
--- a/data/data_sets.py
+++ /dev/null
@@ -1,319 +0,0 @@
-from abc import ABCMeta, abstractmethod
-
-import openml
-import pandas as pd
-from sklearn.utils import shuffle
-from sklearn.preprocessing import LabelEncoder
-import sklearn
-import river
-import numpy as np
-
-
-__all__ = [
-    "Adult",
-]
-
-# =============================================================================
-# Types and constants
-# =============================================================================
-
-# stagger data stream
-STAGGER_FEATURE_NAMES = np.array(
-    [
-        'size',
-        'color',
-        'shape'
-    ]
-)
-STAGGER_CATEGORICAL_FEATURE_NAMES = np.array(
-    [
-        'size',
-        'color',
-        'shape'
-    ]
-)
-STAGGER_LENGTH = 10000
-
-# agrawal data stream
-AGRAWAL_FEATURE_NAMES = np.array(
-    [
-        'salary',
-        'commission',
-        'age',
-        'elevel',
-        'car',
-        'zipcode',
-        'hvalue',
-        'hyears',
-        'loan'
-    ]
-)
-AGRAWAL_CATEGORICAL_FEATURE_NAMES = np.array(
-    [
-        'elevel',
-        'car',
-        'zipcode'
-    ]
-)
-AGRAWAL_LENGTH = 20000
-
-# adult data stream (census)
-OPEN_ML_ADULT_FEATURE_NAMES = np.array(
-    [
-        'age',
-        'workclass',
-        'fnlwgt',
-        'education',
-        'education-num',
-        'marital-status',
-        'occupation',
-        'relationship',
-        'race',
-        'sex',
-        'capitalgain',
-        'capitalloss',
-        'hoursperweek',
-        'native-country'
-    ]
-)
-OPEN_ML_ADULT_CATEGORICAL_FEATURE_NAMES = np.array(
-    [
-        'workclass',
-        'education',
-        'marital-status',
-        'occupation',
-        'relationship',
-        'race',
-        'sex',
-        'native-country'
-    ]
-)
-OPEN_ML_ADULT_NUM_FEATURES = ['age', 'capital-gain', 'capital-loss', 'hours-per-week']
-OPEN_ML_ADULT_NAN_FEATURES = ['workclass', 'occupation', 'native-country']
-OPEN_ML_ADULT_LENGTH = 45222
-
-# bank-marketing data stream (bank)
-OPEN_ML_BANK_MARKETING_FEATURE_NAMES = np.array(
-    [
-        'age',
-        'job',
-        'marital',
-        'education',
-        'default',
-        'balance',
-        'housing',
-        'loan',
-        'contact',
-        'day',
-        'month',
-        'duration',
-        'campaign',
-        'pdays',
-        'previous',
-        'poutcome'
-    ]
-)
-OPEN_ML_BANK_MARKETING_CATEGORICAL_FEATURE_NAMES = np.array(
-    [
-        'job',
-        'marital',
-        'education',
-        'default',
-        'housing',
-        'loan',
-        'contact',
-        'day',
-        'month',
-        'campaign',
-        'poutcome'
-    ]
-)
-OPEN_ML_BANK_MARKETING_RENAME_MAPPER = {
-    'V1': 'age',
-    'V2': 'job',
-    'V3': 'marital',
-    'V4': 'education',
-    'V5': 'default',
-    'V6': 'balance',
-    'V7': 'housing',
-    'V8': 'loan',
-    'V9': 'contact',
-    'V10': 'day',
-    'V11': 'month',
-    'V12': 'duration',
-    'V13': 'campaign',
-    'V14': 'pdays',
-    'V15': 'previous',
-    'V16': 'poutcome'
-}
-OPEN_ML_BANK_MARKETING_NUM_FEATURES = ['age', 'balance', 'duration', 'pdays', 'previous']
-OPEN_ML_BANK_MARKETING_LENGTH = 45211
-
-# bike data stream (bike rental regression)
-BIKE_FEATURE_NAMES = np.array(
-    [
-        'season',
-        'yr',
-        'mnth',
-        'hr',
-        'holiday',
-        'weekday',
-        'workingday',
-        'weathersit',
-        'temp',
-        'atemp',
-        'hum',
-        'windspeed'
-    ]
-)
-BIKE_CATEGORICAL_FEATURE_NAMES = np.array(
-    [
-        'season',
-        'yr',
-        'mnth',
-        'hr',
-        'holiday',
-        'weekday',
-        'workingday',
-        'weathersit'
-    ]
-)
-BIKE_FEATURE_NUM_FEATURES = ['temp', 'atemp', 'hum', 'windspeed']
-
-BIKE_LENGTH = 17379
-
-
-def get_open_ml_dataset(open_ml_id, version=1):
-    dataset = openml.datasets.get_dataset(open_ml_id, version=version, download_data=True)
-    class_label = dataset.default_target_attribute
-    print(f"Loaded openML dataset '{dataset.name}', the target feature is '{class_label}'.")
-    x_data = dataset.get_data()[0]
-    return x_data, class_label
-
-
-# =============================================================================
-# Base Datasets
-# =============================================================================
-
-
-class BaseDataset(metaclass=ABCMeta):
-    """Base class for creating data sets.
-
-    Warning: This class should not be used directly.
-    Use derived classes instead.
-    """
-
-    @abstractmethod
-    def __init__(
-            self,
-            *,
-            dataset,
-            class_label,
-            random_seed=None,
-            drop_na=True,
-            shuffle_dataset=True,
-            feature_encoder=None,
-            features_to_encode=(),
-            feature_scaler=None,
-            features_to_scale=()
-    ):
-        if drop_na:
-            dataset.dropna(inplace=True)
-        if shuffle_dataset:
-            dataset = shuffle(dataset, random_state=random_seed)
-        self.x_data = dataset
-        self.y_data = dataset.pop(class_label)
-        self.feature_names = self.x_data.columns
-        if feature_encoder is not None:
-            self.encode_features(features_to_encode=features_to_encode, feature_encoder=feature_encoder)
-        if feature_scaler is not None:
-            self.scale_features(features_to_scale=features_to_scale, feature_scaler=feature_scaler)
-        self.n_features = len(self.feature_names)
-        self.dataset_size = len(self.x_data)
-        self._label_encoder = LabelEncoder()
-        self._transform_label_column()
-
-    def to_stream(self):
-        return river.stream.iter_pandas(self.x_data, self.y_data)
-
-    def _transform_features(self, feature_names, transformer):
-        for feature in feature_names:
-            self.x_data[feature] = transformer.fit_transform(self.x_data[feature].values.reshape(-1, 1))
-
-    def scale_features(self, features_to_scale, feature_scaler: object):
-        if feature_scaler == 'standard':
-            feature_scaler = sklearn.preprocessing.StandardScaler()
-        if feature_scaler == 'robust':
-            feature_scaler = sklearn.preprocessing.Robustcaler()
-        if not hasattr(feature_scaler, "fit_transform"):
-            raise ValueError("Feature scaler must be either 'standard', 'robust', or a feature transformer instance "
-                             "(like 'sklearn.preprocessing') implementing a 'fit_transform' method.")
-        self._transform_features(feature_names=features_to_scale, transformer=feature_scaler)
-
-    def encode_features(self, features_to_encode, feature_encoder):
-        if feature_encoder == 'ordinal':
-            feature_encoder = sklearn.preprocessing.OrdinalEncoder()
-        if not hasattr(feature_encoder, "fit_transform"):
-            raise ValueError("Feature encoder must be either 'ordinal', or a feature transformer instance "
-                             "(like 'sklearn.preprocessing') implementing a 'fit_transform' method.")
-        self._transform_features(feature_names=features_to_encode, transformer=feature_encoder)
-
-    def _transform_label_column(self):
-        self.y_data = self._label_encoder.fit_transform(y=self.y_data)
-
-    @property
-    def label_encoding(self):
-        return self._label_encoder.get_params()
-
-
-# =============================================================================
-# Public Datasets
-# =============================================================================
-
-
-class Adult(BaseDataset):
-
-    def __init__(
-            self,
-            version=2,
-            random_seed=None,
-            drop_na=True,
-            shuffle_dataset=True,
-            feature_encoder=None,
-            features_to_encode=(),
-            feature_scaler=None,
-            features_to_scale=()
-    ):
-        assert version in [1, 2], "OpenML census dataset version must be '1' or '2'."
-        dataset, class_label = get_open_ml_dataset("adult", version=version)
-        dataset[OPEN_ML_ADULT_NUM_FEATURES] = dataset[OPEN_ML_ADULT_NUM_FEATURES].apply(pd.to_numeric)
-
-        if feature_encoder is not None:
-            if features_to_encode == ():
-                features_to_encode = OPEN_ML_ADULT_CATEGORICAL_FEATURE_NAMES
-        if feature_scaler is not None:
-            if features_to_scale == ():
-                features_to_scale = OPEN_ML_ADULT_NUM_FEATURES
-
-        super(Adult, self).__init__(
-            dataset=dataset,
-            class_label=class_label,
-            drop_na=drop_na,
-            random_seed=random_seed,
-            shuffle_dataset=shuffle_dataset,
-            feature_encoder=feature_encoder,
-            features_to_encode=features_to_encode,
-            feature_scaler=feature_scaler,
-            features_to_scale=features_to_scale
-        )
-        self.categorical_feature_names = OPEN_ML_ADULT_CATEGORICAL_FEATURE_NAMES
-        self.classification = True
-
-
-if __name__ == "__main__":
-    test_dataset = Adult(random_seed=42, feature_encoder='ordinal', feature_scaler='standard')
-    stream = test_dataset.to_stream()
-    for i, sample in enumerate(stream):
-        print(i, sample)
-        if i >= 3:
-            break
diff --git a/data/stream/__init__.py b/data/stream/__init__.py
new file mode 100644
index 0000000..c64e568
--- /dev/null
+++ b/data/stream/__init__.py
@@ -0,0 +1,11 @@
+from .elec2 import Elec2
+from .insects import Insects
+from .credit_card import CreditCard
+from .bike_stream import Bike
+
+__all__ = [
+    "Elec2",
+    "Insects",
+    "CreditCard",
+    "Bike"
+]
diff --git a/data/stream/_base.py b/data/stream/_base.py
new file mode 100644
index 0000000..1ad7dcf
--- /dev/null
+++ b/data/stream/_base.py
@@ -0,0 +1,167 @@
+from random import random
+from typing import Optional, Generator, Union, Dict
+import numpy as np
+from river.datasets.base import Dataset
+
+
+__all__ = [
+    "BatchStream",
+    "FeatureSwitchStream",
+    "ConceptDriftStream",
+    "StreamDataset"
+]
+
+
+class BatchStream(Dataset):
+
+    def __init__(self, stream, task, n_features, n_classes=None, n_outputs=None):
+        super().__init__(task, n_features, n_classes=n_classes, n_outputs=n_outputs)
+        self.stream_gen = stream
+
+    def __iter__(self):
+        for x_i, y_i in self.stream_gen:
+            yield x_i, y_i
+
+
+class ConceptDriftStream(Dataset):
+
+    def __init__(
+            self,
+            stream,
+            drift_stream,
+            position: int = 5000,
+            width: int = 1000,
+            alpha: float = None,
+    ):
+        super().__init__(
+            n_features=stream.n_features,
+            n_classes=stream.n_classes,
+            n_outputs=stream.n_outputs,
+            task=stream.task,
+        )
+        self.alpha = alpha
+        if self.alpha is not None:
+            if 0 < self.alpha <= 90.0:
+                w = int(1 / np.tan(self.alpha * np.pi / 180))
+                self.width = w if w > 0 else 1
+            else:
+                raise ValueError(
+                    f"Invalid alpha value: {alpha}. " f"Valid values are in the range (0.0, 90.0]"
+                )
+        else:
+            self.width = width
+        self.position = position
+        self.stream = stream
+        self.drift_stream = drift_stream
+
+    def __iter__(self):
+        stream_generator = iter(self.stream)
+        drift_stream_generator = iter(self.drift_stream)
+        sample_idx = 0
+
+        while True:
+            sample_idx += 1
+            v = -4.0 * float(sample_idx - self.position) / float(self.width)
+            probability_drift = 1.0 / (1.0 + np.exp(v))
+            try:
+                if random() > probability_drift:
+                    try:
+                        x, y = next(stream_generator)
+                    except StopIteration:
+                        x, y = next(drift_stream_generator)
+                else:
+                    try:
+                        x, y = next(drift_stream_generator)
+                    except StopIteration:
+                        x, y = next(stream_generator)
+            except StopIteration:
+                break
+            yield x, y
+
+
+class FeatureSwitchStream(Dataset):
+
+    def __init__(self, feature_switch, stream, task, n_features, n_classes=None, n_outputs=None):
+        super().__init__(task, n_features, n_classes=n_classes, n_outputs=n_outputs)
+        self.stream = stream
+        self.feature_switch = feature_switch
+
+    def __iter__(self):
+        for x, y in self.stream.__iter__():
+            for feature_1, feature_2 in self.feature_switch.items():
+                x[feature_1], x[feature_2] = x[feature_2], x[feature_1]
+            yield x, y
+
+
+class StreamDataset(Dataset):
+
+    def __init__(self, stream, n_samples, task, n_features, n_outputs, feature_names, cat_feature_names, num_feature_names):
+        super().__init__(task, n_features)
+        self.stream = stream
+        self.n_outputs = n_outputs
+        self.feature_names = feature_names
+        self.cat_feature_names = cat_feature_names
+        self.num_feature_names = num_feature_names
+        self.n_samples = n_samples
+
+    def __iter__(self):
+        n = 0
+        stream_iter = iter(self.stream)
+        while n < self.n_samples:
+            yield next(stream_iter)
+            n += 1
+
+
+def slice_stream(stream: Generator, position: int):
+    first_stream = []
+    second_stream = []
+    for n, stream_i in enumerate(stream):
+        if n < position:
+            first_stream.append(stream_i)
+        else:
+            second_stream.append(stream_i)
+    first_stream = BatchStream(
+        stream=first_stream,
+        task=stream.task, n_features=stream.n_features, n_outputs=stream.n_outputs, n_classes=stream.n_classes)
+    second_stream = BatchStream(
+        stream=second_stream,
+        task=stream.task, n_features=stream.n_features, n_outputs=stream.n_outputs, n_classes=stream.n_classes)
+    return first_stream, second_stream
+
+
+def _get_feature_remapping_dict(feature_remapping_str: str):
+    feature_remapping = {}
+    for features in feature_remapping_str.split(' '):
+        feature_1, feature_2 = features.split('_')
+        feature_remapping[feature_1] = feature_2
+        feature_remapping[feature_2] = feature_1
+    return feature_remapping
+
+
+def get_concept_drift_stream(
+        stream,
+        position: int,
+        drift_stream: Optional[Generator] = None,
+        feature_remapping: Optional[Union[Dict[str, str], str]] = None,
+        width: int = 1
+):
+
+    if drift_stream is None:
+        stream, drift_stream = slice_stream(stream=stream, position=position)
+
+    if feature_remapping is not None:
+        if type(feature_remapping) == str:
+            feature_remapping = _get_feature_remapping_dict(feature_remapping)
+        drift_stream = FeatureSwitchStream(
+            feature_switch=feature_remapping, stream=drift_stream,
+            task=drift_stream.task, n_features=drift_stream.n_features,
+            n_classes=drift_stream.n_classes, n_outputs=drift_stream.n_outputs)
+
+    concept_drift_stream = ConceptDriftStream(
+        stream=stream, drift_stream=drift_stream, position=position, width=width)
+
+    return concept_drift_stream
+
+
+if __name__ == "__main__":
+    pass
diff --git a/data/stream/bike_stream.py b/data/stream/bike_stream.py
new file mode 100644
index 0000000..8f41055
--- /dev/null
+++ b/data/stream/bike_stream.py
@@ -0,0 +1,40 @@
+from typing import Optional
+
+from river.datasets import Bikes as RiverDataset
+from data.stream._base import StreamDataset
+
+
+class Bike(StreamDataset):
+
+    def __init__(
+            self,
+            n_samples: Optional[int] = 182470
+    ):
+        if n_samples is None:
+            n_samples = 182470
+        assert n_samples <= 182470, f"Bike contains at maximum  182470 samples."
+        stream = RiverDataset()
+        feature_names = ['moment', 'station', 'clouds', 'description', 'humidity', 'pressure', 'temperature', 'wind']
+        cat_feature_names = ['station', 'description']
+        num_feature_names = ['clouds', 'humidity', 'pressure', 'temperature', 'wind']
+        super().__init__(
+            stream=stream,
+            n_samples=n_samples,
+            feature_names=feature_names,
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names,
+            task=stream.task,
+            n_features=len(feature_names),
+            n_outputs=stream.n_outputs,
+        )
+
+
+if __name__ == "__main__":
+    dataset = Bike()
+    stream = dataset.stream
+    print(stream.n_samples)
+    for n, (x_i, y_i) in enumerate(stream):
+        print(n, x_i, y_i)
+        if n > 3:
+            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
+            break
diff --git a/data/stream/credit_card.py b/data/stream/credit_card.py
new file mode 100644
index 0000000..12ddbb9
--- /dev/null
+++ b/data/stream/credit_card.py
@@ -0,0 +1,40 @@
+from typing import Optional
+
+from river.datasets import CreditCard as RiverDataset
+from data.stream._base import StreamDataset
+
+
+class CreditCard(StreamDataset):
+
+    def __init__(
+            self,
+            n_samples: Optional[int] = 284_807
+    ):
+        if n_samples is None:
+            n_samples = 284_807
+        assert n_samples <= 284_807, f"CreditCard contains at maximum  284_807 samples."
+        stream = RiverDataset()
+        feature_names = ['Time', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'Amount']
+        cat_feature_names = []
+        num_feature_names = feature_names
+        super().__init__(
+            stream=stream,
+            n_samples=n_samples,
+            feature_names=feature_names,
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names,
+            task=stream.task,
+            n_features=len(feature_names),
+            n_outputs=stream.n_outputs,
+        )
+
+
+if __name__ == "__main__":
+    dataset = CreditCard()
+    stream = dataset.stream
+    print(stream.n_samples)
+    for n, (x_i, y_i) in enumerate(stream):
+        print(n, x_i, y_i)
+        if n > 3:
+            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
+            break
diff --git a/data/stream/elec2.py b/data/stream/elec2.py
new file mode 100644
index 0000000..15d48c8
--- /dev/null
+++ b/data/stream/elec2.py
@@ -0,0 +1,40 @@
+from typing import Optional
+
+from river.datasets import Elec2 as RiverDataset
+from data.stream._base import StreamDataset
+
+
+class Elec2(StreamDataset):
+
+    def __init__(
+            self,
+            n_samples: Optional[int] = 45312
+    ):
+        if n_samples is None:
+            n_samples = 45312
+        assert n_samples <= 45312, f"Elec2 contains at maximum  45312 samples."
+        stream = RiverDataset()
+        feature_names = ['date', 'day', 'period', 'nswprice', 'nswdemand', 'vicprice', 'vicdemand', 'transfer']
+        cat_feature_names = []
+        num_feature_names = feature_names
+        super().__init__(
+            stream=stream,
+            n_samples=n_samples,
+            feature_names=feature_names,
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names,
+            task=stream.task,
+            n_features=len(feature_names),
+            n_outputs=stream.n_outputs,
+        )
+
+
+if __name__ == "__main__":
+    dataset = Elec2()
+    stream = dataset.stream
+    print(stream.n_samples)
+    for n, (x_i, y_i) in enumerate(stream):
+        print(n, x_i, y_i)
+        if n > 3:
+            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
+            break
diff --git a/data/stream/insects.py b/data/stream/insects.py
new file mode 100644
index 0000000..c9c8ffd
--- /dev/null
+++ b/data/stream/insects.py
@@ -0,0 +1,60 @@
+from typing import Optional
+from river.datasets import Insects as RiverDataset
+from data.stream._base import StreamDataset
+
+
+class Insects(StreamDataset):
+
+    def __init__(
+            self,
+            variant="abrupt_balanced",
+            n_samples: Optional[int] = None
+    ):
+        stream = RiverDataset()
+        feature_names = ['Att1', 'Att2', 'Att3', 'Att4', 'Att5', 'Att6', 'Att7', 'Att8', 'Att9', 'Att10', 'Att11',
+                              'Att12', 'Att13', 'Att14', 'Att15', 'Att16', 'Att17', 'Att18', 'Att19', 'Att20', 'Att21',
+                              'Att22', 'Att23', 'Att24', 'Att25', 'Att26', 'Att27', 'Att28', 'Att29', 'Att30', 'Att31',
+                              'Att32', 'Att33']
+        cat_feature_names = []
+        num_feature_names = ['Att1', 'Att2', 'Att3', 'Att4', 'Att5', 'Att6', 'Att7', 'Att8', 'Att9', 'Att10',
+                                  'Att11', 'Att12', 'Att13', 'Att14', 'Att15', 'Att16', 'Att17', 'Att18', 'Att19',
+                                  'Att20', 'Att21', 'Att22', 'Att23', 'Att24', 'Att25', 'Att26', 'Att27', 'Att28',
+                                  'Att29', 'Att30', 'Att31', 'Att32', 'Att33']
+        variant_sizes = {
+            "abrupt_balanced": (52_848, 16_419_025),
+            "abrupt_imbalanced": (355_275, 110_043_637),
+            "gradual_balanced": (24_150, 7_503_750),
+            "gradual_imbalanced": (143_323, 44_371_501),
+            "incremental-abrupt_balanced": (79_986, 24_849_436),
+            "incremental-abrupt_imbalanced": (452_044, 140_004_225),
+            "incremental-reoccurring_balanced": (79_986, 24_849_092),
+            "incremental-reoccurring_imbalanced": (452_044, 140_004_230),
+            "incremental_balanced": (57_018, 17_713_574),
+            "incremental_imbalanced": (452_044, 140_004_218),
+            "out-of-control": (905_145, 277_777_854),
+        }
+        n_samples_variant = variant_sizes[variant][0]
+        if n_samples is not None and n_samples > n_samples_variant:
+            raise ValueError(f"The dataset with variant {variant} contains less samples, {n_samples_variant} "
+                             f"than expected which was {n_samples}. Select less samples.")
+        super().__init__(
+            stream=stream,
+            n_samples=n_samples,
+            feature_names=feature_names,
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names,
+            task=stream.task,
+            n_features=len(feature_names),
+            n_outputs=stream.n_outputs,
+        )
+
+
+if __name__ == "__main__":
+    dataset = Insects()
+    stream = dataset.stream
+    print(stream.n_samples)
+    for n, (x_i, y_i) in enumerate(stream):
+        print(n, x_i, y_i)
+        if n > 3:
+            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
+            break
diff --git a/data/stream/synth/__init__.py b/data/stream/synth/__init__.py
new file mode 100644
index 0000000..a406d66
--- /dev/null
+++ b/data/stream/synth/__init__.py
@@ -0,0 +1,11 @@
+from ._base import SyntheticDataset
+from .agrawal import Agrawal
+from .planes import Planes2D
+from .stagger import Stagger
+
+__all__ = [
+    "Stagger",
+    "Agrawal",
+    "Planes2D",
+    "SyntheticDataset"
+]
diff --git a/data/synthetic.py b/data/stream/synth/_base.py
similarity index 98%
rename from data/synthetic.py
rename to data/stream/synth/_base.py
index 208dc29..668fa83 100644
--- a/data/synthetic.py
+++ b/data/stream/synth/_base.py
@@ -1,5 +1,5 @@
 from abc import ABCMeta, abstractmethod
-from typing import Callable, Optional
+from typing import Callable, Optional, List
 
 from numpy import ndarray
 import river
@@ -34,7 +34,7 @@ def __init__(
             *,
             x_data: ndarray,
             y_data: ndarray,
-            feature_names: list[str]
+            feature_names: List[str]
     ):
         self.x_data = x_data
         self.y_data = y_data
diff --git a/data/stream/synth/agrawal.py b/data/stream/synth/agrawal.py
new file mode 100644
index 0000000..eb66acb
--- /dev/null
+++ b/data/stream/synth/agrawal.py
@@ -0,0 +1,41 @@
+from typing import Optional
+from river.datasets.synth import Agrawal as RiverDataset
+
+from data.stream._base import StreamDataset
+
+
+class Agrawal(StreamDataset):
+
+    def __init__(
+            self,
+            classification_function,
+            random_seed: Optional[int] = None,
+            n_samples: Optional[int] = None
+    ):
+        if n_samples is None:
+            n_samples = 20000
+        stream = RiverDataset(classification_function=classification_function, seed=random_seed)
+        feature_names = ['salary', 'commission', 'age', 'elevel', 'car', 'zipcode', 'hvalue', 'hyears', 'loan']
+        cat_feature_names = ['elevel', 'car', 'zipcode']
+        num_feature_names = ['salary', 'commission', 'age', 'hvalue', 'hyears', 'loan']
+        super().__init__(
+            stream=stream,
+            n_samples=n_samples,
+            feature_names=feature_names,
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names,
+            task=stream.task,
+            n_features=len(feature_names),
+            n_outputs=stream.n_outputs,
+        )
+
+
+if __name__ == "__main__":
+    dataset = Agrawal(classification_function=1, random_seed=42)
+    stream = dataset.stream
+    print(stream.n_samples)
+    for n, (x_i, y_i) in enumerate(stream):
+        print(n, x_i, y_i)
+        if n > 3:
+            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
+            break
diff --git a/data/stream/synth/planes.py b/data/stream/synth/planes.py
new file mode 100644
index 0000000..22a6928
--- /dev/null
+++ b/data/stream/synth/planes.py
@@ -0,0 +1,39 @@
+from typing import Optional
+from river.datasets.synth import Planes2D as RiverDataset
+
+from data.stream._base import StreamDataset
+
+class Planes2D(StreamDataset):
+
+    def __init__(
+            self,
+            random_seed: Optional[int] = None,
+            n_samples:  Optional[int] = None
+    ):
+        if n_samples is None:
+            n_samples = 20000
+        stream = RiverDataset(seed=random_seed)
+        feature_names = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        cat_feature_names = []
+        num_feature_names = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        super().__init__(
+            stream=stream,
+            n_samples=n_samples,
+            feature_names=feature_names,
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names,
+            task=stream.task,
+            n_features=len(feature_names),
+            n_outputs=stream.n_outputs,
+        )
+
+
+if __name__ == "__main__":
+    dataset = Planes2D(random_seed=42)
+    stream = dataset.stream
+    print(stream.n_samples)
+    for n, (x_i, y_i) in enumerate(stream):
+        print(n, x_i, y_i)
+        if n > 3:
+            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
+            break
diff --git a/data/stream/synth/stagger.py b/data/stream/synth/stagger.py
new file mode 100644
index 0000000..88dd1e4
--- /dev/null
+++ b/data/stream/synth/stagger.py
@@ -0,0 +1,41 @@
+from typing import Optional
+from river.datasets.synth import STAGGER as RiverDataset
+
+from data.stream._base import StreamDataset
+
+
+class Stagger(StreamDataset):
+
+    def __init__(
+            self,
+            classification_function,
+            random_seed: Optional[int] = None,
+            n_samples: Optional[int] = 10000
+    ):
+        if n_samples is None:
+            n_samples = 10000
+        stream = RiverDataset(classification_function=classification_function, seed=random_seed)
+        feature_names = ['size', 'color', 'shape']
+        cat_feature_names = ['size', 'color', 'shape']
+        num_feature_names = []
+        super().__init__(
+            stream=stream,
+            n_samples=n_samples,
+            feature_names=feature_names,
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names,
+            task=stream.task,
+            n_features=len(feature_names),
+            n_outputs=stream.n_outputs,
+        )
+
+
+if __name__ == "__main__":
+    dataset = Stagger(classification_function=1, random_seed=42)
+    stream = dataset.stream
+    print(stream.n_samples)
+    for n, (x_i, y_i) in enumerate(stream):
+        print(n, x_i, y_i)
+        if n > 3:
+            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
+            break
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..d4bb2cb
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/api.rst b/docs/api.rst
new file mode 100644
index 0000000..513f25e
--- /dev/null
+++ b/docs/api.rst
@@ -0,0 +1,9 @@
+:github_url: https://github.com/mmschlk/IncrementExplain.git
+
+API Reference
+=============
+
+.. toctree::
+    :maxdepth: 1
+
+    storage
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..45512c3
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,93 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+from sphinx.builders.html import StandaloneHTMLBuilder
+import os
+import sys
+import increment_explain
+
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'IncrementExplain'
+copyright = '2022, Maximilian Muschalik, Rohit Jagtani'
+author = 'Maximilian Muschalik, Rohit Jagtani'
+release = increment_explain.__version__
+version = increment_explain.__version__
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "myst_parser",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.autosectionlabel",
+    "sphinx_autodoc_typehints",
+    "sphinx_toolbox.more_autodoc.autoprotocol",
+]
+
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+source_suffix = {
+    ".rst": "restructuredtext",
+    ".md": "markdown",
+}
+
+intersphinx_mapping = {
+    "python3": ("https://docs.python.org/3", None),
+    "numpy": ("https://numpy.org/doc/stable/", None)
+}
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+# The master toctree document.
+master_doc = "index"
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_title = f"{project} {version} Documentation"
+html_theme = "pydata_sphinx_theme"
+html_context = {"default_mode": "auto"}
+html_theme_options = {
+    "icon_links": [
+        {"name": "GitHub", "url": "https://github.com/mmschlk/IncrementExplain.git", "icon": "fab fa-github-square"},
+    ],
+    "collapse_navigation": True,
+    "navigation_depth": 3,
+    "show_toc_level": 1,
+    "footer_items": ["copyright"],
+    "navbar_align": "content",
+    "show_prev_next": False,
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ["_static"]
+
+
+# set priority when building html
+StandaloneHTMLBuilder.supported_image_types = ["image/svg+xml", "image/gif", "image/png", "image/jpeg"]
+# -- Extension configuration -------------------------------------------------
+# Ignore >>> when copying code
+copybutton_prompt_text = r">>> |\.\.\. "
+copybutton_prompt_is_regexp = True
diff --git a/docs/contribution.rst b/docs/contribution.rst
new file mode 100644
index 0000000..7da4cb7
--- /dev/null
+++ b/docs/contribution.rst
@@ -0,0 +1,11 @@
+:github_url: https://github.com/mmschlk/IncrementExplain.git
+
+Contribution Guideline
+======================
+
+
+Set up the environment
+----------------------
+
+#. Fork the repo and then clone it.
+#. ⚠️ Install python >= 3.7.
\ No newline at end of file
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..5981912
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,32 @@
+:github_url: https://github.com/mmschlk/IncrementExplain.git
+
+.. IncrementExplain documentation master file, created by
+   sphinx-quickstart on Fri Oct 14 14:54:02 2022.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+IncrementExplain
+============================================
+``IncrementExplain`` is a library which unifies explainability methods for incremental models.
+
+.. note::
+   The first formal version *(0.1.0)* of ``IncrementExplain`` is still a work in progress
+
+Welcome to IncrementExplain's documentation!
+============================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   installation
+   api
+   contribution
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/installation.rst b/docs/installation.rst
new file mode 100644
index 0000000..4899a8e
--- /dev/null
+++ b/docs/installation.rst
@@ -0,0 +1,11 @@
+:github_url: https://github.com/mmschlk/IncrementExplain.git
+
+
+Installation
+============
+
+Here are the steps for installing ``IncrementExplain``.
+
+#. Clone the `Repo <https://github.com/mmschlk/IncrementExplain.git>`_ using git.
+
+#. Install the package in the editable mode: ``pip install -e .`` (do not miss the ``.`` in the end).
\ No newline at end of file
diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 0000000..954237b
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/docs/requirements_doc.txt b/docs/requirements_doc.txt
new file mode 100644
index 0000000..8d89a12
--- /dev/null
+++ b/docs/requirements_doc.txt
@@ -0,0 +1,12 @@
+myst-parser
+numpy
+pydata-sphinx-theme
+sphinx
+sphinx-autodoc-typehints
+sphinx-toolbox
+sphinxcontrib-applehelp
+sphinxcontrib-devhelp
+sphinxcontrib-htmlhelp
+sphinxcontrib-jsmath
+sphinxcontrib-qthelp
+sphinxcontrib-serializinghtml
diff --git a/docs/storage.rst b/docs/storage.rst
new file mode 100644
index 0000000..f49b0e4
--- /dev/null
+++ b/docs/storage.rst
@@ -0,0 +1,51 @@
+:github_url: https://github.com/mmschlk/IncrementExplain.git
+
+.. _storage:
+
+Storage
+=======
+
+.. currentmodule:: increment_explain.storage
+
+
+.. autosummary::
+    :nosignatures:
+
+    BaseStorage
+    BatchStorage
+    IntervalStorage
+    SequenceStorage
+    ReservoirStorage
+    UniformReservoirStorage
+    GeometricReservoirStorage
+
+General Protocols and Base Classes
+----------------------------------
+
+Base Storage
+~~~~~~~~~~~~~~~~~~~
+.. autoclass:: BaseStorage
+
+Batch Storage
+~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: BatchStorage
+
+Interval Storage
+~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: IntervalStorage
+
+Sequence Storage
+~~~~~~~~~~~~~~~~~~
+.. autoclass:: SequenceStorage
+
+Reservoir Storage
+~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: ReservoirStorage
+
+Uniform Reservoir Storage
+~~~~~~~~~~~~~~~~~~
+.. autoclass:: UniformReservoirStorage
+
+Geometric Reservoir Storage
+~~~~~~~~~~~~~~~~~~
+.. autoclass:: GeometricReservoirStorage
\ No newline at end of file
diff --git a/data/tests/__init__.py b/experiments/__init__.py
similarity index 100%
rename from data/tests/__init__.py
rename to experiments/__init__.py
diff --git a/experiments/iSAGE_paper/batch_approximation.py b/experiments/iSAGE_paper/batch_approximation.py
new file mode 100644
index 0000000..f4085b1
--- /dev/null
+++ b/experiments/iSAGE_paper/batch_approximation.py
@@ -0,0 +1,138 @@
+from river.datasets.base import REG
+
+from experiments.setup.data import get_dataset
+from experiments.setup.explainer import get_incremental_sage_explainer, \
+    get_batch_sage_explainer, get_interval_sage_explainer, get_imputer_and_storage
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.explainer.sage import IncrementalSageExplainer, BatchSageExplainer, IntervalSageExplainer
+
+DEBUG = True
+
+DATASET_NAME = 'adult'
+DATASET_RANDOM_SEEDS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+DATASET_SHUFFLE = True
+N_SAMPLES = 20000
+
+MODEL_NAME = 'ARF'
+
+N_INNER_SAMPLES: int = 1
+N_INTERVAL_LENGTH: int = 2000
+SMOOTHING_ALPHA: float = 0.001
+FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product']
+RESERVOIR_LENGTH: int = 100
+RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
+
+TRAINING_METRIC_WINDOW = 100
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 25,
+    }
+}
+
+if __name__ == "__main__":
+
+    inc_final_values = []
+    bat_final_values = []
+
+    for DATASET_RANDOM_SEED in DATASET_RANDOM_SEEDS:
+
+        # Get Training Data --------------------------------------------------------------------------------------------
+        dataset = get_dataset(DATASET_NAME, random_seed=DATASET_RANDOM_SEED, n_samples=N_SAMPLES, shuffle=DATASET_SHUFFLE)
+        feature_names = dataset.feature_names
+        n_samples = dataset.n_samples
+        stream = dataset.stream
+
+        # Get Model / Loss and Training Procedure ----------------------------------------------------------------------
+        task = dataset.task
+        loss_function = get_loss_function(task=task)
+        rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+        model, model_function = get_model(
+            model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+        # Get explainers -----------------------------------------------------------------------------------------------
+        imputer, storage = get_imputer_and_storage(
+            model_function=model_function,
+            feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
+            reservoir_kind=RESERVOIR_KIND,
+            reservoir_length=RESERVOIR_LENGTH,
+            cat_feature_names=None,
+            num_feature_names=None
+        )
+
+        incremental_explainer = IncrementalSageExplainer(
+            model_function=model_function,
+            loss_function=loss_function,
+            imputer=imputer,
+            storage=storage,
+            feature_names=feature_names,
+            smoothing_alpha=SMOOTHING_ALPHA,
+            n_inner_samples=N_INNER_SAMPLES
+        )
+
+        batch_explainer = BatchSageExplainer(
+            model_function=model_function,
+            loss_function=loss_function,
+            feature_names=feature_names,
+            n_inner_samples=N_INNER_SAMPLES
+        )
+
+        interval_explainer = IntervalSageExplainer(
+            model_function=model_function,
+            loss_function=loss_function,
+            feature_names=feature_names,
+            n_inner_samples=N_INNER_SAMPLES,
+            interval_length=N_INTERVAL_LENGTH
+        )
+
+        model_performance = []
+
+        # Training Phase -----------------------------------------------------------------------------------------------
+        if DEBUG:
+            print(f"Starting Training for {n_samples}")
+        for (n, (x_i, y_i)) in enumerate(stream, start=1):
+            if task == REG:
+                y_i_pred = model.predict_one(x_i)
+            else:
+                y_i_pred = model.predict_proba_one(x_i)
+            rolling_training_metric.update(y_true=y_i, y_pred=y_i_pred)
+            model.learn_one(x_i, y_i)
+            if DEBUG and n % 1000 == 0:
+                print(f"{n}: performance {rolling_training_metric.get()}\n")
+            if n > n_samples:
+                break
+
+        # Get Explanation Data -----------------------------------------------------------------------------------------
+        dataset = get_dataset(DATASET_NAME, random_seed=DATASET_RANDOM_SEED, n_samples=N_SAMPLES, shuffle=DATASET_SHUFFLE)
+        n_samples = dataset.n_samples
+        stream = dataset.stream
+
+        # Explanation-Phase --------------------------------------------------------------------------------------------
+        if DEBUG:
+            print(f"Starting Explanation for {n_samples}")
+        for (n, (x_i, y_i)) in enumerate(stream, start=1):
+            inc_fi_values = incremental_explainer.explain_one(x_i, y_i)
+            int_fi_values = interval_explainer.explain_one(x_i, y_i)
+            batch_explainer.update_storage(x_i, y_i)
+            bat_fi_values = batch_explainer.importance_values
+            if DEBUG and n % 1000 == 0:
+                print(f"{n}: inc-sage {incremental_explainer.importance_values}")
+                print(f"{n}: int-sage {interval_explainer.importance_values}")
+                print(f"{n}: bat-sage {batch_explainer.importance_values}\n")
+            if n >= n_samples:
+                bat_fi_values = batch_explainer.explain_one(x_i, y_i)
+                int_fi_values = interval_explainer.explain_one(x_i, y_i, force_explain=True)
+                break
+
+        if DEBUG:
+            print(f"final values: inc-sage {incremental_explainer.importance_values}")
+            print(f"final values: int-sage {interval_explainer.importance_values}")
+            print(f"final values: bat-sage {batch_explainer.importance_values}\n")
+
+        inc_final_values.append(incremental_explainer.importance_values)
+        bat_final_values.append(batch_explainer.importance_values)
+
+    if DEBUG:
+        print("all inc values:", inc_final_values)
+        print("all bat values:", bat_final_values)
diff --git a/experiments/iSAGE_paper/concept_drift.py b/experiments/iSAGE_paper/concept_drift.py
new file mode 100644
index 0000000..98c4531
--- /dev/null
+++ b/experiments/iSAGE_paper/concept_drift.py
@@ -0,0 +1,242 @@
+import math
+import time
+
+import numpy as np
+import pandas as pd
+from river.datasets.base import REG
+
+from experiments.setup.data import get_dataset, get_concept_drift_dataset
+from experiments.setup.explainer import get_incremental_sage_explainer, \
+    get_interval_sage_explainer, get_incremental_pfi_explainer, get_imputer_and_storage
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IntervalSageExplainer, IncrementalSageExplainer
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+DEBUG = True
+
+DATASET_1_NAME = 'agrawal 1'
+DATASET_2_NAME = 'agrawal 2'
+DATASET_RANDOM_SEED = 1
+SHUFFLE_DATA = False
+N_SAMPLES_1 = 20000
+N_SAMPLES_2 = 20000
+
+CONCEPT_DRIFT_POSITION = 20000
+CONCEPT_DRIFT_WIDTH = 1
+CONCEPT_DRIFT_SWITCHING_FEATURES = None # {'salary': 'age'}
+
+MODEL_NAME = 'ARF'
+
+N_INNER_SAMPLES: int = 5
+N_INTERVAL_LENGTH: int = 5000
+CONFIDENCE_BOUNDS_DELTA: float = 0.1
+SMOOTHING_ALPHA: float = 0.001
+FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
+RESERVOIR_LENGTH: int = 100
+RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
+
+TRAINING_METRIC_WINDOW = 500
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 25,
+    }
+}
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    dataset_1 = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
+    dataset_2 = get_dataset(
+        dataset_name=DATASET_2_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_2)
+    dataset = get_concept_drift_dataset(
+        dataset_1_name=DATASET_1_NAME, dataset_2_name=DATASET_2_NAME,
+        dataset_1=dataset_1, dataset_2=dataset_2,
+        position=CONCEPT_DRIFT_POSITION, width=CONCEPT_DRIFT_WIDTH,
+        features_to_switch=CONCEPT_DRIFT_SWITCHING_FEATURES,
+    )
+    feature_names = dataset.feature_names
+    cat_feature_names = dataset.cat_feature_names
+    num_feature_names = dataset.num_feature_names
+    n_samples = dataset.n_samples
+    stream = dataset.stream
+
+    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
+    task = dataset.task
+    loss_function = get_loss_function(task=task)
+    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+    model, model_function = get_model(
+        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    imputer, storage = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
+        reservoir_kind=RESERVOIR_KIND,
+        reservoir_length=RESERVOIR_LENGTH,
+        cat_feature_names=cat_feature_names,
+        num_feature_names=num_feature_names
+    )
+
+    incremental_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    incremental_pfi = IncrementalPFI(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_samples=N_INNER_SAMPLES,
+        storage=storage
+    )
+
+    interval_explainer = IntervalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        feature_names=feature_names,
+        n_inner_samples=N_INNER_SAMPLES,
+        interval_length=N_INTERVAL_LENGTH
+    )
+
+    # warm-up because of errors ----------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        model.learn_one(x_i, y_i)
+        if n > 10:
+            break
+
+    plotter = FeatureImportancePlotter(
+        feature_names=feature_names
+    )
+
+    time_pfi = 0.
+    time_sage = 0.
+    time_interval = 0.
+    time_learning = 0.
+
+    sage_fi_values = []
+    interval_fi_values = []
+    pfi_fi_values = []
+
+    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    model_loss = []
+    marginal_loss = []
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        start_time = time.time()
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+        time_sage += time.time() - start_time
+        sage_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='inc')
+
+        # pfi inc
+        start_time = time.time()
+        fi_values = incremental_pfi.explain_one(x_i, y_i, update_storage=False)
+        time_pfi += time.time() - start_time
+        pfi_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='pfi')
+
+        # sage int
+        start_time = time.time()
+        fi_values = interval_explainer.explain_one(x_i, y_i)
+        time_interval += time.time() - start_time
+        interval_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='int')
+
+        # learning
+        start_time = time.time()
+        model.learn_one(x_i, y_i)
+        time_learning += time.time() - start_time
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
+                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
+                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
+                  f"{n}: pfi                 {incremental_pfi.importance_values}\n"
+                  )
+
+        if n >= n_samples:
+            break
+
+    # Store the results in a database ----------------------------------------------------------------------------------
+
+    facet = "left.csv"
+    data_folder = "incremental_setting_5"
+
+    name = "inc_fi_values"
+    df = pd.DataFrame(sage_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "int_fi_values"
+    df = pd.DataFrame(interval_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "pfi_fi_values"
+    df = pd.DataFrame(pfi_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "model_loss"
+    df = pd.DataFrame(model_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "marginal_loss"
+    df = pd.DataFrame(marginal_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
+                      "line_names": ["loss"], "names_to_highlight": ['loss'],
+                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+                      "markevery": {"model_loss": 100, "marginal_loss": 100},
+                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
+                                 "legend_items": [("marginal loss", "dashed", "red"),
+                                                  ("model loss", "solid", "red")]}}
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
+        performance_kw=performance_kw,
+        title=f'Agrawal Stream with Concept Drift',
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
+        v_lines=[{'x': CONCEPT_DRIFT_POSITION, 'ls': '--', 'c': 'black', 'linewidth': 1}],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'inc': '-', 'int': '--', 'pfi': '-'},
+        legend_style={"fontsize": "small"},
+        markevery={'inc': 10, 'int': 1, 'pfi': 10},
+        facet_not_to_highlight=['pfi'],
+        y_min=-0.05,
+        y_max=0.42,
+        save_name="result_data/concept_drift.png"
+    )
diff --git a/experiments/iSAGE_paper/conditional_vs_marginal.py b/experiments/iSAGE_paper/conditional_vs_marginal.py
new file mode 100644
index 0000000..bdf4df0
--- /dev/null
+++ b/experiments/iSAGE_paper/conditional_vs_marginal.py
@@ -0,0 +1,225 @@
+import math
+import time
+
+import numpy as np
+import pandas as pd
+from river.datasets.base import REG
+
+from experiments.setup.data import get_dataset, get_concept_drift_dataset
+from experiments.setup.explainer import get_incremental_sage_explainer, \
+    get_interval_sage_explainer, get_incremental_pfi_explainer, get_imputer_and_storage
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.explainer.sage import IncrementalSageExplainer
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+DEBUG = True
+
+DATASET_1_NAME = 'elec2'
+
+DATASET_RANDOM_SEED = 1
+SHUFFLE_DATA = False
+N_SAMPLES_1 = None
+
+MODEL_NAME = 'ARF'
+
+N_INNER_SAMPLES: int = 2
+SMOOTHING_ALPHA: float = 0.001
+RESERVOIR_LENGTH: int = 100
+
+TRAINING_METRIC_WINDOW = 500
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 25,
+    }
+}
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    dataset = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
+    feature_names = dataset.feature_names
+    cat_feature_names = dataset.cat_feature_names
+    num_feature_names = dataset.num_feature_names
+    n_samples = dataset.n_samples
+    stream = dataset.stream
+
+    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
+    task = dataset.task
+    loss_function = get_loss_function(task=task)
+    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+    model, model_function = get_model(
+        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    # conditional
+    imputer_cond, storage_cond = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution="conditional",
+        reservoir_kind=None,
+        reservoir_length=None,
+        cat_feature_names=cat_feature_names,
+        num_feature_names=num_feature_names
+    )
+    incremental_sage_cond = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer_cond,
+        storage=storage_cond,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    # marginal
+    imputer_marg, storage_marg = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution="marginal joint",
+        reservoir_kind="geometric",
+        reservoir_length=RESERVOIR_LENGTH,
+        cat_feature_names=None,
+        num_feature_names=None
+    )
+    incremental_sage_marg = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer_marg,
+        storage=storage_marg,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    # warm-up because of errors ------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        model.learn_one(x_i, y_i)
+        if n > 10:
+            break
+
+    plotter = FeatureImportancePlotter(
+        feature_names=feature_names
+    )
+
+    time_marg = 0.
+    time_cond = 0.
+    time_learning = 0.
+
+    sage_cond_fi_values = []
+    sage_marg_fi_values = []
+
+    model_loss_tracker_cond = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    marginal_loss_tracker_cond = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    model_loss_cond = []
+    marginal_loss_cond = []
+
+    model_loss_tracker_marg = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    marginal_loss_tracker_marg = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    model_loss_marg = []
+    marginal_loss_marg = []
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker_cond.update(loss_i)
+        model_loss_cond.append({"loss": model_loss_tracker_cond.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage_cond._marginal_prediction_tracker.get())
+        marginal_loss_tracker_cond.update(loss_marginal_i)
+        marginal_loss_cond.append({"loss": marginal_loss_tracker_cond.get()})
+
+        model_loss_tracker_marg.update(loss_i)
+        model_loss_marg.append({"loss": model_loss_tracker_marg.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage_cond._marginal_prediction_tracker.get())
+        marginal_loss_tracker_marg.update(loss_marginal_i)
+        marginal_loss_marg.append({"loss": marginal_loss_tracker_marg.get()})
+
+        # sage inc cond
+        start_time = time.time()
+        fi_values = incremental_sage_cond.explain_one(x_i, y_i)
+        time_cond += time.time() - start_time
+        sage_cond_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='cond')
+
+        # sage inc marg
+        start_time = time.time()
+        fi_values = incremental_sage_marg.explain_one(x_i, y_i)
+        time_marg += time.time() - start_time
+        sage_marg_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='marg')
+
+        # learning
+        start_time = time.time()
+        model.learn_one(x_i, y_i)
+        time_learning += time.time() - start_time
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: conditional\n"
+                  f"{n}: marginal-prediction {incremental_sage_cond._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker_cond.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker_cond.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker_cond.get() - model_loss_tracker_cond.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage_cond.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage_cond.importance_values}\n"
+                  f"{n}: marginal\n"
+                  f"{n}: marginal-prediction {incremental_sage_marg._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker_marg.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker_marg.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker_marg.get() - model_loss_tracker_marg.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage_marg.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage_marg.importance_values}\n"
+                  )
+
+        if n >= n_samples:
+            break
+
+    # Store the results in a database ----------------------------------------------------------------------------------
+
+    data_folder = "conditional_marginal"
+    facet = "right.csv"
+
+    name = "cond_fi_values"
+    df = pd.DataFrame(sage_cond_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "marg_fi_values"
+    df = pd.DataFrame(sage_marg_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "model_loss"
+    df = pd.DataFrame(model_loss_cond)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "marginal_loss"
+    df = pd.DataFrame(marginal_loss_cond)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "cross\nentropy", "color_list": ["red", "black"],
+                      "line_names": ["loss"], "names_to_highlight": ['loss'],
+                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+                      "markevery": {"model_loss": 100, "marginal_loss": 100},
+                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
+                                 "legend_items": [("marginal loss", "dashed", "red"),
+                                                  ("model loss", "solid", "red")]}}
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"model_loss": model_loss_cond, "marginal_loss": marginal_loss_cond},
+        performance_kw=performance_kw,
+        title=f'Agrawal Stream with Concept Drift',
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'cond': '-', 'marg': '--'},
+        legend_style={"fontsize": "small"},
+        markevery={'cond': 100, 'marg': 100},
+        y_min=-0.049,
+        y_max=0.325,
+        save_name="result_data/conditional_vs_marginal.png"
+    )
diff --git a/experiments/iSAGE_paper/confidence_bounds.py b/experiments/iSAGE_paper/confidence_bounds.py
new file mode 100644
index 0000000..65d4fbe
--- /dev/null
+++ b/experiments/iSAGE_paper/confidence_bounds.py
@@ -0,0 +1,206 @@
+import math
+
+import pandas as pd
+from river.datasets.base import REG
+
+from experiments.setup.data import get_dataset
+from experiments.setup.explainer import get_imputer_and_storage
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.explainer.sage import IncrementalSageExplainer
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+DEBUG = True
+
+DATASET_1_NAME = 'elec2'
+DATASET_RANDOM_SEED = 1
+SHUFFLE_DATA = False
+N_SAMPLES_1 = None
+
+
+CONCEPT_DRIFT_WIDTH = 1
+CONCEPT_DRIFT_SWITCHING_FEATURES = None # {'salary': 'age'}
+
+TRAINING_METRIC_WINDOW = 500
+MODEL_NAME = 'ARF'
+
+CONFIDENCE_BOUNDS_1_DELTA = 0.1
+CONFIDENCE_BOUNDS_2_DELTA = 0.05
+N_INNER_SAMPLES: int = 1
+N_INTERVAL_LENGTH: int = 2000
+SMOOTHING_ALPHA: float = 0.001
+FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
+RESERVOIR_LENGTH: int = 100
+RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 25,
+    }
+}
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    dataset = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
+    feature_names = dataset.feature_names
+    cat_feature_names = dataset.cat_feature_names
+    num_feature_names = dataset.num_feature_names
+    n_samples = dataset.n_samples
+    stream = dataset.stream
+
+    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
+    task = dataset.task
+    loss_function = get_loss_function(task=task)
+    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+    model, model_function = get_model(
+        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+    # Get explainers ---------------------------------------------------------------------------------------------------
+    imputer, storage = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
+        reservoir_kind=RESERVOIR_KIND,
+        reservoir_length=RESERVOIR_LENGTH,
+        cat_feature_names=cat_feature_names,
+        num_feature_names=num_feature_names
+    )
+
+    incremental_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    # warm-up because of errors ------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        model.learn_one(x_i, y_i)
+        if n > 10:
+            break
+
+    plotter = FeatureImportancePlotter(
+        feature_names=feature_names
+    )
+
+    sage_fi_values = []
+
+    confidence_bounds_1 = []
+    confidence_bounds_2 = []
+    std = []
+
+    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    model_loss = []
+    marginal_loss = []
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+
+        # storing data
+        plotter.update(fi_values, facet_name='inc')
+        sage_fi_values.append(fi_values)
+        confidence_bounds_1.append(incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_1_DELTA))
+        confidence_bounds_2.append(incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_2_DELTA))
+        std.append({feature: math.sqrt(value.get()) for feature, value in incremental_sage.variances.items()})
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: performance       {rolling_training_metric.get()}\n"
+                  f"{n}: inc-sage          {incremental_sage.importance_values}\n"
+                  f"{n}: sum-sage          {sum(list(incremental_sage.importance_values.values()))}\n"
+                  f"{n}: inc-sage-bnd (1)  {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_1_DELTA)}\n"
+                  f"{n}: inc-sage-bnd (2)  {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_2_DELTA)}\n"
+                  f"{n}: inc-sage-var      {incremental_sage.variances}\n"
+                  )
+
+        if n >= n_samples:
+            break
+
+    # Store the results in a database ----------------------------------------------------------------------------------
+
+    facet = ".csv"
+    data_folder = "confidence_bounds"
+
+    name = "inc_fi_values"
+    df = pd.DataFrame(sage_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "confidence_bounds_1"
+    df = pd.DataFrame(confidence_bounds_1)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "confidence_bounds_2"
+    df = pd.DataFrame(confidence_bounds_2)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "model_loss"
+    df = pd.DataFrame(model_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "marginal_loss"
+    df = pd.DataFrame(marginal_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "cross\nentropy", "color_list": ["red", "black"],
+                      "line_names": ["loss"], "names_to_highlight": ['loss'],
+                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+                      "markevery": {"model_loss": 100, "marginal_loss": 100},
+                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
+                                 "legend_items": [("marginal loss", "dashed", "red"),
+                                                  ("model loss", "solid", "red")]}}
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
+        performance_kw=performance_kw,
+        title=f"{DATASET_1_NAME} data stream, confidence bounds with " + r"$\delta =$ " + f"{CONFIDENCE_BOUNDS_1_DELTA}",
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['nswprice', 'vicprice', 'nswdemand'],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'inc': '-'},
+        legend_style={},
+        markevery={'inc': 10},
+        std={'inc': confidence_bounds_1},
+        y_min=-0.1,
+        y_max=0.75,
+        save_name="result_data/confidence_bounds_delta_1.png"
+    )
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
+        performance_kw=performance_kw,
+        title=f"{DATASET_1_NAME} data stream, confidence bounds with " + r"$\delta =$ " + f"{CONFIDENCE_BOUNDS_2_DELTA}",
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['nswprice', 'vicprice', 'nswdemand'],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'inc': '-'},
+        legend_style={},
+        markevery={'inc': 10},
+        std={'inc': confidence_bounds_2},
+        y_min=-0.1,
+        y_max=0.75,
+        save_name="result_data/confidence_bounds_delta_2.png"
+    )
diff --git a/experiments/iSAGE_paper/explaining_change.py b/experiments/iSAGE_paper/explaining_change.py
new file mode 100644
index 0000000..859458d
--- /dev/null
+++ b/experiments/iSAGE_paper/explaining_change.py
@@ -0,0 +1,148 @@
+import math
+import time
+
+import numpy as np
+import pandas as pd
+from river.datasets.base import REG
+from river.drift import ADWIN
+
+from experiments.setup.data import get_dataset, get_concept_drift_dataset
+from experiments.setup.explainer import get_incremental_sage_explainer, \
+    get_interval_sage_explainer, get_incremental_pfi_explainer
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+DEBUG = True
+
+DATASET_1_NAME = 'elec2'
+
+DATASET_RANDOM_SEED = 1
+SHUFFLE_DATA = False
+N_SAMPLES_1 = None
+
+CONCEPT_DRIFT_SWITCHING_FEATURES = None  # {'salary': 'age'}
+
+MODEL_NAME = 'ARF'
+
+N_INNER_SAMPLES = 1
+
+TRAINING_METRIC_WINDOW = 500
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 25,
+    }
+}
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    dataset = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
+    feature_names = dataset.feature_names
+    n_samples = dataset.n_samples
+    stream = dataset.stream
+
+    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
+    task = dataset.task
+    loss_function = get_loss_function(task=task)
+    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+    model, model_function = get_model(
+        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+    # Get explainers ---------------------------------------------------------------------------------------------------
+    incremental_sage = get_incremental_sage_explainer(
+        feature_names=feature_names,
+        model_function=model_function,
+        loss_function=loss_function,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    # Change Detectors -------------------------------------------------------------------------------------------------
+    fi_detectors = {feature_name: ADWIN(delta=0.001) for feature_name in feature_names}
+    change_points = []
+    change_points_fi = []
+
+    # warm-up because of errors ----------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        model.learn_one(x_i, y_i)
+        if n > 10:
+            break
+
+    # storing ----------------------------------------------------------------------------------------------------------
+    plotter = FeatureImportancePlotter(
+        feature_names=feature_names
+    )
+    model_performance = []
+
+    # run --------------------------------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+
+        # predicting
+        if task == REG:
+            y_i_pred = model.predict_one(x_i)
+        else:
+            y_i_pred = model.predict_proba_one(x_i)
+        rolling_training_metric.update(y_true=y_i, y_pred=y_i_pred)
+        model_performance.append(rolling_training_metric.get())
+
+        # sage inc
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+
+        # storing data
+        plotter.update(fi_values, facet_name='inc')
+
+        # change detection
+        detected_drifts = 0
+        for feature_name in feature_names:
+            fi_detector = fi_detectors[feature_name]
+            fi_detector.update(fi_values[feature_name])
+            if fi_detector.drift_detected:
+                detected_drifts += 1
+
+        if detected_drifts >= 1:
+            change_points.append(n)
+            change_points_fi.append({**incremental_sage.importance_values, 'number_of_drifts': detected_drifts})
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i               {x_i}\n"
+                  f"{n}: inc-sage          {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-var      {incremental_sage.variances}\n"
+                  )
+
+        if n >= n_samples:
+            break
+
+    # Store the results in a database ----------------------------------------------------------------------------------
+
+    #v_lines = [{'x': CONCEPT_DRIFT_POSITION, 'ls': '--', 'c': 'black', 'linewidth': 1}]
+    v_lines = []
+    v_lines.extend([{'x': fi_drift, 'ls': '--', 'c': 'gray', 'linewidth': 1}
+                    for fi_drift in change_points])
+    plotter.plot(
+        figsize=(5, 10),
+        model_performance={"cross\nentropy": model_performance},
+        title=f'Agrawal Stream with Concept Drift',
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['nswprice', 'vicprice', 'nswdemand'],
+        v_lines=v_lines,
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'inc': '-'},
+        legend_style={},
+        markevery={'inc': 10},
+        y_min=-0.1,
+        y_max=0.75
+    )
+
+
+    change_points_fi = pd.DataFrame(change_points_fi)
+    change_points_fi['t'] = change_points
+    change_points_fi.to_csv("result_data/changes_detected.csv", index=False)
diff --git a/experiments/iSAGE_paper/plots/conditional_marginal.py b/experiments/iSAGE_paper/plots/conditional_marginal.py
new file mode 100644
index 0000000..fd9bfd5
--- /dev/null
+++ b/experiments/iSAGE_paper/plots/conditional_marginal.py
@@ -0,0 +1,109 @@
+import copy
+import os
+import pandas as pd
+
+from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
+
+if __name__ == "__main__":
+    SAVE_PATH = "conditional_marginal/conditional_marginal.png"
+
+    # load data --------------------------------------------------------------------------------------------------------
+    DATA_DIR = "conditional_marginal"
+    # left data
+    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
+    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
+    cond_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "cond_fi_values_left.csv"))
+    marg_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "marg_fi_values_left.csv"))
+    # right data
+
+    marginal_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_right.csv"))
+    model_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_right.csv"))
+    cond_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "cond_fi_values_right.csv"))
+    marg_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "marg_fi_values_right.csv"))
+
+    # plot data --------------------------------------------------------------------------------------------------------
+    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
+                     'line_names': ['loss']}
+    top_right_data = {'y_data': {"model_loss": model_loss_right_df, "marginal_loss": marginal_loss_right_df},
+                      'line_names': ['loss']}
+    bot_left_data = {'y_data': {'cond': cond_fi_values_left_df,
+                                'marg': marg_fi_values_left_df},
+                     'line_names': list(cond_fi_values_left_df.columns)}
+    bot_right_data = {'y_data': {'cond': cond_fi_values_right_df,
+                                 'marg': marg_fi_values_right_df},
+                      'line_names': list(cond_fi_values_right_df.columns)}
+
+    # plot styles ------------------------------------------------------------------------------------------------------
+
+    # top styles / performance row
+    top_styles = {
+        "y_min": 0, "y_max": 1,
+        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
+        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+        "fill_between_props": [{
+            'facet_1': 'model_loss', 'facet_2': 'marginal_loss',
+            'line_name_1': 'loss', 'line_name_2': 'loss',
+            'hatch': '///',
+            'color': 'red',
+            'alpha': 0.1,
+        }]
+    }
+    top_right_styles = copy.deepcopy(top_styles)
+    top_left_styles = copy.deepcopy(top_styles)
+    top_left_styles["y_label"] = "loss"
+    top_left_styles["title"] = f"agrawal stream"
+    top_right_styles["title"] = f"elec2 stream"
+    #top_right_styles['y_ticks'] = []
+
+    # bottom styles / feature importance row
+    bot_styles = {
+        "x_label": "Samples",
+        "h_lines": [{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        "line_styles": {'cond': 'solid', 'marg': 'dotted'},
+        "markevery": {'cond': 30, 'marg': 30, },
+    }
+    bot_left_styles = copy.deepcopy(bot_styles)
+    bot_left_styles["y_label"] = "SAGE values"
+    bot_left_styles["legend_style"] = {"fontsize": "small", 'title': 'features'}
+    bot_left_styles["names_to_highlight"] = ['salary', 'commission', 'age']
+    bot_left_styles["y_min"] = -0.04
+    bot_left_styles["y_max"] = 0.19
+    bot_left_styles["y_ticks"] = [0., 0.05, 0.1, 0.15]
+    bot_right_styles = copy.deepcopy(bot_styles)
+    bot_right_styles["y_min"] = -0.09
+    bot_right_styles["y_max"] = 0.598
+    bot_right_styles["y_ticks"] = [0., 0.1, 0.2, 0.3, 0.4, 0.5]
+    bot_right_styles["legend_style"] = {"fontsize": "small", 'title': 'features'}
+    bot_right_styles["names_to_highlight"] = ['nswprice', 'vicprice']
+
+    # legends
+    top_right_styles["secondary_legends"] = [{
+        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", 'title': 'loss', "borderaxespad": 0,
+                         "bbox_to_anchor": (1.02, 1), "frameon": False},
+        "legend_items": [("mean prediction", "dashed", "red"),
+                         ("model", "solid", "red")]
+    }]
+    bot_right_styles["secondary_legends"] = [
+        {
+        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", "title": "distribution", "borderaxespad": 0,
+                         "bbox_to_anchor": (1.02, 1), "frameon": False},
+        "legend_items": [("marg.", "solid", "black"),
+                         ("cond.", "dotted", "black")]
+    }]
+
+    plot_double_stacked_line_plots(
+        figsize=(12.5, 5),
+        top_left_data=top_left_data,
+        top_left_styles=top_left_styles,
+        bot_left_data=bot_left_data,
+        bot_left_styles=bot_left_styles,
+        top_right_data=top_right_data,
+        top_right_styles=top_right_styles,
+        bot_right_data=bot_right_data,
+        bot_right_styles=bot_right_styles,
+        top_portion=0.25,
+        show=True,
+        save_path=SAVE_PATH,
+        different_scales=True,
+        wspace=0.08
+    )
\ No newline at end of file
diff --git a/experiments/iSAGE_paper/plots/confidence_bounds.py b/experiments/iSAGE_paper/plots/confidence_bounds.py
new file mode 100644
index 0000000..e19ea46
--- /dev/null
+++ b/experiments/iSAGE_paper/plots/confidence_bounds.py
@@ -0,0 +1,80 @@
+import copy
+import os
+import pandas as pd
+
+from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots
+
+
+if __name__ == "__main__":
+    DATA_DIR = "confidence_bounds"
+
+    SAVE_PATH = os.path.join(DATA_DIR, "confidence_bounds.png")
+
+    # load data --------------------------------------------------------------------------------------------------------
+    marginal_loss_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_.csv"))
+    model_loss_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_.csv"))
+    sage_fi_values_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_.csv"))
+    confidence_bounds_1_df = pd.read_csv(os.path.join(DATA_DIR, "confidence_bounds_1_.csv"))
+    confidence_bounds_2_df = pd.read_csv(os.path.join(DATA_DIR, "confidence_bounds_2_.csv"))
+
+    # plot data --------------------------------------------------------------------------------------------------------
+    top_left_data = {'y_data': {"model_loss": model_loss_df, "marginal_loss": marginal_loss_df},
+                     'line_names': ['loss']}
+    top_right_data = top_left_data
+    bot_left_data = {'y_data': {'inc': sage_fi_values_df},
+                     'line_names': list(sage_fi_values_df.columns)}
+    bot_right_data = bot_left_data
+
+    # plot styles ------------------------------------------------------------------------------------------------------
+
+    # top styles / performance row
+    top_styles = {
+        "y_min": 0, "y_max": 1,
+        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
+        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+        "fill_between_props": [{'facet_1': 'model_loss', 'facet_2': 'marginal_loss','line_name_1': 'loss', 'line_name_2': 'loss','hatch': '///','color': 'red','alpha': 0.1}]
+    }
+    top_right_styles = copy.copy(top_styles)
+    top_left_styles = copy.copy(top_styles)
+
+    top_left_styles["y_label"] = "loss"
+    top_left_styles["title"] = r"$\delta =$ " + f"{0.1}"
+    top_right_styles["title"] = r"$\delta =$ " + f"{0.05}"
+
+    # bottom styles / feature importance row
+    bot_styles = {
+        "y_min": -0.15, "y_max": 0.79, "x_label": "Samples",
+        "names_to_highlight": ['nswprice', 'vicprice', 'nswdemand'],
+        "h_lines": [{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+    }
+    bot_left_styles = copy.copy(bot_styles)
+    bot_left_styles["std"] = {'inc': confidence_bounds_1_df}
+    bot_left_styles["y_label"] = "SAGE values"
+    bot_right_styles = copy.copy(bot_styles)
+    bot_right_styles["std"] = {'inc': confidence_bounds_2_df}
+
+    # legends
+    top_right_styles["secondary_legends"] = [{
+        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", 'title': 'loss', "borderaxespad": 0,
+                         "bbox_to_anchor": (1.02, 1), "frameon": False},
+        "legend_items": [("mean prediction", "dashed", "red"),
+                         ("model", "solid", "red")]
+    }]
+    bot_right_styles["legend_style"] = {
+        "fontsize": "small", 'title': 'features', "borderaxespad": 0, "frameon": False,
+        "loc": 'upper left', "bbox_to_anchor": (1.02, 1)}
+
+    plot_double_stacked_line_plots(
+        figsize=(12.5, 5),
+        top_left_data=top_left_data,
+        top_left_styles=top_left_styles,
+        bot_left_data=bot_left_data,
+        bot_left_styles=bot_left_styles,
+        top_right_data=top_right_data,
+        top_right_styles=top_right_styles,
+        bot_right_data=bot_right_data,
+        bot_right_styles=bot_right_styles,
+        top_portion=0.25, title="Confidence Bounds for elec2 data stream",
+        show=True,
+        save_path=SAVE_PATH
+    )
diff --git a/experiments/iSAGE_paper/plots/incremental_setting.py b/experiments/iSAGE_paper/plots/incremental_setting.py
new file mode 100644
index 0000000..11311cf
--- /dev/null
+++ b/experiments/iSAGE_paper/plots/incremental_setting.py
@@ -0,0 +1,141 @@
+import copy
+import os
+import pandas as pd
+
+from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
+
+
+if __name__ == "__main__":
+    DATA_DIR = "incremental_setting_5"
+
+    SAVE_PATH = os.path.join(DATA_DIR, "incremental_setting.png")
+    SAVE_PATH_LEFT = os.path.join(DATA_DIR, "incremental_setting_left.png")
+    SAVE_PATH_RIGHT = os.path.join(DATA_DIR, "incremental_setting_right.png")
+
+    # load data --------------------------------------------------------------------------------------------------------
+    # left data
+    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
+    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
+    inc_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_left.csv"))
+    int_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_left.csv"))
+    pfi_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "pfi_fi_values_left.csv"))
+    # right data
+    marginal_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_right.csv"))
+    model_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_right.csv"))
+    inc_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_right.csv"))
+    int_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_right.csv"))
+    inc_welford_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "inc_welford_fi_values_right.csv"))
+
+    # plot data --------------------------------------------------------------------------------------------------------
+    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
+                     'line_names': ['loss']}
+    top_right_data = {'y_data': {"model_loss": model_loss_right_df, "marginal_loss": marginal_loss_right_df},
+                      'line_names': ['loss']}
+    bot_left_data = {'y_data': {'inc': inc_fi_values_left_df,
+                                'int': int_fi_values_left_df,
+                                },
+                     'line_names': list(inc_fi_values_left_df.columns)}
+    bot_right_data = {'y_data': {'inc': inc_fi_values_right_df,
+                                'bat': inc_welford_fi_values_right_df
+                                },
+                      'line_names': list(inc_fi_values_right_df.columns)}
+
+    # plot styles ------------------------------------------------------------------------------------------------------
+
+    # top styles / performance row
+    top_styles = {
+        "y_min": 0, "y_max": 1,
+        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
+        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+        "fill_between_props": [{
+            'facet_1': 'model_loss', 'facet_2': 'marginal_loss',
+            'line_name_1': 'loss', 'line_name_2': 'loss',
+            'hatch': '///',
+            'color': 'red',
+            'alpha': 0.1,
+        }]
+    }
+    top_right_styles = copy.deepcopy(top_styles)
+    top_left_styles = copy.deepcopy(top_styles)
+    top_left_styles["y_label"] = "loss"
+    top_left_styles["title"] = f"iSAGE vs. interval SAGE (agrawal function 2 to 3)"
+    top_left_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
+    top_right_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
+    top_right_styles["title"] = f"iSAGE vs. SAGE (agrawal function 3 to 2)"
+
+    # bottom styles / feature importance row
+    bot_styles = {
+        "y_min": -0.03, "y_max": 0.24, "x_label": "Samples",
+        "names_to_highlight": ['salary', 'commission', 'age', 'elevel'],
+        "h_lines": [{'y': 0., 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}],
+        "line_styles": {'inc': 'solid', 'int': 'dashed', 'bat': 'dashdot', 'pfi': 'dotted'},
+        "markevery": {'inc': 100, 'int': 1, 'bat': 100, 'pfi': 100},
+        "x_ticks": [0, 10_000, 20_000, 30_000, 40_000],
+        "y_ticks": [0, 0.1, 0.2],
+    }
+    bot_left_styles = copy.deepcopy(bot_styles)
+    bot_left_styles["y_label"] = "SAGE values"
+    bot_left_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
+    bot_right_styles = copy.deepcopy(bot_styles)
+    bot_right_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
+
+
+
+
+    # legends
+    top_right_styles["secondary_legends"] = [{
+        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", 'title': 'loss', "borderaxespad": 0,
+                         "bbox_to_anchor": (1.02, 1), "frameon": False},
+        "legend_items": [("mean prediction", "dashed", "red"),
+                         ("model", "solid", "red")]
+    }]
+    bot_right_styles["legend_style"] = {
+        "fontsize": "small", 'title': 'features', "borderaxespad": 0, "frameon": False,
+        "loc": 'upper left', "bbox_to_anchor": (1.02, 1)}
+    bot_right_styles["secondary_legends"] = [{
+        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", "title": "explainer", "borderaxespad": 0,
+                         "bbox_to_anchor": (1.02, 0.6), "frameon": False},
+        "legend_items": [("iSAGE", "solid", "black"),
+                         ("interval SAGE", "dashed", "black"),
+                         #("iPFI", "dotted", 'black'),
+                         ("SAGE", "dashdot", "black")]
+    }]
+
+    plot_double_stacked_line_plots(
+        figsize=(12.5, 5),
+        top_left_data=top_left_data,
+        top_left_styles=top_left_styles,
+        bot_left_data=bot_left_data,
+        bot_left_styles=bot_left_styles,
+        top_right_data=top_right_data,
+        top_right_styles=top_right_styles,
+        bot_right_data=bot_right_data,
+        bot_right_styles=bot_right_styles,
+        top_portion=0.25,
+        show=True,
+        save_path=SAVE_PATH,
+        wspace=0.03
+        #title=r"iSAGE on agrawal streams ($\alpha: 0.001$)"
+    )
+
+    '''
+    plot_stacked_line_plots(
+        figsize=(12.5, 5),
+        top_data=top_left_data,
+        top_styles=top_left_styles,
+        bot_data=bot_left_data,
+        bot_styles=bot_left_styles,
+        show=True,
+        save_path=SAVE_PATH_LEFT
+    )
+
+    plot_stacked_line_plots(
+        figsize=(12.5, 5),
+        top_data=top_right_data,
+        top_styles=top_right_styles,
+        bot_data=bot_right_data,
+        bot_styles=bot_right_styles,
+        show=True,
+        save_path=SAVE_PATH_RIGHT
+    )
+    '''
diff --git a/experiments/iSAGE_paper/plots/reoccurring_drift.py b/experiments/iSAGE_paper/plots/reoccurring_drift.py
new file mode 100644
index 0000000..8145492
--- /dev/null
+++ b/experiments/iSAGE_paper/plots/reoccurring_drift.py
@@ -0,0 +1,98 @@
+import copy
+import os
+import pandas as pd
+
+from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
+
+
+if __name__ == "__main__":
+    DATA_DIR = "reoccurring_drift"
+
+    SAVE_PATH_LEFT = os.path.join(DATA_DIR, "reoccurring_drift.png")
+
+    # load data --------------------------------------------------------------------------------------------------------
+    # left data
+    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
+    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
+    inc_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_left.csv"))
+    int_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_left.csv"))
+    inc_welford_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_welford_fi_values_left.csv"))
+
+
+    # plot data --------------------------------------------------------------------------------------------------------
+    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
+                     'line_names': ['loss']}
+    bot_left_data = {'y_data': {'inc': inc_fi_values_left_df,
+                                'int': int_fi_values_left_df,
+                                'bat': inc_welford_fi_values_left_df
+                                },
+                     'line_names': list(inc_fi_values_left_df.columns)}
+
+    # plot styles ------------------------------------------------------------------------------------------------------
+
+    # top styles / performance row
+    top_styles = {
+        "y_min": 0, "y_max": 1,
+        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
+        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+        "fill_between_props": [{
+            'facet_1': 'model_loss', 'facet_2': 'marginal_loss',
+            'line_name_1': 'loss', 'line_name_2': 'loss',
+            'hatch': '///',
+            'color': 'red',
+            'alpha': 0.1,
+        }]
+    }
+    top_left_styles = copy.deepcopy(top_styles)
+    top_left_styles["y_label"] = "loss"
+    top_left_styles["title"] = f"iSAGE vs. interval SAGE (agrawal function 2 to 3)"
+    top_left_styles["v_lines"] = [{'x': 25_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
+                                  {'x': 50_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
+                                  {'x': 75_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
+
+    # bottom styles / feature importance row
+    bot_styles = {
+        "y_min": -0.03, "y_max": 0.31, "x_label": "Samples",
+        "names_to_highlight": ['salary', 'commission', 'age', 'elevel'],
+        "h_lines": [{'y': 0., 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}],
+        "line_styles": {'inc': 'solid', 'int': 'dashed', 'bat': 'dashdot', 'pfi': 'dotted'},
+        "markevery": {'inc': 100, 'int': 1, 'bat': 100, 'pfi': 100},
+        "x_ticks": [0, 10_000, 20_000, 30_000, 40_000, 50_000, 60_000, 70_000, 80_000, 90_000, 100_000],
+        "y_ticks": [0, 0.1, 0.2],
+    }
+    bot_left_styles = copy.deepcopy(bot_styles)
+    bot_left_styles["y_label"] = "SAGE values"
+    bot_left_styles["v_lines"] = [{'x': 25_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
+                                  {'x': 50_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
+                                  {'x': 75_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
+
+
+    # legends
+    top_left_styles["secondary_legends"] = [{
+        "legend_props": {"loc": 'center right', "ncol": 1, "fontsize": "small", 'title': 'loss'},
+        "legend_items": [("mean prediction", "dashed", "red"),
+                         ("model", "solid", "red")]
+    }]
+    bot_left_styles["legend_style"] = {
+        "fontsize": "small", 'title': 'features', "ncol": 2,
+        "loc": 'upper left', "bbox_to_anchor": (0, 1)}
+    bot_left_styles["secondary_legends"] = [{
+        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", "title": "explainer",
+                         "bbox_to_anchor": (0.18, 1)},
+        "legend_items": [("iSAGE", "solid", "black"),
+                         ("interval SAGE", "dashed", "black"),
+                         #("iPFI", "dotted", 'black'),
+                         ("SAGE", "dashdot", "black")]
+    }]
+
+
+    plot_stacked_line_plots(
+        figsize=(12.5, 5),
+        top_data=top_left_data,
+        top_styles=top_left_styles,
+        bot_data=bot_left_data,
+        bot_styles=bot_left_styles,
+        show=True,
+        save_path=SAVE_PATH_LEFT,
+        top_portion=0.25
+    )
diff --git a/experiments/iSAGE_paper/plots/static_sage_in_dynamic_setting.py b/experiments/iSAGE_paper/plots/static_sage_in_dynamic_setting.py
new file mode 100644
index 0000000..aa553df
--- /dev/null
+++ b/experiments/iSAGE_paper/plots/static_sage_in_dynamic_setting.py
@@ -0,0 +1,78 @@
+import copy
+import os
+import pandas as pd
+
+from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
+
+
+if __name__ == "__main__":
+    DATA_DIR = "static_sage_in_dynamic_setting"
+
+    SAVE_PATH = os.path.join(DATA_DIR, "static_sage_in_dynamic_setting.png")
+
+    # load data --------------------------------------------------------------------------------------------------------
+    # left data
+    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
+    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
+    inc_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_left.csv"))
+    int_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_left.csv"))
+    inc_welford_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_welford_fi_values_left.csv"))
+
+    # plot data --------------------------------------------------------------------------------------------------------
+    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
+                     'line_names': ['loss']}
+    bot_left_data = {'y_data': {'inc': inc_fi_values_left_df,
+                                #'int': int_fi_values_left_df,
+                                'bat': inc_welford_fi_values_left_df},
+                     'line_names': list(inc_fi_values_left_df.columns)}
+
+    # plot styles ------------------------------------------------------------------------------------------------------
+
+    # top styles / performance row
+    top_styles = {
+        "y_min": 0, "y_max": 1,
+        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
+        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+        "fill_between_props": [
+            {'facet_1': 'model_loss', 'facet_2': 'marginal_loss', 'line_name_1': 'loss', 'line_name_2': 'loss',
+             'hatch': '///', 'color': 'red', 'alpha': 0.1}]
+    }
+    top_left_styles = copy.deepcopy(top_styles)
+    top_left_styles["y_label"] = "loss"
+    top_left_styles["v_lines"] = [{'x': 20_000, 'ls': '--', 'c': 'grey', 'linewidth': 1}]
+    top_left_styles["secondary_legends"] = [{
+        "legend_props": {"loc": "upper left", "ncol": 1, "fontsize": "small"},
+        "legend_items": [("mean prediction loss", "dashed", "red"), ("model loss", "solid", "red")]
+    }]
+    top_left_styles["y_ticks"] = [0, 0.5, 1.0]
+
+    # bottom styles / feature importance row
+    bot_styles = {
+        "y_min": -0.0499, "y_max": 0.29, "x_label": "Samples",
+        "names_to_highlight": ['salary', 'commission', 'age', 'elevel'],
+        "h_lines": [{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        "line_styles": {'inc': 'solid', 'int': 'dashed', 'bat': 'dashdot'},
+        "markevery": {'inc': 100, 'int': 1, 'bat': 100},
+        "x_ticks": [0, 5_000, 10_000,  15_000, 20_000, 25_000, 30_000, 35_000, 40_000]
+    }
+    bot_left_styles = copy.deepcopy(bot_styles)
+    bot_left_styles["y_label"] = "SAGE values"
+    bot_left_styles["v_lines"] = [{'x': 20_000, 'ls': '--', 'c': 'grey', 'linewidth': 1}]
+    bot_left_styles["legend_style"] = {"fontsize": "small", 'title': 'features', "loc": "upper left"}
+    bot_left_styles['facet_not_to_highlight'] = ['int']
+    bot_left_styles["y_ticks"] = [0, 0.1, 0.2]
+    bot_left_styles["secondary_legends"] = [{
+        "legend_props": {"loc": "upper left", "ncol": 1, "fontsize": "small", "bbox_to_anchor": (0.1, 1), "title": "explainer"},
+        "legend_items": [("iSAGE", "solid", "black"), ("SAGE", "dashdot", "black")]
+    }]
+
+    plot_stacked_line_plots(
+        figsize=(12.5, 5),
+        top_data=top_left_data,
+        top_styles=top_left_styles,
+        bot_data=bot_left_data,
+        bot_styles=bot_left_styles,
+        show=True, top_portion=0.2,
+        save_path=SAVE_PATH
+    )
+
diff --git a/experiments/iSAGE_paper/reoccurring_concept_drift.py b/experiments/iSAGE_paper/reoccurring_concept_drift.py
new file mode 100644
index 0000000..66577f9
--- /dev/null
+++ b/experiments/iSAGE_paper/reoccurring_concept_drift.py
@@ -0,0 +1,324 @@
+import math
+import time
+
+import numpy as np
+import pandas as pd
+from river.datasets.base import REG
+
+from experiments.setup.data import get_dataset, get_concept_drift_dataset
+from experiments.setup.explainer import get_incremental_sage_explainer, \
+    get_interval_sage_explainer, get_incremental_pfi_explainer, get_imputer_and_storage
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IntervalSageExplainer, IncrementalSageExplainer
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+DEBUG = True
+
+DATASET_1_NAME = 'agrawal 1'
+DATASET_2_NAME = 'agrawal 2'
+DATASET_3_NAME = 'agrawal 1'
+DATASET_4_NAME = 'agrawal 2'
+DATASET_RANDOM_SEED = 1
+SHUFFLE_DATA = False
+N_SAMPLES = 25000
+
+MODEL_NAME = 'ARF'
+
+N_INNER_SAMPLES: int = 4
+N_INTERVAL_LENGTH: int = 5000
+CONFIDENCE_BOUNDS_DELTA: float = 0.1
+SMOOTHING_ALPHA: float = 0.001
+FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
+RESERVOIR_LENGTH: int = 100
+RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
+
+TRAINING_METRIC_WINDOW = 500
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 15,
+    }
+}
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    dataset = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=10)
+    dataset_1 = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
+    dataset_2 = get_dataset(
+        dataset_name=DATASET_2_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
+    dataset_3 = get_dataset(
+        dataset_name=DATASET_3_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
+    dataset_4 = get_dataset(
+        dataset_name=DATASET_4_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
+    feature_names = dataset.feature_names
+    cat_feature_names = dataset.cat_feature_names
+    num_feature_names = dataset.num_feature_names
+    stream_0 = dataset
+    n_samples = N_SAMPLES * 4
+    stream_1 = dataset_1.stream
+    stream_2 = dataset_2.stream
+    stream_3 = dataset_3.stream
+    stream_4 = dataset_4.stream
+
+    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
+    task = dataset_1.task
+    loss_function = get_loss_function(task=task)
+    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+    model, model_function = get_model(
+        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    imputer, storage = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
+        reservoir_kind=RESERVOIR_KIND,
+        reservoir_length=RESERVOIR_LENGTH,
+        cat_feature_names=cat_feature_names,
+        num_feature_names=num_feature_names
+    )
+
+    incremental_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    incremental_welford_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        dynamic_setting=False,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    interval_explainer = IntervalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        feature_names=feature_names,
+        n_inner_samples=N_INNER_SAMPLES,
+        interval_length=N_INTERVAL_LENGTH
+    )
+
+    # warm-up because of errors ----------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream_0, start=1):
+        model.learn_one(x_i, y_i)
+
+
+    time_sage = 0.
+    time_interval = 0.
+    time_learning = 0.
+
+    sage_fi_values = []
+    sage_welford_fi_values = []
+    interval_fi_values = []
+
+    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    model_loss = []
+    marginal_loss = []
+
+    for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
+        sage_welford_fi_values.append(fi_values)
+
+        # sage inc
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+        sage_fi_values.append(fi_values)
+
+        # sage int
+        fi_values = interval_explainer.explain_one(x_i, y_i)
+        interval_fi_values.append(fi_values)
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
+                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
+                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
+                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
+                  )
+
+        if n >= N_SAMPLES:
+            break
+
+    for (n, (x_i, y_i)) in enumerate(stream_2, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
+        sage_welford_fi_values.append(fi_values)
+
+        # sage inc
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+        sage_fi_values.append(fi_values)
+
+        # sage int
+        fi_values = interval_explainer.explain_one(x_i, y_i)
+        interval_fi_values.append(fi_values)
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
+                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
+                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
+                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
+                  )
+
+        if n >= N_SAMPLES:
+            break
+
+    for (n, (x_i, y_i)) in enumerate(stream_3, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
+        sage_welford_fi_values.append(fi_values)
+
+        # sage inc
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+        sage_fi_values.append(fi_values)
+
+        # sage int
+        fi_values = interval_explainer.explain_one(x_i, y_i)
+        interval_fi_values.append(fi_values)
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
+                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
+                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
+                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
+                  )
+
+        if n >= N_SAMPLES:
+            break
+
+    for (n, (x_i, y_i)) in enumerate(stream_4, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
+        sage_welford_fi_values.append(fi_values)
+
+        # sage inc
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+        sage_fi_values.append(fi_values)
+
+        # sage int
+        fi_values = interval_explainer.explain_one(x_i, y_i)
+        interval_fi_values.append(fi_values)
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
+                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
+                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
+                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
+                  )
+
+        if n >= N_SAMPLES:
+            break
+
+    # Store the results in a database ----------------------------------------------------------------------------------
+
+    facet = "left.csv"
+    data_folder = "reoccurring_drift"
+
+    name = "inc_fi_values"
+    df = pd.DataFrame(sage_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "inc_welford_fi_values"
+    df = pd.DataFrame(sage_welford_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "int_fi_values"
+    df = pd.DataFrame(interval_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "model_loss"
+    df = pd.DataFrame(model_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "marginal_loss"
+    df = pd.DataFrame(marginal_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
\ No newline at end of file
diff --git a/experiments/iSAGE_paper/sampling_procedures_check.py b/experiments/iSAGE_paper/sampling_procedures_check.py
new file mode 100644
index 0000000..ca9f6b5
--- /dev/null
+++ b/experiments/iSAGE_paper/sampling_procedures_check.py
@@ -0,0 +1,259 @@
+import random
+
+import numpy as np
+import pandas as pd
+import river
+from matplotlib import pyplot as plt
+
+from data.stream.synth import Agrawal
+from experiments.setup.loss import get_loss_function
+from increment_explain.imputer import MarginalImputer
+from increment_explain.imputer.tree_imputer import TreeImputer
+from increment_explain.storage import GeometricReservoirStorage
+from increment_explain.storage.tree_storage import TreeStorage
+from river.datasets.base import REG
+
+from scipy.stats import multivariate_normal as mvn
+
+
+def model_function_placeholder(x_i):
+    return 1.0
+
+
+def get_covariance_stream(size):
+
+    mvn.rvs(mean=[0., 0.], cov=[[1, 0.8], [0.8, 1]], size=5000)
+
+    cov = np.array([[1,0.8,0,0],[0.8,1,0,0],[0,0,1,0],[0,0,0,1]])
+    x_data = mvn.rvs(mean=[0.,0.,0.,0.], cov=cov, size=size)
+    feature_names = ['_'.join(('N', str(i))) for i in range(1, 5)]
+    x_data = pd.DataFrame(x_data, columns=feature_names)
+    x_data['C_1'] = np.array([random.randint(1, 5) for _ in range(size)])
+    y_data = pd.DataFrame(np.array([1 for _ in range(size)]))
+    stream = river.stream.iter_pandas(x_data, y_data, feature_names=feature_names)
+    return stream, feature_names
+
+
+if __name__ == "__main__":
+
+    RANDOM_SEED = 1
+
+    # Model and training setup
+    model_function = model_function_placeholder
+    loss_function = get_loss_function(task=REG)
+
+    sample_from_leaf_reservoirs = True  # let the model predict (False) or sample from leaf reservoirs (True)
+
+    n_samples_agrawal = 5000
+    n_samples_covariance_test = 5000
+
+    # agrawal Test -----------------------------------------------------------------------------------------------------
+    dataset_1 = Agrawal(classification_function=0, random_seed=RANDOM_SEED, n_samples=n_samples_agrawal)
+    n_samples = n_samples_agrawal
+    stream_1 = dataset_1.stream
+    feature_names = dataset_1.feature_names
+    cat_feature_names = dataset_1.cat_feature_names
+    num_feature_names = dataset_1.num_feature_names
+
+    # init imputer and storage
+    storage_tree = TreeStorage(cat_feature_names=cat_feature_names, num_feature_names=num_feature_names)
+    imputer_cond = TreeImputer(model_function, storage_object=storage_tree, direct_predict_numeric=True, use_storage=True)
+    storage_reservoir = GeometricReservoirStorage(store_targets=False, size=100, constant_probability=1.0)
+    imputer_marg = MarginalImputer(model_function, storage_object=storage_reservoir, sampling_strategy='joint')
+
+    x_data_agrawal = []
+
+    commission_sampled_cond = []
+    commission_sampled_marg = []
+
+    age_sampled_cond = []
+    age_sampled_marg = []
+
+    performances_agrawal = []
+
+    for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
+        storage_tree.update(x_i, y_i)
+        storage_reservoir.update(x_i, y_i)
+
+        x_data_agrawal.append(x_i)
+
+        if sample_from_leaf_reservoirs:
+            commission_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='commission', x_i=x_i))
+            age_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='age', x_i=x_i))
+        else:
+            commission_sampled_cond.append(imputer_cond._sample(feature_name='commission', x_i=x_i))
+            age_sampled_cond.append(imputer_cond._sample(feature_name='age', x_i=x_i))
+
+        commission_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['commission'])['commission'])
+        age_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['age'])['age'])
+
+        performance = {}
+        for feature_name, model in storage_tree._storage_x.items():
+            performance[feature_name] = storage_tree.performances[feature_name].get()
+        performances_agrawal.append(performance)
+
+        if n >= n_samples:
+            break
+
+    # Covariance Test --------------------------------------------------------------------------------------------------
+    stream_1, feature_names = get_covariance_stream(size=n_samples_covariance_test)
+    n_samples = n_samples_covariance_test
+    feature_names = feature_names
+    cat_feature_names = ['C_1']
+    num_feature_names = ['N_1', 'N_2', 'N_3', 'N_4']
+
+    # init imputer
+    storage_tree = TreeStorage(cat_feature_names=cat_feature_names, num_feature_names=num_feature_names)
+    imputer_cond = TreeImputer(model_function, storage_object=storage_tree, direct_predict_numeric=True, use_storage=True)
+    storage_reservoir = GeometricReservoirStorage(store_targets=False, size=100, constant_probability=1.0)
+    imputer_marg = MarginalImputer(model_function, storage_object=storage_reservoir, sampling_strategy='joint')
+
+    x_data_covariance_test = []
+
+    C_1_sampled_cond = []
+    C_1_sampled_marg = []
+
+    N_1_sampled_cond = []
+    N_1_sampled_marg = []
+
+    performances_covariance_test = []
+
+    # Concept 1 ----------------------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
+        storage_tree.update(x_i, y_i)
+        storage_reservoir.update(x_i, y_i)
+
+        x_data_covariance_test.append(x_i)
+
+        if sample_from_leaf_reservoirs:
+            C_1_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='C_1', x_i=x_i))
+            N_1_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='N_1', x_i=x_i))
+        else:
+            C_1_sampled_cond.append(imputer_cond._sample(feature_name='C_1', x_i=x_i))
+            N_1_sampled_cond.append(imputer_cond._sample(feature_name='N_1', x_i=x_i))
+
+        C_1_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['C_1'])['C_1'])
+        N_1_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['N_1'])['N_1'])
+
+        performance = {}
+        for feature_name, model in storage_tree._storage_x.items():
+            performance[feature_name] = storage_tree.performances[feature_name].get()
+        performances_covariance_test.append(performance)
+
+        if n >= n_samples:
+            break
+
+    # plotting ---------------------------------------------------------------------------------------------------------
+    fig_size_vertical = 3
+    fig_size_horizontal = 9
+
+    # plot agrawal -----------------------------------------------------------------------------------------------------
+    markevery_step = 30
+    markevery_ground_truth_step = 15
+
+    alpha_hue = np.asarray([i for i in range(1, n_samples + 1)]) / n_samples
+    alpha_hue = alpha_hue[::markevery_step]
+
+    performances_agrawal = pd.DataFrame(performances_agrawal)
+    x_data_agrawal = pd.DataFrame(x_data_agrawal)
+
+    fig, axis = plt.subplots(figsize=(fig_size_horizontal*2/3, fig_size_vertical), nrows=1, ncols=2, sharey="all")
+    axis_1 = axis[0]
+    axis_2 = axis[1]
+
+    axis_1.scatter(
+        x_data_agrawal['commission'].loc[::markevery_ground_truth_step],
+        x_data_agrawal['salary'].loc[::markevery_ground_truth_step],
+        marker="o", c="black", alpha=alpha_hue
+    )
+    axis_1.scatter(
+        commission_sampled_marg[::markevery_step],
+        x_data_agrawal['salary'].loc[::markevery_step],
+        marker="+", c="blue", alpha=alpha_hue
+    )
+    axis_1.scatter(
+        commission_sampled_cond[::markevery_step],
+        x_data_agrawal['salary'].loc[::markevery_step],
+        marker="x", c="red", alpha=alpha_hue
+    )
+    axis_1.axis(xmin=-5_000, xmax=90_000, ymin=15_000, ymax=155_000)
+    axis_1.set_xlabel(r"$X_{commission}$")
+    axis_1.set_ylabel(r"$X_{salary}$")
+    axis_1.set_yticks([25_000, 50_000, 75_000, 100_000, 125_000, 150_000])
+    axis_1.set_yticklabels(["25k", "50k", "75k", "100k", "125k", "150k"])
+    axis_1.set_xticks([0, 20_000, 40_000, 60_000, 80_000])
+    axis_1.set_xticklabels(["0", "20k", "40k", "60k", "80k"])
+    axis_1.set_title(r"dependent")
+
+    axis_2.scatter(
+        x_data_agrawal['age'].loc[::markevery_ground_truth_step],
+        x_data_agrawal['salary'].loc[::markevery_ground_truth_step],
+        marker="o", c="black", alpha=alpha_hue
+    )
+    axis_2.scatter(
+        age_sampled_marg[::markevery_step],
+        x_data_agrawal['salary'].loc[::markevery_step],
+        marker="+", c="blue", alpha=alpha_hue
+    )
+    axis_2.scatter(
+        age_sampled_cond[::markevery_step],
+        x_data_agrawal['salary'].loc[::markevery_step],
+        marker="x", c="red", alpha=alpha_hue
+    )
+    axis_2.axis(xmin=5, xmax=85)
+    axis_2.set_xlabel(r"$X_{age}$")
+    axis_2.set_xlim([15, 85])
+    axis_2.set_xticks([20, 30, 40, 50, 60, 70, 80])
+    axis_2.set_title(r"independent")
+
+    plt.tight_layout()
+    plt.subplots_adjust(wspace=0, hspace=0, left=0.12, right=0.995)
+    #plt.savefig("agrawal_conditonal_vs_marginal.png", dpi=200)
+    plt.show()
+
+    # plot covariance test ---------------------------------------------------------------------------------------------
+    performances_covariance_test = pd.DataFrame(performances_covariance_test)
+    x_data_covariance_test = pd.DataFrame(x_data_covariance_test)
+
+    markevery_step = 30
+    markevery_ground_truth_step = 15
+
+    alpha_hue = np.asarray([i for i in range(1, n_samples + 1)]) / n_samples
+    alpha_hue = alpha_hue[::markevery_step]
+
+    fig_2, axes_2 = plt.subplots(figsize=(fig_size_horizontal*1/3, fig_size_vertical), nrows=1, ncols=1)
+    axis_3 = axes_2
+
+    axis_3.scatter(
+        x_data_covariance_test['N_1'].loc[::markevery_ground_truth_step],
+        x_data_covariance_test['N_2'].loc[::markevery_ground_truth_step],
+        marker="o", c="black", alpha=alpha_hue
+    )
+    axis_3.scatter(
+        N_1_sampled_marg[::markevery_step],
+        x_data_covariance_test['N_2'].loc[::markevery_step],
+        marker="+", c="blue", alpha=alpha_hue
+    )
+    axis_3.scatter(
+        N_1_sampled_cond[::markevery_step],
+        x_data_covariance_test['N_2'].loc[::markevery_step],
+        marker="x", c="red", alpha=alpha_hue
+    )
+    axis_3.axis(xmin=-3.8, xmax=3.8, ymin=-4.8, ymax=3.8)
+    axis_3.set_xlabel(r"$X_{2}$")
+    axis_3.set_ylabel(r"$X_{1}$")
+    axis_3.set_xticks([-3, -2, -1, 0, 1, 2, 3])
+    axis_3.set_yticks([-3, -2, -1, 0, 1, 2, 3])
+
+    axis_3.scatter([], [], marker="o", c="black", label="data")
+    axis_3.scatter([], [], marker="+", c="blue", label="marginal")
+    axis_3.scatter([], [], marker="x", c="red", label="conditional")
+    axis_3.legend(loc="lower center", fontsize="small", frameon=True, ncol=3, columnspacing=0.5, handletextpad=0.1)
+
+    plt.title(r"$\rho_{X_{1}, X_{2}} = 0.8$")
+
+    plt.tight_layout()
+    plt.subplots_adjust(wspace=0, hspace=0, left=0.18, right=0.99)
+
+    #plt.savefig("covariance_conditonal_vs_marginal.png", dpi=200)
+    plt.show()
diff --git a/experiments/iSAGE_paper/static_sage_in_dynamic_setting.py b/experiments/iSAGE_paper/static_sage_in_dynamic_setting.py
new file mode 100644
index 0000000..024e4ff
--- /dev/null
+++ b/experiments/iSAGE_paper/static_sage_in_dynamic_setting.py
@@ -0,0 +1,238 @@
+import math
+import time
+
+import numpy as np
+import pandas as pd
+from river.datasets.base import REG
+
+from experiments.setup.data import get_dataset, get_concept_drift_dataset
+from experiments.setup.explainer import get_incremental_sage_explainer, \
+    get_interval_sage_explainer, get_incremental_pfi_explainer, get_imputer_and_storage
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IntervalSageExplainer, IncrementalSageExplainer
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+DEBUG = True
+
+DATASET_1_NAME = 'agrawal 2'
+DATASET_2_NAME = 'agrawal 1'
+DATASET_RANDOM_SEED = 1
+SHUFFLE_DATA = False
+N_SAMPLES_1 = 20000
+N_SAMPLES_2 = 20000
+
+CONCEPT_DRIFT_POSITION = 20000
+CONCEPT_DRIFT_WIDTH = 1
+CONCEPT_DRIFT_SWITCHING_FEATURES = None # {'salary': 'age'}
+
+MODEL_NAME = 'ARF'
+
+N_INNER_SAMPLES: int = 5
+N_INTERVAL_LENGTH: int = 5000
+CONFIDENCE_BOUNDS_DELTA: float = 0.1
+SMOOTHING_ALPHA: float = 0.001
+FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
+RESERVOIR_LENGTH: int = 100
+RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
+
+TRAINING_METRIC_WINDOW = 500
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 25,
+    }
+}
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    dataset_1 = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
+    dataset_2 = get_dataset(
+        dataset_name=DATASET_2_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_2)
+    dataset = get_concept_drift_dataset(
+        dataset_1_name=DATASET_1_NAME, dataset_2_name=DATASET_2_NAME,
+        dataset_1=dataset_1, dataset_2=dataset_2,
+        position=CONCEPT_DRIFT_POSITION, width=CONCEPT_DRIFT_WIDTH,
+        features_to_switch=CONCEPT_DRIFT_SWITCHING_FEATURES,
+    )
+    feature_names = dataset.feature_names
+    cat_feature_names = dataset.cat_feature_names
+    num_feature_names = dataset.num_feature_names
+    n_samples = dataset.n_samples
+    stream = dataset.stream
+
+    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
+    task = dataset.task
+    loss_function = get_loss_function(task=task)
+    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+    model, model_function = get_model(
+        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    imputer, storage = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
+        reservoir_kind=RESERVOIR_KIND,
+        reservoir_length=RESERVOIR_LENGTH,
+        cat_feature_names=cat_feature_names,
+        num_feature_names=num_feature_names
+    )
+
+    incremental_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    incremental_welford_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        dynamic_setting=False,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    interval_explainer = IntervalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        feature_names=feature_names,
+        n_inner_samples=N_INNER_SAMPLES,
+        interval_length=N_INTERVAL_LENGTH
+    )
+
+    # warm-up because of errors ----------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        model.learn_one(x_i, y_i)
+        if n > 10:
+            break
+
+    plotter = FeatureImportancePlotter(
+        feature_names=feature_names
+    )
+
+    time_sage = 0.
+    time_interval = 0.
+    time_learning = 0.
+
+    sage_fi_values = []
+    sage_welford_fi_values = []
+    interval_fi_values = []
+
+    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    model_loss = []
+    marginal_loss = []
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        start_time = time.time()
+        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
+        time_sage += time.time() - start_time
+        sage_welford_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='inc-w')
+
+        # sage inc
+        start_time = time.time()
+        fi_values = incremental_sage.explain_one(x_i, y_i)
+        time_sage += time.time() - start_time
+        sage_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='inc')
+
+        # sage int
+        start_time = time.time()
+        fi_values = interval_explainer.explain_one(x_i, y_i)
+        time_interval += time.time() - start_time
+        interval_fi_values.append(fi_values)
+        plotter.update(fi_values, facet_name='int')
+
+        # learning
+        start_time = time.time()
+        model.learn_one(x_i, y_i)
+        time_learning += time.time() - start_time
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
+                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
+                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
+                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
+                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
+                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
+                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
+                  )
+
+        if n >= n_samples:
+            break
+
+    # Store the results in a database ----------------------------------------------------------------------------------
+
+    facet = "right.csv"
+    data_folder = "incremental_setting_5"
+
+    name = "inc_fi_values"
+    df = pd.DataFrame(sage_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "inc_welford_fi_values"
+    df = pd.DataFrame(sage_welford_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "int_fi_values"
+    df = pd.DataFrame(interval_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "model_loss"
+    df = pd.DataFrame(model_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "marginal_loss"
+    df = pd.DataFrame(marginal_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
+                      "line_names": ["loss"], "names_to_highlight": ['loss'],
+                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+                      "markevery": {"model_loss": 100, "marginal_loss": 100},
+                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
+                                 "legend_items": [("marginal loss", "dashed", "red"),
+                                                  ("model loss", "solid", "red")]}}
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
+        performance_kw=performance_kw,
+        title=f'Agrawal Stream with Concept Drift',
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
+        v_lines=[{'x': CONCEPT_DRIFT_POSITION, 'ls': '--', 'c': 'black', 'linewidth': 1}],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'inc': '-', 'int': '--', 'inc-w': 'dotted'},
+        legend_style={"fontsize": "small"},
+        markevery={'inc': 10, 'int': 1, 'inc-w': 10},
+        y_min=-0.05,
+        y_max=0.42,
+    )
diff --git a/experiments/network_ideas/link_prediction.py b/experiments/network_ideas/link_prediction.py
new file mode 100644
index 0000000..0fa32c2
--- /dev/null
+++ b/experiments/network_ideas/link_prediction.py
@@ -0,0 +1,353 @@
+import math
+import random
+
+import river.datasets.synth.concept_drift_stream
+from river.metrics import Accuracy
+from river.utils import Rolling
+from river.ensemble import AdaptiveRandomForestClassifier
+from river.tree import HoeffdingTreeClassifier
+from river.linear_model import LogisticRegression
+from river import neural_net as nn
+from river.neighbors import KNNClassifier
+from matplotlib import pyplot as plt
+from river.datasets.base import BINARY_CLF
+
+from data.stream._base import BatchStream
+from experiments.setup.explainer import get_imputer_and_storage
+from experiments.setup.loss import get_loss_function
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IncrementalSageExplainer, BatchSageExplainer
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+N_NODE_FEATURES = 1
+
+
+def _get_label_function_1_old(node_1, node_2):
+    if node_1['node_1_interest_1'] >= 0.5 and node_2['node_2_interest_1'] >= 0.5:
+        return 1
+    if node_1['node_1_interest_2'] >= 0.5 and node_2['node_2_interest_2'] >= 0.5:
+        return 1
+    return 0
+
+
+def _get_label_function_2_old(node_1, node_2):
+    if node_1['node_1_interest_2'] >= 0.6 and node_2['node_2_interest_2'] >= 0.6:
+        return 1
+    if node_1['node_1_interest_3'] >= 0.5 and node_2['node_2_interest_3'] >= 0.5:
+        return 1
+    return 0
+
+
+def _get_label_function_1(node_1, node_2):
+    alpha = 30
+    beta = 25
+    gamma = 10
+    z_1 = alpha * (node_1['node_1_interest_1'] * node_2['node_2_interest_1'])
+    z_2 = beta * (node_1['node_1_interest_2'] * node_2['node_2_interest_2'])
+    z_3 = gamma * node_1['graph_connectivity']
+    p = 1 / (1 + math.exp(-(z_1 + z_2 + z_3)))
+    random_number = random.random()
+    if random_number < p:
+        return 1, p
+    return 0, p
+
+
+def _get_label_function_2(node_1, node_2):
+    alpha = 30
+    beta = 25
+    gamma = 10
+    z_1 = alpha * (node_1['node_1_interest_2'] * node_2['node_2_interest_2'])
+    z_2 = beta * (node_1['node_1_interest_3'] * node_2['node_2_interest_3'])
+    z_3 = gamma * node_1['graph_connectivity']
+    p = 1 / (1 + math.exp(-(z_1 + z_2 + z_3)))
+    random_number = random.random()
+    #print(int(random_number < p), p, node_1['node_1_interest_1'], node_2['node_2_interest_1'])
+    #print(int(random_number < p), p, node_1['node_1_interest_3'], node_2['node_2_interest_3'])
+    if random_number < p:
+        return 1, p
+    return 0, p
+
+
+def get_sample(classification_function: int = 1, n_interests=3):
+    node_1 = {'node_1_interest_' + str(i): random.uniform(-1, 1) for i in range(1, n_interests + 1)}
+    node_1['node_1_att_1'] = random.randint(1, 3)
+    node_1['graph_connectivity'] = random.uniform(-1, 1)
+    node_1['node_1_att_2'] = random.randint(1, 3)
+    node_2 = {'node_2_interest_' + str(i): random.uniform(-1, 1) for i in range(1, n_interests + 1)}
+    node_2['node_2_att_1'] = random.randint(1, 3)
+
+    #node_1.update({'interest_1': node_1['node_1_interest_1'] * node_2['node_2_interest_1'],'interest_2': node_1['node_1_interest_2'] * node_2['node_2_interest_2'],'interest_3': node_1['node_1_interest_3'] * node_2['node_2_interest_3']})
+    #node_1['interest_1_interaction'] = node_1['node_1_interest_1'] * node_2['node_2_interest_1']
+    #node_1['interest_2_interaction'] = node_1['node_1_interest_2'] * node_2['node_2_interest_2']
+    #node_1['interest_2_interaction'] = node_1['node_1_interest_3'] * node_2['node_2_interest_3']
+
+
+    if classification_function == 1:
+        y_i, p_i = _get_label_function_1(node_1, node_2)
+    elif classification_function == 2:
+        y_i, p_i = _get_label_function_2(node_1, node_2)
+    else:
+        raise NotImplementedError("Only classificaiton function 1 and 2 implemented.")
+
+    x_i = {**node_1, **node_2}
+
+    return x_i, y_i
+
+def print_probas(n=10_000, n_interests=3):
+    ys = []
+    ps = []
+    for i in range(n):
+        node_1 = {'node_1_interest_' + str(i): random.uniform(-0.5, 0.5) for i in range(1, n_interests + 1)}
+        node_1['node_1_att_1'] = random.randint(1, 3)
+        node_2 = {'node_2_interest_' + str(i): random.uniform(-0.5, 0.5) for i in range(1, n_interests + 1)}
+        node_2['node_2_att_1'] = random.randint(1, 3)
+        y_i, p_i = _get_label_function_1(node_1, node_2)
+        ys.append(y_i)
+        ps.append(p_i)
+
+    plt.hist(ys)
+    plt.title("Target")
+    plt.show()
+
+    plt.hist(ps)
+    plt.title("Probability")
+    plt.show()
+
+
+def easy_problem(n=10_000, alpha=5, beta=5):
+    ys = []
+    ps = []
+    xs = []
+    x_1 = []
+    x_2 = []
+    dataset = []
+    alpha = alpha
+    for i in range(n):
+        node_1_interest_1 = random.uniform(-1, 1)
+        node_2_interest_1 = random.uniform(-1, 1)
+        interest_1 = node_1_interest_1 * node_2_interest_1
+        node_1_interest_2 = random.uniform(-1, 1)
+        node_2_interest_2 = random.uniform(-1, 1)
+        interest_2 = node_1_interest_2 * node_2_interest_2
+        p = 1 / (1 + math.exp(-(alpha * interest_1 + beta * interest_2)))
+        xs.append(interest_1)
+        x_1.append(node_1_interest_1)
+        x_2.append(node_2_interest_1)
+        ps.append(p)
+        y = 0
+        if random.random() <= p:
+            y = 1
+        ys.append(y)
+        dataset.append({"node_1_interest_1": node_1_interest_1, "node_2_interest_1": node_2_interest_1, #"interest_1": interest_1,
+                        "node_1_interest_2": node_1_interest_2, "node_2_interest_2": node_2_interest_2, #"interest_2": interest_2
+                        })
+
+    plt.scatter(xs, ys)
+    plt.title(f"Label, alpha: {alpha}")
+    plt.xlabel("Feature 1 * Feature 2")
+    plt.show()
+
+    plt.scatter(xs, ps)
+    plt.title(f"Probability, alpha: {alpha}")
+    plt.xlabel("Feature 1 * Feature 2")
+    plt.show()
+
+    plt.scatter(x_1, ys)
+    plt.title(f"Label, alpha: {alpha}")
+    plt.xlabel("Feature 1")
+    plt.show()
+
+    plt.scatter(x_1, ps)
+    plt.title(f"Probability, alpha: {alpha}")
+    plt.xlabel("Feature 1")
+    plt.show()
+
+    return dataset, ys
+
+
+
+if __name__ == "__main__":
+    import sys
+
+
+
+    #print_probas(n=10_000)
+
+    N_SAMPLES = 50_000
+
+    #dataset, ys = easy_problem(n=N_SAMPLES, alpha=30, beta=25)
+    #stream = zip(dataset, ys)
+    #feature_names = list(dataset[0].keys())
+
+    DRIFT_POSITION = int(N_SAMPLES * 0.5)
+    DRIFT_WIDTH = int(N_SAMPLES * 0.2)
+
+    x_i, _ = get_sample(classification_function=1)
+    feature_names = list(x_i.keys())
+
+    model = AdaptiveRandomForestClassifier(n_models=10, binary_split=True, max_depth=10)
+    #model = HoeffdingTreeClassifier(max_depth=10)
+    #model = LogisticRegression()
+
+
+    model_function = RiverToPredictionFunction(
+        model,
+        classification=True,
+        feature_names=feature_names
+    ).predict
+
+    loss_function = get_loss_function(task=BINARY_CLF)
+
+    imputer, storage = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution='marginal joint',
+        reservoir_kind='geometric',
+        reservoir_length=1000,
+        cat_feature_names=[],
+        num_feature_names=feature_names
+    )
+
+    incremental_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=5
+    )
+
+    incremental_pfi = IncrementalPFI(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_samples=5
+    )
+
+    metric = Accuracy()
+    metric = Rolling(metric, window_size=1000)
+
+    plotter = FeatureImportancePlotter(
+        feature_names=feature_names
+    )
+
+    model_loss_tracker = ExponentialSmoothingTracker(alpha=0.001)
+    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=0.001)
+    model_loss = []
+    marginal_loss = []
+
+    performance = []
+
+    stream_1 = iter([get_sample(classification_function=1) for i in range(N_SAMPLES*2)])
+    stream_1 = BatchStream(stream=stream_1, n_features=len(feature_names), task=BINARY_CLF)
+    stream_2 = iter([get_sample(classification_function=2) for i in range(N_SAMPLES*2)])
+    stream_2 = BatchStream(stream=stream_2, n_features=len(feature_names), task=BINARY_CLF)
+    stream = river.datasets.synth.concept_drift_stream.ConceptDriftStream(
+        stream=stream_1, drift_stream=stream_2, position=DRIFT_POSITION, width=DRIFT_WIDTH)
+
+    y_all = []
+
+    for n, (x_i, y_i) in enumerate(stream):
+        y_all.append(y_i)
+
+        y_pred = model.predict_one(x_i)
+        model.learn_one(x_i, y_i)
+
+
+        if n > 100:
+            y_i_pred = model_function(x_i)
+            loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+            model_loss_tracker.update(loss_i)
+            model_loss.append({"loss": model_loss_tracker.get()})
+            loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+            marginal_loss_tracker.update(loss_marginal_i)
+            marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+            fi_values = incremental_sage.explain_one(x_i, y_i)
+            plotter.update(fi_values, facet_name='sage')
+
+            fi_values = incremental_pfi.explain_one(x_i, y_i)
+            plotter.update(fi_values, facet_name='pfi')
+
+        metric.update(y_i, y_pred)
+        performance.append({'performance': metric.get()})
+
+        if n % 1000 == 0:
+            print(f"{n} performance {metric.get()}")
+            print(f"{n} iSAGE       {incremental_sage.importance_values}")
+            print(f"{n} iPFI        {incremental_pfi.importance_values}")
+
+        if n >= N_SAMPLES:
+            break
+
+    plt.hist(y_all)
+    plt.show()
+
+    performance_kw_sage = {
+        "y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
+        "line_names": ["loss"], "names_to_highlight": ['loss'],
+        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+        "markevery": {"model_loss": 100, "marginal_loss": 100},
+        "secondary_legends": [
+            {"legend_props": {"loc": "center right", "ncol": 1, "fontsize": "small"},
+             "legend_items": [("marginal loss", "dashed", "red"), ("model loss", "solid", "red")]}],
+        "fill_between_props": [{'facet_1': 'model_loss', 'facet_2': 'marginal_loss', 'line_name_1': 'loss',
+                                'line_name_2': 'loss', 'hatch': '///', 'color': 'red', 'alpha': 0.1}]
+    }
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
+        performance_kw=performance_kw_sage,
+        title=f'Online Link Prediction based on common Interests',
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['node_1_interest_1', 'node_2_interest_1', 'node_1_interest_2',
+                            'node_2_interest_2', 'node_1_interest_3', 'node_2_interest_3',
+                            'graph_connectivity'
+                            ],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'sage': '-', 'pfi': ''},
+        legend_style={"fontsize": "small", "ncol": 4, "loc": "upper center"},
+        markevery={'sage': 100, 'pfi': 100},
+        y_ticks=[0, 0.02, 0.04, 0.06, 0.08, 0.1, 0.12],
+        y_min=-0.02,
+        y_max=0.14,
+        color_list=['#ef27a6', '#4ea5d9', '#7d53de', '#44001A', '#ef27a6', '#4ea5d9', '#7d53de', '#44cfcb', '#44001A'],
+        save_name="sage_probabilistic_edge_gradual_long.png"
+    )
+
+    # PFI --------------------------------------------------------------------------------------------------------------
+
+    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "accuracy", "color_list": ["red"],
+                      "line_names": ["performance"], "names_to_highlight": ['performance']}
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"accuracy": performance},
+        performance_kw=performance_kw,
+        title=f'Online Link Prediction based on common Interests',
+        y_label='PFI values',
+        x_label='Samples',
+        names_to_highlight=['node_1_interest_1', 'node_2_interest_1', 'node_1_interest_2',
+                            'node_2_interest_2', 'node_1_interest_3', 'node_2_interest_3',
+                            'graph_connectivity'
+                            ],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'sage': '', 'pfi': '-'},
+        legend_style={"fontsize": "small", "ncol": 4, "loc": "upper center"},
+        markevery={'sage': 100, 'pfi': 100},
+        y_ticks=[0, 0.1, 0.2],
+        y_min=-0.04,
+        y_max=0.24,
+        color_list=['#ef27a6', '#4ea5d9', '#7d53de', '#44001A', '#ef27a6', '#4ea5d9', '#7d53de', '#44cfcb', '#44001A'],
+        save_name="pfi_probabilistic_edge_gradual_long.png"
+    )
+
+
+
diff --git a/experiments/pfi/agrawal.py b/experiments/pfi/agrawal.py
new file mode 100644
index 0000000..f2fe401
--- /dev/null
+++ b/experiments/pfi/agrawal.py
@@ -0,0 +1,130 @@
+import numpy as np
+
+from river.ensemble import AdaptiveRandomForestClassifier
+from river import metrics
+from river.utils import Rolling
+
+from data.stream.synth import Agrawal
+
+from increment_explain.imputer import MarginalImputer
+from increment_explain.storage import UniformReservoirStorage
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.explainer.pfi import IncrementalPFI
+from increment_explain.visualization import FeatureImportancePlotter
+
+from sage.utils import CrossEntropyLoss
+
+
+def cross_entropy_loss(y_prediction, y_true):
+    y_prediction = np.asarray([y_prediction])
+    y_prediction = y_prediction.reshape((len(y_prediction), 1))
+    y_true = np.asarray([y_true])
+    y_true = y_true.reshape((len(y_true))).astype(int)
+    return CROSS_ENTROPY(pred=y_prediction, target=y_true)
+
+
+if __name__ == "__main__":
+    CROSS_ENTROPY = CrossEntropyLoss(reduction='mean')
+    RANDOM_SEED = 1
+    CLASSIFICATION_FUNCTIONS = (1, 2)
+
+    STREAM_LENGTH = 10000
+    STREAM_POSITION = int(STREAM_LENGTH * 0.5)
+    N_STREAM_1 = STREAM_POSITION
+    N_STREAM_2 = STREAM_LENGTH - N_STREAM_1
+
+    EXPLAINER_NAME = "inc-PFI"
+
+    SMOOTHING_ALPHA = 0.001
+
+    for i in range(1):
+
+        # Setup Data ---------------------------------------------------------------------------------------------------
+        dataset_1 = Agrawal(classification_function=CLASSIFICATION_FUNCTIONS[0], random_seed=RANDOM_SEED, n_samples=N_STREAM_1)
+        dataset_2 = Agrawal(classification_function=CLASSIFICATION_FUNCTIONS[1], random_seed=RANDOM_SEED, n_samples=N_STREAM_1)
+        stream_1 = dataset_1.stream
+        stream_2 = dataset_2.stream
+
+        feature_names = list(dataset_1.feature_names)
+
+        # Model and training setup
+        model = AdaptiveRandomForestClassifier(seed=RANDOM_SEED)
+        model_function = RiverToPredictionFunction(model, feature_names).predict
+        metric = metrics.Accuracy()
+        metric = Rolling(metric, window_size=500)
+
+        loss_function = cross_entropy_loss
+
+        plotter = FeatureImportancePlotter(
+            feature_names=feature_names
+        )
+
+        model_performance = []
+
+        # Setup explainers ---------------------------------------------------------------------------------------------
+
+        # Instantiating objects
+        storage = UniformReservoirStorage(store_targets=False, size=100)
+        # storage = SequenceStorage(store_targets=True)
+        imputer = MarginalImputer(model_function, 'product', storage)
+        # imputer = DefaultImputer(model, values={'N_1': 5, 'N_2': 3, 'N_10': 2})
+        incremental_explainer = IncrementalPFI(
+            model_function=model_function,
+            feature_names=feature_names,
+            storage=storage,
+            imputer=imputer,
+            n_samples=2,
+            loss_function=loss_function,
+            smoothing_alpha=SMOOTHING_ALPHA,
+            dynamic_setting=True
+        )
+
+        # warm-up because of errors ------------------------------------------------------------------------------------
+        for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
+            model.learn_one(x_i, y_i)
+            if n > 10:
+                break
+
+        # Concept 1 ----------------------------------------------------------------------------------------------------
+
+        for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
+            y_i_pred = model.predict_one(x_i)
+            metric.update(y_true=y_i, y_pred=y_i_pred)
+            model_performance.append({'accuracy': metric.get()})
+            inc_fi_values = incremental_explainer.explain_one(x_i, y_i)
+            plotter.update(importance_values=inc_fi_values, facet_name='inc')
+            model.learn_one(x_i, y_i)
+            if n % 1000 == 0:
+                print(f"{n}: performance {metric.get()}\n"
+                      f"{n}: {EXPLAINER_NAME} {incremental_explainer.importance_values}")
+            if n >= N_STREAM_1:
+                break
+
+        # Concept 2 ----------------------------------------------------------------------------------------------------
+
+        for (n, (x_i, y_i)) in enumerate(stream_2, start=1):
+            y_i_pred = model.predict_one(x_i)
+            metric.update(y_true=y_i, y_pred=y_i_pred)
+            model_performance.append({'accuracy': metric.get()})
+            inc_fi_values = incremental_explainer.explain_one(x_i, y_i)
+            plotter.update(importance_values=inc_fi_values, facet_name='inc')
+            model.learn_one(x_i, y_i)
+            if n % 1000 == 0:
+                print(f"{n}: performance {metric.get()}\n"
+                      f"{n}: {EXPLAINER_NAME} {incremental_explainer.importance_values}")
+            if n >= N_STREAM_2:
+                break
+
+        # Plotting -----------------------------------------------------------------------------------------------------
+        plotter.plot(
+            figsize=(5, 10),
+            model_performance={'accuracy': model_performance},
+            title=f'Agrawal Stream (fn. {CLASSIFICATION_FUNCTIONS[0]} to fn. {CLASSIFICATION_FUNCTIONS[1]})',
+            y_label='SAGE values',
+            x_label='Samples',
+            names_to_highlight=['salary', 'commission', 'age', 'elevel'],
+            v_lines=[{'x': N_STREAM_1, 'ls': '--', 'c': 'black', 'linewidth': 1}],
+            h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+            line_styles={'inc': '-', 'int': '--'},
+            legend_style={}
+        )
diff --git a/experiments/run_KernelSHAP.py b/experiments/run_KernelSHAP.py
deleted file mode 100644
index e69de29..0000000
diff --git a/experiments/run_sage.py b/experiments/run_sage.py
deleted file mode 100644
index fb10b1a..0000000
--- a/experiments/run_sage.py
+++ /dev/null
@@ -1,263 +0,0 @@
-import time
-
-import matplotlib.pyplot as plt
-import river.compat
-from river.utils import Rolling
-from tqdm import tqdm
-
-from explainer.sage import IncrementalSAGE
-from data.synthetic import SyntheticDataset
-
-import copy
-import numpy as np
-
-from river.ensemble import AdaptiveRandomForestRegressor, AdaptiveRandomForestClassifier
-from river.linear_model import Perceptron, LinearRegression
-from river import metrics
-from river.stream import iter_array
-
-import sage
-
-from utils.converters import RiverToPredictionFunction, PredictionFunctionToRiverInput
-from visualization.line_plots import plot_multi_line_graph
-
-
-def _dataset_target_function_regression_1(x):
-    return np.dot(x, _WEIGHTS_1)
-
-
-def _dataset_target_function_classification_1(x):
-    return int(np.dot(x, _WEIGHTS_1) >= _CLASSIFICATION_THRESHOLD_1)
-
-
-def _dataset_target_function_regression_2(x):
-    return np.dot(x, _WEIGHTS_2)
-
-
-def _dataset_target_function_classification_2(x):
-    return int(np.dot(x, _WEIGHTS_2) >= _CLASSIFICATION_THRESHOLD_2)
-
-
-if __name__ == "__main__":
-
-    DEBUG = True
-    RANDOM_SEED = 1
-
-    # Setup Data -------------------------------------------------------------------------------------------------------
-
-    # Problem Definition
-    _N_FEATURES = 10
-    _WEIGHTS_1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-    _WEIGHTS_2 = [0, 0, 0, 0, 0, 0, 0, 0, 0, 10]
-    _CLASSIFICATION_THRESHOLD_1 = 27.5
-    _CLASSIFICATION_THRESHOLD_2 = 5
-    CLASSIFICATION = False
-    REAL_MODEL = True
-
-    # Samples per stream
-    N_STREAM_1 = 10000
-    N_STREAM_2 = 10000
-    BATCH_SIZE = 1
-
-    # Default Values
-    _DEFAULT_VALUES = [0.5 for _ in range(_N_FEATURES)]
-
-    if CLASSIFICATION:
-        _DATASET_TARGET_FUNCTION_1 = _dataset_target_function_classification_1
-        _DATASET_TARGET_FUNCTION_2 = _dataset_target_function_classification_2
-    else:
-        _DATASET_TARGET_FUNCTION_1 = _dataset_target_function_regression_1
-        _DATASET_TARGET_FUNCTION_2 = _dataset_target_function_regression_2
-
-    # Get Data Streams
-    dataset_1 = SyntheticDataset(n_numeric=_N_FEATURES, n_features=_N_FEATURES, n_samples=N_STREAM_1*BATCH_SIZE,
-                                 noise_std=0, target_function=_DATASET_TARGET_FUNCTION_1, random_seed=RANDOM_SEED,
-                                 classification=CLASSIFICATION)
-    x_stream_1 = dataset_1.x_data[:N_STREAM_1]
-    y_stream_1 = dataset_1.y_data[:N_STREAM_1]
-    dataset_2 = SyntheticDataset(n_numeric=_N_FEATURES, n_features=_N_FEATURES, n_samples=N_STREAM_2*BATCH_SIZE,
-                                 noise_std=0, target_function=_DATASET_TARGET_FUNCTION_2, random_seed=RANDOM_SEED,
-                                 classification=CLASSIFICATION)
-    x_stream_2 = dataset_2.x_data[:N_STREAM_2]
-    y_stream_2 = dataset_2.y_data[:N_STREAM_2]
-
-    # Get the feature names for both streams
-    feature_names = dataset_1.feature_names
-
-    # Get Model and Performance Metric
-    if CLASSIFICATION:
-        metric = metrics.Accuracy()
-        if REAL_MODEL:
-            model = AdaptiveRandomForestClassifier(seed=RANDOM_SEED)
-            model_function = RiverToPredictionFunction(model, feature_names).predict
-        else:
-            model_function = PredictionFunctionToRiverInput(prediction_function=_dataset_target_function_classification_1)
-    else:
-        metric = metrics.MAE()
-        if REAL_MODEL:
-            model = AdaptiveRandomForestRegressor(seed=RANDOM_SEED)
-            model_function = RiverToPredictionFunction(model, feature_names).predict
-        else:
-            model_function = PredictionFunctionToRiverInput(prediction_function=_dataset_target_function_regression_1)
-    metric = Rolling(metric, window_size=1000)
-    time_incremental = 0.0000001
-    time_batch = 0.0000001
-    incremental_SAGE_values = []
-
-    # IncrementalSAGE Stream 1 -----------------------------------------------------------------------------------------
-    time.sleep(0.05)
-    explainer = IncrementalSAGE(
-        model_fn=model_function,
-        feature_names=feature_names,
-        loss_function='mse',
-        empty_prediction=None,
-        sub_sample_size=10,
-        # default_values=_DEFAULT_VALUES
-        default_values=None
-    )
-    for (n, (x_i, y_i)) in enumerate(iter_array(x_stream_1, y_stream_1, feature_names=feature_names)):
-        n += 1
-        if REAL_MODEL:
-            model.learn_one(x_i, y_i)
-        start_time = time.time()
-        y_i_pred = model_function(x_i)
-        metric.update(y_true=y_i, y_pred=y_i_pred[0])
-        SAGE_values = explainer.explain_one(x_i=x_i, y_i=y_i)
-        incremental_SAGE_values.append(copy.deepcopy(SAGE_values))
-        time_incremental += time.time() - start_time
-        if n >= N_STREAM_1:
-            break
-        if DEBUG and n % 1000 == 0:
-            print(n)
-            print("Score:", metric.get())
-            print("Sage Values", SAGE_values)
-    SAGE_values = explainer.SAGE_trackers
-    # sage_values_mean = [round(sage_value.mean, 2) for _, sage_value in SAGE_values.items()]
-    # sage_values_std = [round(sage_value.std, 2) for _, sage_value in SAGE_values.items()]
-    sage_values_mean = [round(sage_value(), 2) for _, sage_value in SAGE_values.items()]
-    print("IncrementalSAGE Explanation(")
-    print("  (Mean):", sage_values_mean)
-    print(")")
-    print("Time:", round(time_incremental, 2))
-    """
-    # Original SAGE Stream 1 -------------------------------------------------------------------------------------------
-    time.sleep(0.05)
-    start_time = time.time()
-    imputer = sage.DefaultImputer(model=model_function, values=np.asarray(_DEFAULT_VALUES))
-    estimator = sage.PermutationEstimator(imputer, 'mse')
-    sage_values = estimator(x_stream_1, y_stream_1,
-                            n_permutations=N_STREAM_1,
-                            detect_convergence=False,
-                            batch_size=1)
-    time_batch += time.time() - start_time
-    sage_values_mean = [round(sage_value, 2) for sage_value in sage_values.values]
-    sage_values_std = [round(sage_value, 2) for sage_value in sage_values.std]
-    print("Batch SAGE Explanation(")
-    print("  (Mean):", sage_values_mean)
-    print("  (Std):", sage_values_std)
-    print(")")
-    print("Time:", round(time_batch, 2))
-
-    # Kernel SAGE Stream 1 -------------------------------------------------------------------------------------------
-    time.sleep(0.05)
-    start_time = time.time()
-    imputer = sage.DefaultImputer(model=model_function, values=np.asarray(_DEFAULT_VALUES))
-    estimator = sage.KernelEstimator(imputer, 'mse')
-    sage_values = estimator(x_stream_1, y_stream_1,
-                            n_samples=N_STREAM_1,
-                            detect_convergence=False,
-                            batch_size=1)
-    time_batch += time.time() - start_time
-    sage_values_mean = [round(sage_value, 2) for sage_value in sage_values.values]
-    sage_values_std = [round(sage_value, 2) for sage_value in sage_values.std]
-    print("Batch Kernel SAGE Explanation(")
-    print("  (Mean):", sage_values_mean)
-    print("  (Std):", sage_values_std)
-    print(")")
-    print("Time:", round(time_batch, 2))
-    """
-
-    # Concept Drift ----------------------------------------------------------------------------------------------------
-    if not REAL_MODEL:
-        if CLASSIFICATION:
-            model_function = PredictionFunctionToRiverInput(prediction_function=_dataset_target_function_classification_2)
-        else:
-            model_function = PredictionFunctionToRiverInput(prediction_function=_dataset_target_function_regression_2)
-
-    # IncrementalSAGE Stream 2 -----------------------------------------------------------------------------------------
-    time.sleep(0.05)
-    explainer.model_fn = model_function
-    start_time = time.time()
-    for (n, (x_i, y_i)) in enumerate(iter_array(x_stream_2, y_stream_2, feature_names=feature_names)):
-        n += 1
-        if REAL_MODEL:
-            model.learn_one(x_i, y_i)
-        start_time = time.time()
-        y_i_pred = model_function(x_i)
-        metric.update(y_true=y_i, y_pred=y_i_pred[0])
-        SAGE_values = explainer.explain_one(x_i=x_i, y_i=y_i)
-        incremental_SAGE_values.append(copy.deepcopy(SAGE_values))
-        time_incremental += time.time() - start_time
-        if n >= N_STREAM_2:
-            break
-        if DEBUG and n % 1000 == 0:
-            print(n)
-            print("Score:", metric.get())
-            print("Sage Values", SAGE_values)
-    SAGE_values = explainer.SAGE_trackers
-    end_time = time.time()
-    sage_values_mean = [round(sage_value(), 2) for _, sage_value in SAGE_values.items()]
-    print("IncrementalSAGE Explanation(")
-    print("  (Mean):", sage_values_mean)
-    print(")")
-    print("Time:", round(time_incremental, 2))
-
-    """
-    # Original SAGE Stream 2 -------------------------------------------------------------------------------------------
-    time.sleep(0.05)
-    start_time = time.time()
-    imputer = sage.DefaultImputer(model=model_function, values=np.asarray(_DEFAULT_VALUES))
-    estimator = sage.PermutationEstimator(imputer, 'mse')
-    sage_values = estimator(x_stream_2, y_stream_2,
-                            n_permutations=N_STREAM_2,
-                            detect_convergence=False,
-                            batch_size=1)
-    time_batch += time.time() - start_time
-    sage_values_mean = [round(sage_value, 2) for sage_value in sage_values.values]
-    sage_values_std = [round(sage_value, 2) for sage_value in sage_values.std]
-    print("Batch SAGE Explanation(")
-    print("  (Mean):", sage_values_mean)
-    print("  (Std):", sage_values_std)
-    print(")")
-    print("Time:", round(time_batch, 2))
-
-    time.sleep(0.05)
-    start_time = time.time()
-    imputer = sage.DefaultImputer(model=model_function, values=np.asarray(_DEFAULT_VALUES))
-    estimator = sage.KernelEstimator(imputer, 'mse')
-    sage_values = estimator(x_stream_2, y_stream_2,
-                            n_samples=N_STREAM_1,
-                            detect_convergence=False,
-                            batch_size=1)
-    time_batch += time.time() - start_time
-    sage_values_mean = [round(sage_value, 2) for sage_value in sage_values.values]
-    sage_values_std = [round(sage_value, 2) for sage_value in sage_values.std]
-    print("Batch KernelSAGE Explanation(")
-    print("  (Mean):", sage_values_mean)
-    print("  (Std):", sage_values_std)
-    print(")")
-    print("Time:", round(time_batch, 2))
-
-    # Summary Time
-    print()
-    print("Time Dif. (inc / batch):", time_incremental / time_batch)
-    """
-
-    # Plotting ---------------------------------------------------------------------------------------------------------
-
-    fig, axis = plt.subplots()
-    axis = plot_multi_line_graph(axis=axis,
-                                 y_data=incremental_SAGE_values,
-                                 names_to_highlight=feature_names,
-                                 line_names=feature_names)
-    plt.show()
diff --git a/explainer/__init__.py b/experiments/setup/__init__.py
similarity index 100%
rename from explainer/__init__.py
rename to experiments/setup/__init__.py
diff --git a/experiments/setup/data.py b/experiments/setup/data.py
new file mode 100644
index 0000000..f73f793
--- /dev/null
+++ b/experiments/setup/data.py
@@ -0,0 +1,97 @@
+from typing import Optional, Union
+
+import data.batch
+import data.stream
+import data.stream.synth
+from data.stream._base import get_concept_drift_stream
+from data.stream._base import StreamDataset
+
+DATASET_NAMES = ['bank', 'adult', 'bike', 'elec2', 'agrawal', 'stagger', 'planes', 'insects', 'credit_card']
+BATCH_DATASETS = ['bank', 'adult', 'bike']
+SYNTH_DATASETS = ['agrawal', 'stagger', 'planes']
+STREAM_DATASETS = ['elec2', 'insects', 'credit_card', "stream_bike"]
+
+__all__ = [
+    "get_dataset",
+    "get_concept_drift_dataset"
+]
+
+
+def _get_batch_dataset(dataset_name, shuffle_dataset=False, random_seed=None, n_samples=None):
+    if dataset_name == 'bank':
+        dataset = data.batch.BankMarketing(random_seed=random_seed, shuffle_dataset=shuffle_dataset, n_samples=n_samples)
+    elif dataset_name == 'adult':
+        dataset = data.batch.Adult(random_seed=random_seed, shuffle_dataset=shuffle_dataset, n_samples=n_samples)
+    elif dataset_name == 'bike':
+        dataset = data.batch.BikeSharing(random_seed=random_seed, shuffle_dataset=shuffle_dataset, n_samples=n_samples)
+    else:
+        raise NotImplementedError(f"The batch dataset with name {dataset_name} is not implemented.")
+    return dataset
+
+
+def get_dataset(dataset_name, shuffle=False, random_seed=None, n_samples=None):
+    dataset_parameter = dataset_name.split(' ')[1] if len(dataset_name.split(' ')) > 1 else None
+    dataset_name = dataset_name.split(' ')[0]
+    if dataset_name in BATCH_DATASETS:
+        dataset = _get_batch_dataset(dataset_name=dataset_name, shuffle_dataset=shuffle, random_seed=random_seed, n_samples=n_samples)
+    elif dataset_name == 'elec2':
+        dataset = data.stream.Elec2(n_samples=n_samples)
+    elif dataset_name == 'stream_bike':
+        dataset = data.stream.Bike(n_samples=n_samples)
+    elif dataset_name == 'agrawal':
+        if dataset_parameter is None:
+            dataset_parameter = 1
+        dataset = data.stream.synth.Agrawal(classification_function=int(dataset_parameter), random_seed=random_seed, n_samples=n_samples)
+    elif dataset_name == 'stagger':
+        if dataset_parameter is None:
+            dataset_parameter = 1
+        dataset = data.stream.synth.Stagger(classification_function=int(dataset_parameter), random_seed=random_seed, n_samples=n_samples)
+    elif dataset_name == 'planes':
+        dataset = data.stream.synth.Planes2D(random_seed=random_seed, n_samples=n_samples)
+    elif dataset_name == 'insects':
+        if dataset_parameter is None:
+            dataset_parameter = "abrupt_balanced"
+        dataset = data.stream.Insects(variant=dataset_parameter, n_samples=n_samples)
+    elif dataset_name == 'credit_card':
+        dataset = data.stream.CreditCard()
+    else:
+        raise NotImplementedError(f"The dataset {dataset_name} is not implemented. Implemented Datasets are {DATASET_NAMES}.")
+    return dataset
+
+
+def get_concept_drift_dataset(dataset_1_name: str, dataset_2_name: str,
+                              dataset_1, dataset_2,
+                              position: Union[float, int] = 0.5,
+                              width: Union[float, int] = 0.05,
+                              features_to_switch: Optional[Union[dict, str]] = None):
+    dataset_name = dataset_1_name.split(' ')[0]
+    stream_1 = dataset_1.stream
+    if dataset_name in BATCH_DATASETS or dataset_name in STREAM_DATASETS:
+        stream_2 = None
+        n_samples = dataset_1.n_samples
+    else:
+        stream_2 = dataset_2.stream
+        n_samples = dataset_1.n_samples + dataset_2.n_samples
+    if position < 1:
+        position = int(position * n_samples)
+    if width < 1:
+        width = int(width * n_samples)
+
+    # create the concept drift stream
+    concept_drift_stream = get_concept_drift_stream(
+        stream=stream_1, drift_stream=stream_2,
+        position=position, width=width, feature_remapping=features_to_switch)
+
+    # create the new dataset that contains the concept drift stream
+    dataset = StreamDataset(
+        stream=concept_drift_stream,
+        n_samples=n_samples,
+        feature_names=dataset_1.feature_names,
+        cat_feature_names=dataset_1.cat_feature_names,
+        num_feature_names=dataset_1.num_feature_names,
+        task=dataset_1.task,
+        n_features=dataset_1.n_features,
+        n_outputs=dataset_1.n_outputs
+    )
+
+    return dataset
diff --git a/experiments/setup/explainer.py b/experiments/setup/explainer.py
new file mode 100644
index 0000000..101a9cd
--- /dev/null
+++ b/experiments/setup/explainer.py
@@ -0,0 +1,147 @@
+import typing
+
+from increment_explain.storage import GeometricReservoirStorage, UniformReservoirStorage, TreeStorage
+from increment_explain.imputer import MarginalImputer, TreeImputer
+from increment_explain.explainer.sage import IncrementalSageExplainer, BatchSageExplainer, IntervalSageExplainer
+from increment_explain.explainer.pfi import IncrementalPFI
+
+
+def get_imputer_and_storage(
+        model_function,
+        feature_removal_distribution,
+        reservoir_kind,
+        reservoir_length,
+        cat_feature_names=None,
+        num_feature_names=None
+):
+    if feature_removal_distribution in {'marginal joint', 'marginal product'}:
+        if reservoir_kind == 'geometric':
+            storage = GeometricReservoirStorage(
+                size=reservoir_length,
+                store_targets=False
+            )
+        elif reservoir_kind == 'uniform':
+            storage = UniformReservoirStorage(
+                size=reservoir_length,
+                store_targets=False
+            )
+        else:
+            raise NotImplementedError(f"Only 'geometric', or 'uniform' storage implemented, not {reservoir_kind}.")
+
+        imputer = MarginalImputer(
+            model_function=model_function,
+            storage_object=storage,
+            sampling_strategy=feature_removal_distribution.split(" ")[1]
+        )
+    elif feature_removal_distribution == 'conditional':
+        assert cat_feature_names is not None and num_feature_names is not None, \
+            "cat. and num. feature names must be provided"
+
+        storage = TreeStorage(
+            cat_feature_names=cat_feature_names,
+            num_feature_names=num_feature_names
+        )
+
+        imputer = TreeImputer(
+            model_function=model_function,
+            storage_object=storage,
+            use_storage=True,
+            direct_predict_numeric=True
+        )
+    else:
+        raise NotImplementedError(f"Only 'marginal joint', 'marginal product', and 'conditional' feature removal "
+                                  f"distributions are implemented. Not {feature_removal_distribution}")
+
+    return imputer, storage
+
+
+def get_incremental_sage_explainer(
+        model_function,
+        feature_names,
+        loss_function,
+        distribution_kind: str = 'marginal',
+        reservoir_kind: str = 'geometric',
+        reservoir_length: int = 1000,
+        smoothing_alpha: float = 0.001,
+        n_inner_samples: int = 1,
+        cat_feature_names: typing.Optional[typing.List[str]] = None,
+        num_feature_names: typing.Optional[typing.List[str]] = None
+
+) -> IncrementalSageExplainer:
+
+    imputer, storage = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution=distribution_kind,
+        reservoir_kind=reservoir_kind,
+        reservoir_length=reservoir_length,
+        cat_feature_names=cat_feature_names,
+        num_feature_names=num_feature_names
+    )
+    explainer = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=smoothing_alpha,
+        n_inner_samples=n_inner_samples
+    )
+    return explainer
+
+
+def get_incremental_pfi_explainer(
+        model_function,
+        feature_names,
+        loss_function,
+        reservoir_kind: str = 'geometric',
+        sample_strategy: str = 'joint',
+        reservoir_length: int = 1000,
+        smoothing_alpha: float = 0.001,
+        n_inner_samples: int = 1,
+) -> IncrementalPFI:
+
+    imputer, storage = get_imputer_and_storage(model_function, 'marginal', reservoir_kind, reservoir_length, sample_strategy)
+    explainer = IncrementalPFI(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        feature_names=feature_names,
+        smoothing_alpha=smoothing_alpha,
+        n_samples=n_inner_samples,
+        storage=storage
+    )
+    return explainer
+
+
+def get_batch_sage_explainer(
+        model_function,
+        loss_function,
+        feature_names,
+        n_inner_samples: int = 1
+) -> BatchSageExplainer:
+
+    explainer = BatchSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        feature_names=feature_names,
+        n_inner_samples=n_inner_samples
+    )
+    return explainer
+
+
+def get_interval_sage_explainer(
+        model_function,
+        loss_function,
+        feature_names,
+        n_inner_samples: int = 1,
+        interval_length: int = 100
+) -> IntervalSageExplainer:
+
+    explainer = IntervalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        feature_names=feature_names,
+        n_inner_samples=n_inner_samples,
+        interval_length=interval_length
+    )
+    return explainer
diff --git a/experiments/setup/loss.py b/experiments/setup/loss.py
new file mode 100644
index 0000000..79e555d
--- /dev/null
+++ b/experiments/setup/loss.py
@@ -0,0 +1,68 @@
+import numpy as np
+from river.datasets.base import BINARY_CLF, REG, MULTI_CLF
+from sage.utils import MSELoss, CrossEntropyLoss
+from river.metrics import Accuracy, MAE, CrossEntropy
+from river.utils import Rolling
+
+__all__ = [
+    "get_loss_function",
+    "cross_entropy_loss",
+    "mse_loss"
+]
+
+CROSS_ENTROPY = CrossEntropyLoss(reduction='mean')
+MSE = MSELoss(reduction='mean')
+
+
+
+
+def mse_loss(y_prediction, y_true):
+    y_prediction = np.asarray([y_prediction])
+    y_prediction = y_prediction.reshape((len(y_prediction), 1))
+    y_true = np.asarray([y_true])
+    y_true = y_true.reshape((len(y_true))).astype(int)
+    return MSE(pred=y_prediction, target=y_true)
+
+
+def cross_entropy_loss(y_prediction, y_true):
+    y_prediction = np.asarray([y_prediction])
+    y_prediction = y_prediction.reshape((len(y_prediction), 1))
+    y_true = np.asarray([y_true])
+    y_true = y_true.reshape((len(y_true))).astype(int)
+    return CROSS_ENTROPY(pred=y_prediction, target=y_true)
+
+
+def cross_entropy_loss_multi(y_prediction, y_true):
+    y_prediction = np.asarray([y_prediction])
+    y_prediction = y_prediction.reshape((len(y_prediction), 1))
+    y_true = np.asarray([y_true])
+    y_true = y_true.reshape((len(y_true))).astype(int)
+    return CROSS_ENTROPY(pred=y_prediction, target=y_true)
+
+
+def get_loss_function(task):
+    if task == BINARY_CLF:  # TODO make task checks with River Global variables
+        loss_function = cross_entropy_loss
+    elif task == REG:
+        loss_function = mse_loss
+    elif task == MULTI_CLF:
+        loss_function = cross_entropy_loss_multi
+    else:
+        raise NotImplementedError(f"No standard loss implemented for task {task}.")
+    return loss_function
+
+
+def get_training_metric(task, rolling_window=1000):
+    if task == BINARY_CLF:
+        training_metric = CrossEntropy()
+    elif task == REG:
+        training_metric = MAE()
+    elif task == MULTI_CLF:
+        training_metric = CrossEntropy()
+    else:
+        raise NotImplementedError(f"No standard loss implemented for task {task}.")
+    if rolling_window > 0:
+        rolling_training_metric = Rolling(training_metric, rolling_window)
+    else:
+        rolling_training_metric = training_metric
+    return rolling_training_metric, training_metric
diff --git a/experiments/setup/model.py b/experiments/setup/model.py
new file mode 100644
index 0000000..f668211
--- /dev/null
+++ b/experiments/setup/model.py
@@ -0,0 +1,24 @@
+from river.datasets import base
+from river.ensemble import AdaptiveRandomForestClassifier, AdaptiveRandomForestRegressor
+from increment_explain.utils.converters import RiverToPredictionFunction
+
+
+def _get_arf_model(task, **kwargs):
+    if task == base.BINARY_CLF:
+        model = AdaptiveRandomForestClassifier(**kwargs)
+    else:
+        model = AdaptiveRandomForestRegressor(**kwargs)
+    return model
+
+
+def get_model(model_name, task, feature_names, **model_kw):
+    if model_name == 'ARF':
+        model = _get_arf_model(task, **model_kw)
+    else:
+        raise NotImplementedError
+    model_function = RiverToPredictionFunction(
+        model,
+        classification=task == base.BINARY_CLF,
+        feature_names=feature_names
+    ).predict
+    return model, model_function
diff --git a/experiments/storage/check_reservoir_mechanisms.py b/experiments/storage/check_reservoir_mechanisms.py
new file mode 100644
index 0000000..d5c3cae
--- /dev/null
+++ b/experiments/storage/check_reservoir_mechanisms.py
@@ -0,0 +1,39 @@
+from increment_explain.storage import UniformReservoirStorage, GeometricReservoirStorage
+import matplotlib.pyplot as plt
+
+
+def dummy_stream(t):
+    return [({'t': i}, i) for i in range(t)]
+
+
+if __name__ == "__main__":
+
+    STORE_TARGETS = False
+    SIZE = 100
+    T = 1000
+
+    reservoir_values = {i: 0 for i in range(T)}
+    stream = dummy_stream(t=T)
+    for _ in range(1000):
+        storage = UniformReservoirStorage(store_targets=STORE_TARGETS, size=SIZE)
+        for x_i, y_i in stream:
+            storage.update(x_i, y_i)
+        for feature in storage.get_data()[0]:
+            reservoir_values[feature['t']] += 1
+
+    plt.bar(reservoir_values.keys(), reservoir_values.values())
+    plt.title(f"Uniform Reservoir: size {SIZE}, T {T}")
+    plt.show()
+
+    reservoir_values = {i: 0 for i in range(T)}
+    stream = dummy_stream(t=T)
+    for _ in range(1000):
+        storage = GeometricReservoirStorage(store_targets=STORE_TARGETS, size=SIZE, constant_probability=1)
+        for x_i, y_i in stream:
+            storage.update(x_i, y_i)
+        for feature in storage.get_data()[0]:
+            reservoir_values[feature['t']] += 1
+
+    plt.bar(reservoir_values.keys(), reservoir_values.values())
+    plt.title(f"Geometric Reservoir: size {SIZE}, T {T}")
+    plt.show()
diff --git a/explainer/tests/__init__.py b/experiments/visualization/__init__.py
similarity index 100%
rename from explainer/tests/__init__.py
rename to experiments/visualization/__init__.py
diff --git a/experiments/visualization/boxplots.py b/experiments/visualization/boxplots.py
new file mode 100644
index 0000000..c9165a3
--- /dev/null
+++ b/experiments/visualization/boxplots.py
@@ -0,0 +1,103 @@
+import numpy as np
+import pandas as pd
+from matplotlib import pyplot as plt
+
+# from increment_explain.visualization.color import DEFAULT_COLOR_LIST
+
+DEFAULT_COLOR_LIST = ['#1d4289']
+
+
+def grouped_boxplot(data, feature_names, explainer_names, save_path, legend, width=0.5, min_space=0.1, feature_spacing=0.5, y_min=-1., y_max=1., legend_pos=0):
+    n_explainer = len(explainer_names)
+    n_features = len(feature_names)
+
+    feature_names_short = [name[0:3]+'.' if len(name) > 3 else name[0:3] for name in feature_names]
+
+    group_length = width * n_explainer + min_space * (n_explainer-1)
+    x_range = np.array([x * group_length for x in range(n_features)])
+    x_range = np.array([x_range[i] + feature_spacing * i for i in range(n_features)])
+    x_locations = sorted(
+        [feature_loc + explainer * (width + min_space) for explainer in range(n_explainer) for feature_loc in x_range])
+    feature_x_locations = x_range + group_length / 2 - width / 2
+
+    fig = plt.figure(0, figsize=[9, 4.8])
+    ax = fig.add_subplot(111)
+    x_position_index = 0
+    for feature in feature_names:
+        color_index = 0
+        for explainer in explainer_names:
+            if explainer == 'batch_total' or explainer == 'batch_interval':
+                color = 'black'
+            else:
+                color = DEFAULT_COLOR_LIST[color_index]
+                color_index += 1
+            #ax.boxplot(
+            #    data[explainer][feature], positions=[x_locations[x_position_index]],
+            #    widths=width, patch_artist=True,
+            #    medianprops={'color': 'red', 'linestyle': '-'},boxprops=dict(facecolor=color, color=None)
+            #)
+            violin_parts = ax.violinplot(
+                data[explainer][feature], positions=[x_locations[x_position_index]],
+                widths=width,
+                showmeans=True, showmedians=True
+            )
+            for pc in violin_parts['bodies']:
+                #pc.set_facecolor(color)
+                #pc.set_edgecolor(color)
+                pc.set_color(color)
+            for partname in ('cbars', 'cmins', 'cmaxes', 'cmeans'):
+                vp = violin_parts[partname]
+                vp.set_edgecolor(color)
+                vp.set_linewidth(1)
+            vp = violin_parts['cmedians']
+            vp.set_edgecolor('red')
+            vp.set_linewidth(1)
+
+            x_position_index += 1
+    x_position_index = 0
+    color_index = 0
+    for explainer in explainer_names:
+        if explainer == 'batch_total' or explainer == 'batch_interval':
+            color = 'black'
+        else:
+            color = DEFAULT_COLOR_LIST[color_index]
+            color_index += 1
+        legend_name = legend[x_position_index]
+        ax.plot([], c=color, label=legend_name)
+        x_position_index += 1
+
+    ax.legend(edgecolor="0.8", fancybox=False, loc=legend_pos)
+    ax.set_xticks(feature_x_locations, feature_names_short)
+    ax.set_ylim(y_min, y_max)
+    ax.axhline(y=0, color=(0.5, 0.5, 0.5, 0.3), ls='--')
+    ax.set_ylabel('SAGE Values')
+    ax.set_xlabel('Features')
+    plt.tight_layout()
+    # plt.savefig(save_path, dpi=300)
+    plt.show()
+
+if __name__ == "__main__":
+
+    bat = [{'age': 0.028108668948204605, 'capital-gain': 0.05819510844543615, 'capital-loss': -0.009544486429741544, 'hours-per-week': 0.013155252465282052, 'fnlwgt': -0.0029857507201568488, 'workclass': -0.017613726611499247, 'education': 0.00042386958935468827, 'marital-status': 0.05439771678248133, 'occupation': 0.0016695371446097832, 'relationship': 0.08693410403102586, 'race': -0.005477809004474466, 'sex': 0.002669468345976513, 'native-country': -0.016564094138247252, 'education-num': 0.05385642606196482}, {'age': 0.027682255884510155, 'capital-gain': 0.055071822538957, 'capital-loss': -0.009050878850901029, 'hours-per-week': 0.01507412109689512, 'fnlwgt': -0.015753524269126337, 'workclass': -0.011765950551764718, 'education': -0.006686509514523585, 'marital-status': 0.08200240882771447, 'occupation': -0.003363941900762748, 'relationship': 0.06269993831493424, 'race': -0.007287603328323027, 'sex': 0.01845915407224611, 'native-country': -0.022423876746061333, 'education-num': 0.059211911293994245}, {'age': 0.02629933586076304, 'capital-gain': 0.047348888828515276, 'capital-loss': -0.008380769643279714, 'hours-per-week': 0.007628869030043197, 'fnlwgt': -0.009889991206343277, 'workclass': -0.0019752946425159617, 'education': -0.004365397345300091, 'marital-status': 0.08520484051636022, 'occupation': -0.0015253027093378555, 'relationship': 0.08201273651008828, 'race': -0.020244490079056546, 'sex': 0.0015202149566501612, 'native-country': -0.014535516949131752, 'education-num': 0.055141044503774064}, {'age': 0.02086520576527711, 'capital-gain': 0.05584322953650521, 'capital-loss': -0.004451512804291052, 'hours-per-week': 0.005407017217543419, 'fnlwgt': -0.014223539832536403, 'workclass': -0.010672812804902657, 'education': -0.0019274026641863853, 'marital-status': 0.07939688843223419, 'occupation': -0.00531566351257984, 'relationship': 0.08223375779888964, 'race': -0.012255298884018497, 'sex': 0.007240672284945291, 'native-country': -0.01716196642752966, 'education-num': 0.06409383725688973}, {'age': 0.03172187298173967, 'capital-gain': 0.05036333938639466, 'capital-loss': -0.017535865242490987, 'hours-per-week': 0.02263163451454883, 'fnlwgt': -0.010640433634705583, 'workclass': -0.016220548261482247, 'education': -0.006509064981150751, 'marital-status': 0.07753561376098746, 'occupation': -0.003088613293430229, 'relationship': 0.08879378828102573, 'race': -0.008228024623543877, 'sex': -0.001257477176132556, 'native-country': -0.016788444707527907, 'education-num': 0.05506402160315771}, {'age': 0.022885249213733968, 'capital-gain': 0.05188981662209141, 'capital-loss': -0.0027212247719945056, 'hours-per-week': 0.00735226411503803, 'fnlwgt': -0.030371798621983047, 'workclass': 0.0030405837219790945, 'education': -0.011568819869035936, 'marital-status': 0.05701380307681665, 'occupation': -0.006279217802575051, 'relationship': 0.10828757377516984, 'race': -0.014709792824596288, 'sex': 0.008093334216971913, 'native-country': -0.011103819544325651, 'education-num': 0.06916272816360407}, {'age': 0.017188197096873293, 'capital-gain': 0.055104508872217864, 'capital-loss': -0.0043284752591741665, 'hours-per-week': 0.006456527408449193, 'fnlwgt': -0.014094963296313321, 'workclass': -0.014463412048273606, 'education': -0.0016933164783028856, 'marital-status': 0.07471348400793593, 'occupation': -0.00790547489291557, 'relationship': 0.09988813376009567, 'race': -0.01540886370994617, 'sex': 0.00838854098765192, 'native-country': -0.017578997993545512, 'education-num': 0.056055099365500016}, {'age': 0.010354291915876435, 'capital-gain': 0.05330415567352984, 'capital-loss': -0.0018205928845302774, 'hours-per-week': 0.0092418954085724, 'fnlwgt': -0.016693608415608147, 'workclass': -0.012153118246136775, 'education': -0.0045776481276726, 'marital-status': 0.08940861686604011, 'occupation': -0.001520708784934167, 'relationship': 0.07302702595104182, 'race': -0.007689926107507798, 'sex': 0.009310812298269513, 'native-country': -0.020448972951357527, 'education-num': 0.06652569120600935}, {'age': 0.019755915593179133, 'capital-gain': 0.057325516840486486, 'capital-loss': -0.012006296604516168, 'hours-per-week': 0.011072765922383139, 'fnlwgt': -0.016524759309212735, 'workclass': -0.0012707924559163684, 'education': -0.0002908179329278292, 'marital-status': 0.0835673896045199, 'occupation': 0.005234255075447237, 'relationship': 0.0787406870013968, 'race': -0.008605934869527833, 'sex': 0.005992083815394923, 'native-country': -0.021159927171322278, 'education-num': 0.04913790222487249}, {'age': 0.02647903389514357, 'capital-gain': 0.05688205032555184, 'capital-loss': 0.0006749931554564515, 'hours-per-week': 0.012228737037665475, 'fnlwgt': -0.010864641096924343, 'workclass': -0.01321899165924379, 'education': 0.004665774153179211, 'marital-status': 0.05506181592877661, 'occupation': -0.005296510368831303, 'relationship': 0.09392146296134306, 'race': -0.009885204257821112, 'sex': -0.003951912535501598, 'native-country': -0.012028018331600254, 'education-num': 0.054405072119631005}]
+    inc = [{'age': 0.026212565230670844, 'capital-gain': 0.051853815484481695, 'capital-loss': -0.03375947462819334, 'hours-per-week': -0.005952738049372549, 'fnlwgt': 0.010891015572607606, 'workclass': 0.007855645437528758, 'education': 0.020711098122370295, 'marital-status': 0.06818484889581049, 'occupation': 0.008780778089487512, 'relationship': 0.11479715867765075, 'race': -0.0396267917209024, 'sex': 0.01715652361833196, 'native-country': -0.01924195964756475, 'education-num': 0.05166071497440728}, {'age': 0.031953365210154086, 'capital-gain': 0.040233816057044174, 'capital-loss': 0.015308258533997407, 'hours-per-week': 0.024763094442210132, 'fnlwgt': -0.023864240785632842, 'workclass': -0.005790773616792286, 'education': 0.002009900888309228, 'marital-status': 0.09022362975547611, 'occupation': 0.018962049430597573, 'relationship': 0.06255732633063874, 'race': -0.029619480401851726, 'sex': 0.007644171018501113, 'native-country': -0.006982363425793644, 'education-num': 0.04791278779268208}, {'age': 0.03105109977374419, 'capital-gain': 0.050636700735616574, 'capital-loss': 0.0010303195419861496, 'hours-per-week': 0.006107991972910388, 'fnlwgt': -0.007916388612774319, 'workclass': 0.014082826220676083, 'education': -0.010737141118534802, 'marital-status': 0.06908178217647476, 'occupation': 0.031675961832971174, 'relationship': 0.059901137800125445, 'race': -0.007157037614547071, 'sex': -0.0009777636433992965, 'native-country': -0.007626602269783604, 'education-num': 0.058542561430969496}, {'age': 0.0019795192237329323, 'capital-gain': 0.06266937606807628, 'capital-loss': 0.014599549092842078, 'hours-per-week': 0.007318639276298247, 'fnlwgt': -0.010726128120754559, 'workclass': 0.004489314667250441, 'education': 0.0008595487221832177, 'marital-status': 0.0873553837252126, 'occupation': 0.005523453993294373, 'relationship': 0.07941329917704586, 'race': -0.016871830093080787, 'sex': 0.0033822590452917494, 'native-country': -0.024440122334842395, 'education-num': 0.08584025865593141}, {'age': 0.011240568934383448, 'capital-gain': 0.04916588978528996, 'capital-loss': -0.002950679370124982, 'hours-per-week': 0.031098753850121487, 'fnlwgt': -0.01817694217174261, 'workclass': 0.0011091359097070786, 'education': 0.005897980198222674, 'marital-status': 0.08630570758662444, 'occupation': -0.010636354979754884, 'relationship': 0.10562996823761453, 'race': -0.03278296714088403, 'sex': 0.01516986822424592, 'native-country': 0.00030476243689187574, 'education-num': 0.04474164286708921}, {'age': 0.02527621561549269, 'capital-gain': 0.05942634621416649, 'capital-loss': -0.00778843166475, 'hours-per-week': -0.006697969601842125, 'fnlwgt': -0.002620768074243089, 'workclass': -0.011759245580332561, 'education': 0.009436931521865479, 'marital-status': 0.050490512312519775, 'occupation': -0.007253971655294637, 'relationship': 0.1055138636402115, 'race': -0.004324591297986123, 'sex': 0.00974487665515802, 'native-country': -0.016807565593019727, 'education-num': 0.08101708872067219}, {'age': 0.013665420626564943, 'capital-gain': 0.04084367707389386, 'capital-loss': -0.010685161970382525, 'hours-per-week': 0.026638396909574487, 'fnlwgt': -0.006610950459774061, 'workclass': -0.01297298975404451, 'education': 0.014663972299887171, 'marital-status': 0.05502675604932766, 'occupation': -0.01721292825385452, 'relationship': 0.10073932601856056, 'race': 0.01145610023483624, 'sex': -0.006986945163094203, 'native-country': -0.0017781354711467162, 'education-num': 0.06348274995816154}, {'age': -0.009320204798651387, 'capital-gain': 0.06737366771417695, 'capital-loss': 0.026711433068659206, 'hours-per-week': 0.01883960509326243, 'fnlwgt': 0.007472164747561111, 'workclass': -0.047188634961137156, 'education': -0.003399483946126723, 'marital-status': 0.10785094801889761, 'occupation': -0.01781491820830615, 'relationship': 0.06674819763646075, 'race': -0.0032157086034249644, 'sex': 0.00456514963644816, 'native-country': -0.010325849140494201, 'education-num': 0.07132928098600357}, {'age': 0.024523050501542555, 'capital-gain': 0.07033876936681563, 'capital-loss': 0.002557534743437581, 'hours-per-week': 0.02273157756914485, 'fnlwgt': -0.01480068897641932, 'workclass': -0.017999555202484696, 'education': 0.011004783289738721, 'marital-status': 0.09243051384466423, 'occupation': 0.01293094691824848, 'relationship': 0.05214557258523368, 'race': 0.004067037722099184, 'sex': -0.007338460285802911, 'native-country': -0.04306589901901369, 'education-num': 0.07354986167466582}, {'age': 0.027940164264492553, 'capital-gain': 0.03407085118913663, 'capital-loss': 0.0038879367482070283, 'hours-per-week': 0.0037577188617372538, 'fnlwgt': -0.0018272174107804018, 'workclass': 0.008329442815579695, 'education': -0.011475516553836418, 'marital-status': 0.06979644873463138, 'occupation': 0.03565973822196416, 'relationship': 0.0738045663238286, 'race': -0.012000996879253012, 'sex': 0.001722160244286283, 'native-country': -0.017234569261312706, 'education-num': 0.05323684425911136}]
+
+    feature_names = list(inc[0].keys())
+
+    bat = pd.DataFrame(bat)
+    inc = pd.DataFrame(inc)
+
+
+
+    data = {'batch_total': bat, 'incremental': inc}
+
+    explainer_names = list(data.keys())
+
+    grouped_boxplot(
+        data,
+        feature_names,
+        explainer_names,
+        save_path=None,
+        legend=['Batch SAGE (left)', 'iSAGE (right)'], legend_pos=2,
+        width=0.5, min_space=0.1, feature_spacing=0.5,
+        y_min=-0.06, y_max=0.12
+    )
\ No newline at end of file
diff --git a/experiments/visualization/plot_feature_importance.py b/experiments/visualization/plot_feature_importance.py
new file mode 100644
index 0000000..74076b2
--- /dev/null
+++ b/experiments/visualization/plot_feature_importance.py
@@ -0,0 +1,101 @@
+import copy
+import os
+from fractions import Fraction
+import matplotlib.pyplot as plt
+import pandas as pd
+
+from increment_explain.visualization.config import SAVE_DPI
+from increment_explain.visualization.line_plots import plot_multi_line_graph
+
+
+def plot_stacked_line_plots(top_data, bot_data, top_styles, bot_styles,
+                            figsize=(10, 5), top_portion=0.5, title=None, show=False,
+                            save_path=None, top_margin=0.88):
+    ratio = Fraction(top_portion).limit_denominator()
+    top_ratio, bot_ratio = (ratio.numerator, ratio.denominator)
+    top_styles = copy.deepcopy(top_styles)
+    bot_styles = copy.deepcopy(bot_styles)
+    fig, (top_axis, bot_axis) = plt.subplots(nrows=2, ncols=1, sharex='all',
+                                             figsize=figsize,
+                                             gridspec_kw={'height_ratios': [top_ratio, bot_ratio]})
+    plt.subplots_adjust(hspace=0.000)
+    bot_axis = plot_multi_line_graph(axis=bot_axis,
+                                     y_data=bot_data['y_data'],
+                                     #x_data=bot_data['x_data'],
+                                     line_names=bot_data['line_names'],
+                                     **bot_styles)
+    top_axis = plot_multi_line_graph(axis=top_axis,
+                                     y_data=top_data['y_data'],
+                                     #x_data=top_data['x_data'],
+                                     line_names=top_data['line_names'],
+                                     **top_styles)
+    plt.tight_layout()
+    if title is not None:
+        fig.suptitle(title)
+        plt.subplots_adjust(hspace=0.000, wspace=0.000, top=top_margin)
+    else:
+        plt.subplots_adjust(hspace=0.000, wspace=0.000)
+    if save_path is not None:
+        plt.savefig(save_path)
+    if show:
+        plt.show()
+        return None
+    else:
+        return fig, (top_axis, bot_axis)
+
+
+def plot_double_stacked_line_plots(top_left_data, top_right_data, bot_left_data, bot_right_data,
+                                   top_left_styles, top_right_styles, bot_left_styles, bot_right_styles,
+                                   figsize=(10, 5), top_portion=1., left_proportion=1., title=None, show=False,
+                                   different_scales=False,
+                                   save_path=None, top_margin=0.88, wspace=0.0):
+    ratio_vertical = Fraction(top_portion).limit_denominator()
+    top_ratio, bot_ratio = (ratio_vertical.numerator, ratio_vertical.denominator)
+    ratio_horizontal = Fraction(left_proportion).limit_denominator()
+    left_ratio, right_ratio = (ratio_horizontal.numerator, ratio_horizontal.denominator)
+    sharey = 'row' if not different_scales else 'none'
+    fig, axes = plt.subplots(nrows=2, ncols=2, sharex='col', sharey=sharey,
+                             dpi=SAVE_DPI,
+                             figsize=figsize,
+                             gridspec_kw={'height_ratios': [top_ratio, bot_ratio],
+                                          'width_ratios': [left_ratio, right_ratio]})
+
+    (top_left_axis, top_right_axis), (bot_left_axis, bot_right_axis) = axes
+    top_left_axis = plot_multi_line_graph(axis=top_left_axis,
+                                          y_data=top_left_data['y_data'],
+                                          #x_data=top_left_data['x_data'],
+                                          line_names=top_left_data['line_names'],
+                                          **top_left_styles)
+    top_right_axis = plot_multi_line_graph(axis=top_right_axis,
+                                           y_data=top_right_data['y_data'],
+                                           #x_data=top_right_data['x_data'],
+                                           line_names=top_right_data['line_names'],
+                                           **top_right_styles)
+    bot_left_axis = plot_multi_line_graph(axis=bot_left_axis,
+                                          y_data=bot_left_data['y_data'],
+                                          #x_data=bot_left_data['x_data'],
+                                          line_names=bot_left_data['line_names'],
+                                          **bot_left_styles)
+    bot_right_axis = plot_multi_line_graph(axis=bot_right_axis,
+                                           y_data=bot_right_data['y_data'],
+                                           #x_data=bot_right_data['x_data'],
+                                           line_names=bot_right_data['line_names'],
+                                           **bot_right_styles)
+    plt.tight_layout()
+
+    if title is not None:
+        fig.suptitle(title)
+        plt.subplots_adjust(hspace=0., wspace=wspace, top=top_margin)
+    else:
+        plt.subplots_adjust(hspace=0., wspace=wspace)
+    if save_path is not None:
+        plt.savefig(save_path)
+    if show:
+        plt.show()
+        return None
+    else:
+        return fig, (top_left_axis, top_right_axis, bot_left_axis, bot_right_axis)
+
+
+if __name__ == "__main__":
+    pass
diff --git a/explainer/sage.py b/explainer/sage.py
deleted file mode 100644
index 2385564..0000000
--- a/explainer/sage.py
+++ /dev/null
@@ -1,152 +0,0 @@
-"""
-This module gathers SAGE Explanation Methods
-"""
-
-# Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
-#          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
-
-import copy
-from abc import ABCMeta
-from abc import abstractmethod
-
-import numpy as np
-
-from sampling.sampler import BatchSampler, ReservoirSampler, HistogramSampler
-from utils.trackers import WelfordTracker, ExponentialSmoothingTracker
-
-__all__ = [
-    "IncrementalSAGE",
-]
-
-# =============================================================================
-# Types and constants
-# =============================================================================
-
-EPS = 1e-10
-
-# TODO change MAE and MSE loss to be used with arrays
-def mae_loss(y_true, y_prediction):
-    return abs(y_true - y_prediction)
-
-
-def mse_loss(y_true, y_prediction):
-    return (y_true - y_prediction) ** 2
-
-
-# =============================================================================
-# Base Sampler Class
-# =============================================================================
-
-
-class BaseIncrementalExplainer(metaclass=ABCMeta):
-    """Base class for incremental explainer algorithms.
-
-    Warning: This class should not be used directly.
-    Use derived classes instead.
-    """
-
-    @abstractmethod
-    def __init__(
-            self,
-            *,
-            model_fn,
-            feature_names,
-            random_state
-    ):
-        self.model_fn = model_fn
-        self.feature_names = feature_names
-        self.random_state = random_state
-        self.number_of_features = len(feature_names)
-        self.seen_samples = 0
-
-    @abstractmethod
-    def explain_one(self, x, y):
-        pass
-
-
-# =============================================================================
-# Public SAGE Explainers
-# =============================================================================
-
-
-class IncrementalSAGE(BaseIncrementalExplainer):
-
-    def __init__(
-            self,
-            model_fn,
-            *,
-            feature_names,
-            loss_function='mse',
-            random_state=None,
-            sub_sample_size=1,
-            empty_prediction=None,
-            default_values: list = None
-    ):
-        super(IncrementalSAGE, self).__init__(
-            model_fn=model_fn,
-            feature_names=feature_names,
-            random_state=random_state
-        )
-        # TODO enable custom loss functions as parameters
-        assert loss_function in ['mae', 'mse'], "Loss function must be either 'mae' or 'mse'."
-        if loss_function == 'mae':
-            self.loss_function = mae_loss
-        else:
-            self.loss_function = mse_loss
-        self.default_values = default_values
-        if default_values is None:
-            self.sampler = ReservoirSampler(feature_names=feature_names,
-                                            constant_probability=1.,
-                                            store_targets=False,
-                                            reservoir_length=300,
-                                            sample_with_replacement=False)
-        else:
-            self.default_values = {feature: default_values[i] for feature, i in zip(feature_names, range(len(feature_names)))}
-        self.sub_sample_size = sub_sample_size
-        self.marginal_prediction = ExponentialSmoothingTracker(alpha=0.005)
-        self.SAGE_trackers = {feature_name: ExponentialSmoothingTracker(alpha=0.005) for feature_name in feature_names}
-        self.pfi_trackers = {feature_name: ExponentialSmoothingTracker(alpha=0.005) for feature_name in feature_names}
-
-    @property
-    def SAGE_values(self):
-        return {feature_name: float(self.SAGE_trackers[feature_name].tracked_value) for feature_name in self.feature_names}
-
-    def _update_sampler(self, x, y):
-        self.sampler.update(x, y)
-
-    def explain_one(self, x_i, y_i):
-        if self.seen_samples < 1:  # TODO remove warmup-phase
-            self.seen_samples += 1
-            if self.default_values is None:
-                self._update_sampler(x_i, y_i)
-            return self.SAGE_values
-        permutation_chain = np.random.permutation(self.feature_names)
-        if self.default_values is None:
-            x_marginal, _ = self.sampler.sample(k=1)
-        else:
-            x_marginal = [self.default_values]
-        marginal_prediction = self.model_fn(x_marginal[0])[0]
-        self.marginal_prediction.update(marginal_prediction)
-        sample_loss = self.loss_function(y_true=y_i, y_prediction=self.marginal_prediction())
-        x_S = {}
-        for feature in permutation_chain:
-            x_S[feature] = x_i[feature]
-            y = 0
-            if self.default_values is None:
-                x_marginals, _ = self.sampler.sample(k=self.sub_sample_size)
-            else:
-                x_marginals = [self.default_values for _ in range(self.sub_sample_size)]
-            k = 0
-            while k < min(self.sub_sample_size, len(x_marginals)):
-                x_marginal = {**x_marginals[k], **x_S}
-                y += self.model_fn(x_marginal)[0]
-                k += 1
-            y /= k
-            feature_loss = self.loss_function(y_true=y_i, y_prediction=y)
-            marginal_contribution = sample_loss - feature_loss
-            self.SAGE_trackers[feature].update(marginal_contribution)
-            sample_loss = feature_loss
-        self.seen_samples += 1
-        if self.default_values is None:
-            self._update_sampler(x_i, y_i)
-        return self.SAGE_values
diff --git a/increment_explain/__init__.py b/increment_explain/__init__.py
new file mode 100644
index 0000000..f38884b
--- /dev/null
+++ b/increment_explain/__init__.py
@@ -0,0 +1,2 @@
+__version__ = "0.1.0"
+
diff --git a/increment_explain/explainer/__init__.py b/increment_explain/explainer/__init__.py
new file mode 100644
index 0000000..fdb916c
--- /dev/null
+++ b/increment_explain/explainer/__init__.py
@@ -0,0 +1,8 @@
+from .base import BaseIncrementalExplainer, BaseIncrementalFeatureImportance
+from .pfi import IncrementalPFI
+
+__all__ = [
+    "BaseIncrementalExplainer",
+    "BaseIncrementalFeatureImportance",
+    "IncrementalPFI"
+]
diff --git a/increment_explain/explainer/base.py b/increment_explain/explainer/base.py
new file mode 100644
index 0000000..079120c
--- /dev/null
+++ b/increment_explain/explainer/base.py
@@ -0,0 +1,157 @@
+"""
+This module gathers base Explanation Methods
+"""
+
+# Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
+#          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
+#          Rohit Jagtani
+
+import copy
+import math
+import abc
+import typing
+
+from increment_explain.imputer import BaseImputer, MarginalImputer
+from increment_explain.storage import GeometricReservoirStorage, UniformReservoirStorage
+from increment_explain.storage.base import BaseStorage
+from increment_explain.utils.trackers import ExponentialSmoothingTracker, WelfordTracker, MultiValueTracker, Tracker
+from increment_explain.utils.validators.loss import validate_loss_function
+
+
+class BaseIncrementalExplainer(metaclass=abc.ABCMeta):
+    """Base class for incremental explainer algorithms.
+
+    Warning: This class should not be used directly.
+    Use derived classes instead.
+
+    Attributes:
+        feature_names (list[str]): List of feature names that are explained.
+        number_of_features (int): Number of features that are explained.
+        seen_samples (int): Number of instances observed.
+    """
+
+    @abc.abstractmethod
+    def __init__(
+            self,
+            model_function: typing.Callable,
+            feature_names: list[str]
+    ):
+        """
+        Args:
+            model_function (Callable): The Model function to be explained.
+            feature_names (list): List of feature names to be explained for the model.
+        """
+        self._model_function = model_function
+        self.feature_names = feature_names
+        self.number_of_features: int = len(feature_names)
+        self.seen_samples: int = 0
+
+    def __repr__(self):
+        return f"Explainer for {self.number_of_features} features after {self.seen_samples} samples."
+
+
+class BaseIncrementalFeatureImportance(BaseIncrementalExplainer):
+    """Base class for incremental feature importance explainer algorithms.
+
+   Warning: This class should not be used directly.
+   Use derived classes instead.
+   """
+
+    @abc.abstractmethod
+    def __init__(
+            self,
+            model_function,
+            loss_function,
+            feature_names,
+            storage: typing.Optional[BaseStorage] = None,
+            imputer: typing.Optional[BaseImputer] = None,
+            dynamic_setting: bool = False,
+            smoothing_alpha: typing.Optional[float] = None
+    ):
+        super().__init__(model_function, feature_names)
+        self._loss_function = validate_loss_function(loss_function, self._model_function)
+
+        self._smoothing_alpha = 0.001 if smoothing_alpha is None else smoothing_alpha
+        if dynamic_setting:
+            assert 0. < smoothing_alpha <= 1., f"The smoothing parameter needs to be in the range of ']0,1]' and not " \
+                                               f"'{self._smoothing_alpha}'."
+            base_tracker = ExponentialSmoothingTracker(alpha=self._smoothing_alpha)
+        else:
+            base_tracker = WelfordTracker()
+        self._marginal_loss_tracker: Tracker = copy.deepcopy(base_tracker)
+        self._model_loss_tracker: Tracker = copy.deepcopy(base_tracker)
+        self._marginal_prediction_tracker: Tracker = MultiValueTracker(base_tracker=copy.deepcopy(base_tracker))
+        self._importance_trackers: dict[str, Tracker] = {
+            feature_name: copy.deepcopy(base_tracker) for feature_name in feature_names}
+        self._variance_trackers: dict[str, Tracker] = {
+            feature_name: copy.deepcopy(base_tracker) for feature_name in feature_names}
+        self._storage: BaseStorage = storage
+        if self._storage is None:
+            if dynamic_setting:
+                self._storage = GeometricReservoirStorage(store_targets=False, size=100)
+            else:
+                self._storage = UniformReservoirStorage(store_targets=False, size=100)
+        self._imputer: BaseImputer = imputer
+        if self._imputer is None:
+            self._imputer = MarginalImputer(self._model_function, 'joint', self._storage)
+
+    @abc.abstractmethod
+    def explain_one(self, *args, **kwargs):
+        raise NotImplementedError
+
+    @property
+    def importance_values(self):
+        """Incremental Importance Values property."""
+        return {feature_name: float(self._importance_trackers[feature_name].tracked_value)
+                for feature_name in self.feature_names}
+
+    @property
+    def variances(self):
+        """Incremental Variances values property."""
+        return {feature_name: self._variance_trackers[feature_name].get() for feature_name in self.feature_names}
+
+    def get_normalized_importance_values(self, mode: str = 'sum') -> dict[str, float]:
+        """Normalizes the importance scores.
+
+        Args:
+            mode (str): The normalization mode to be applied. Possible values are 'sum' and 'delta'.
+                - sum: Normalizes the importance scores by division through the sum of importance scores.
+                - delta: Normalizes the importance scores by division through the difference between the max of the
+                importance scores and the min of the importance scores.
+
+        Returns:
+            (dict[str, float]): The normalized importance values.
+        """
+        return self._normalize_importance_values(self.importance_values, mode=mode)
+
+    def get_confidence_bound(self, delta: float):
+        """Calculates Delta-Confidence Bounds.
+
+        Args:
+            delta (float): The confidence parameter. Must be a value in the interval of ]0,1].
+
+        Returns:
+            (dict[str, float]): The upper confidence bound around the point estimate of the importance values.
+                This value needs to be added to the top and bottom of the point estimate.
+        """
+        assert 0 < delta <= 1., f"Delta must be float in the interval of ]0,1] and not {delta}."
+        return {
+            feature_name:
+                (1 - self._smoothing_alpha) ** self.seen_samples +
+                (1 / math.sqrt(delta)) * math.sqrt(self.variances[feature_name].get()) *
+                math.sqrt(self._smoothing_alpha / (2 - self._smoothing_alpha))
+            for feature_name in self.feature_names}
+
+    @staticmethod
+    def _normalize_importance_values(importance_values: dict[str, float], mode: str = 'sum') -> dict[str, float]:
+        importance_values_list = list(importance_values.values())
+        if mode == 'delta':
+            factor = max(importance_values_list) - min(importance_values_list)
+        elif mode == 'sum':
+            factor = sum(importance_values_list)
+        else:
+            raise NotImplementedError(f"mode must be either 'sum', or 'delta' not '{mode}'")
+        try:
+            return {feature: importance_value / factor for feature, importance_value in importance_values.items()}
+        except ZeroDivisionError:
+            return {feature: 0.0 for feature, importance_value in importance_values.items()}
diff --git a/increment_explain/explainer/pfi.py b/increment_explain/explainer/pfi.py
new file mode 100644
index 0000000..1af4157
--- /dev/null
+++ b/increment_explain/explainer/pfi.py
@@ -0,0 +1,76 @@
+"""
+This module gathers SAGE Explanation Methods
+"""
+
+# Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
+#          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
+#          Rohit Jagtani
+
+from typing import Optional, Union, Callable
+from .base import BaseIncrementalFeatureImportance
+import numpy as np
+from river.metrics.base import Metric
+
+__all__ = [
+    "IncrementalPFI",
+]
+
+
+class IncrementalPFI(BaseIncrementalFeatureImportance):
+
+    def __init__(
+            self,
+            model_function: Callable,
+            feature_names: list[str],
+            storage,
+            imputer,
+            loss_function: Union[Callable, Metric],
+            n_inner_samples: int = 5,
+            smoothing_alpha: float = 0.001,
+            dynamic_setting: bool = True
+    ):
+        super(IncrementalPFI, self).__init__(
+            model_function=model_function,
+            loss_function=loss_function,
+            feature_names=feature_names,
+            dynamic_setting=dynamic_setting,
+            smoothing_alpha=smoothing_alpha,
+            storage=storage,
+            imputer=imputer
+        )
+        self.n_inner_samples = n_inner_samples
+
+    def explain_one(
+            self,
+            x_i,
+            y_i,
+            n_inner_samples: Optional[int] = None,
+            update_storage: bool = True
+    ):
+        if self.seen_samples >= 1:
+            if n_inner_samples is None:
+                n_inner_samples = self.n_inner_samples
+            original_prediction = self._model_function(x_i)[0]
+            original_loss = self._loss_function(y_i, original_prediction)
+            for feature in self.feature_names:
+                feature_subset = [feature]
+                predictions = self._imputer.impute(
+                    feature_subset=feature_subset,
+                    x_i=x_i,
+                    n_samples=n_inner_samples
+                )
+                losses = []
+                for prediction in predictions:
+                    loss = self._loss_function(y_i, prediction[0])
+                    losses.append(loss)
+                avg_loss = np.mean(losses)
+                pfi = avg_loss - original_loss
+                self._update_pfi(pfi, feature)
+                self._variance_trackers[feature].update((pfi - self.importance_values[feature]) ** 2)
+        self.seen_samples += 1
+        if update_storage:
+            self._storage.update(x_i, y_i)
+        return self.importance_values
+
+    def _update_pfi(self, value, feature):
+        self._importance_trackers[feature].update(value)
diff --git a/increment_explain/explainer/sage.py b/increment_explain/explainer/sage.py
new file mode 100644
index 0000000..bc6db18
--- /dev/null
+++ b/increment_explain/explainer/sage.py
@@ -0,0 +1,321 @@
+"""
+This module gathers SAGE Explanation Methods
+"""
+
+# Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
+#          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
+#          Rohit Jagtani
+
+import random
+from typing import Optional, Callable, Any, Union
+
+from river.metrics.base import Metric
+
+
+import numpy as np
+from tqdm import tqdm
+
+from increment_explain.explainer import BaseIncrementalFeatureImportance
+from increment_explain.imputer import MarginalImputer, BaseImputer
+from increment_explain.storage import IntervalStorage, BatchStorage
+from increment_explain.storage.base import BaseStorage
+from increment_explain.utils.validators.loss import validate_loss_function
+
+__all__ = [
+    "IncrementalSage",
+    "BatchSage",
+    "IntervalSage"
+]
+
+# =============================================================================
+# Public SAGE Explainers
+# =============================================================================
+
+
+
+class IncrementalSage(BaseIncrementalFeatureImportance):
+    """Incremental SAGE Explainer
+
+    Computes SAGE importance values incrementally by applying exponential smoothing.
+    For each input instance tuple x_i, y_i one update of the explanation procedure is performed.
+
+    Attributes:
+        marginal_prediction (np.ndarray): The current marginal prediction of the model_function (smoothed over time).
+    """
+
+    def __init__(
+            self,
+            model_function: Callable,
+            loss_function: Union[Callable, Metric],
+            feature_names: list,
+            *,
+            smoothing_alpha: Optional[float] = None,
+            storage: Optional[BaseStorage] = None,
+            imputer: Optional[BaseImputer] = None,
+            n_inner_samples: int = 1,
+            dynamic_setting: bool = True,
+            loss_bigger_is_better: bool = False
+    ):
+        """
+        Args:
+            model_function (Callable): The Model function to be explained.
+            loss_function (Union[Callable, Metric]): The loss function for which the importance values are calculated.
+                This can either be a callable function or a predefined river.metric.base.Metric.
+                - callable function: The loss_function needs to follow the signature of loss_function(y_true, y_pred)
+                    and handle the output dimensions of the model function. Smaller values are interpreted as being
+                    better. y_pred needs to be an array / list with the shape corresponding to the output dimension
+                    (e.g. single-value outputs (e.g. regression, classification): y_pred_1 = [0], y_pred_2 = [1],
+                    etc.; for multi-label outputs (e.g. probability scores) y_pred_1 = [0.72, 0.28],
+                    y_pred_2 = [0.01, 0.99]).
+                - river.metric.base.Metric: Any Metric implemented in river (e.g. river.metrics.CrossEntropy() for
+                    classification or river.metrics.MSE() for regression).
+            feature_names (list): List of feature names to be explained for the model.
+            smoothing_alpha (float): The smoothing parameter for the exponential smoothing of the importance values.
+                Should be in the interval between ]0,1]. Defaults to 0.001.
+            storage (BaseStorage): Optional incremental data storage Mechanism. Defaults to
+                `GeometricReservoirStorage(size=100)` for dynamic modelling settings and to
+                `UniformReservoirStorage(size=100)` in static modelling settings.
+            imputer (BaseImputer): Incremental imputing strategy to be used. Defaults to
+                `MarginalImputer(sampling_strategy='joint')`.
+            n_inner_samples (int): Number of model evaluation per feature and explanation step (observation).
+                Defaults to 1.
+            dynamic_setting (bool): Flag to indicate if the modelling setting is dynamic `True` (changing model, and
+                adaptive explanation) or a static modelling setting `False` (all observations contribute equally to the
+                final importance) is assumed. Defaults to `True`.
+            loss_bigger_is_better (bool): Flag that indicates if a smaller loss value indicates a better fit ('True') or
+                not ('False'). This is only used to represent the marginal- and model-loss more sensibly.
+        """
+        super(IncrementalSage, self).__init__(
+            model_function=model_function,
+            loss_function=loss_function,
+            feature_names=feature_names,
+            dynamic_setting=dynamic_setting,
+            smoothing_alpha=smoothing_alpha,
+            storage=storage,
+            imputer=imputer
+        )
+        self._loss_direction = 1. if loss_bigger_is_better else 0.
+        self.n_inner_samples = n_inner_samples
+        self.marginal_prediction = np.zeros(shape=1)
+
+    @property
+    def marginal_loss(self):
+        """Marginal loss (loss of the model without any features, default prediction loss) property,
+        which is smoothed over time."""
+        return self._marginal_loss_tracker.get() + self._loss_direction
+
+    @property
+    def model_loss(self):
+        """Model loss (loss of model with features) property, which is smoothed over time."""
+        return self._model_loss_tracker.get() + self._loss_direction
+
+    @property
+    def explained_loss(self):
+        """Explained loss (difference between the current marginal and model loss.) property."""
+        return self.marginal_loss - self.model_loss
+
+    def explain_one(
+            self,
+            x_i: dict,
+            y_i: Any,
+            update_storage: bool = True
+    ) -> dict[str, float]:
+        """Explain one observation (x_i, y_i).
+
+        Args:
+            x_i (dict[str, Any]): The input features of the current observation as a dict of feature names to feature
+                values.
+            y_i (Any): Target label of the current observation.
+            update_storage (bool): Flag if the underlying incremental data storage mechanism is to be updated with the
+                new observation (`True`) or not (`False`). Defaults to `True`.
+
+        Returns:
+            (dict[str, float]): The current SAGE feature importance scores.
+        """
+        if self.seen_samples >= 1:
+            permutation_chain = np.random.permutation(self.feature_names)
+            y_i_pred = self._model_function(x_i)[0]
+            model_loss = self._loss_function(y_i, y_i_pred)
+            self._model_loss_tracker.update(model_loss)
+            self.marginal_prediction = self._marginal_prediction_tracker.update(y_i_pred).get_normalized()
+            sample_loss = self._loss_function(y_i, self.marginal_prediction)
+            self._marginal_loss_tracker.update(sample_loss)
+            features_not_in_S = set(self.feature_names)
+            for feature in permutation_chain:
+                features_not_in_S.remove(feature)
+                predictions = self._imputer.impute(
+                    feature_subset=features_not_in_S,
+                    x_i=x_i,
+                    n_samples=self.n_inner_samples
+                )
+                y = np.mean(np.asarray(predictions), axis=0)[0]
+                feature_loss = self._loss_function(y_i, y)
+                marginal_contribution = sample_loss - feature_loss
+                sample_loss = feature_loss
+                self._importance_trackers[feature].update(marginal_contribution)
+                self._variance_trackers[feature].update((marginal_contribution - self.importance_values[feature])**2)
+        self.seen_samples += 1
+        if update_storage:
+            self._storage.update(x_i, y_i)
+        return self.importance_values
+
+
+class BatchSage:
+
+    def __init__(
+            self,
+            model_function: Callable,
+            feature_names: list,
+            loss_function: Union[Callable, Metric],
+            n_inner_samples: int = 1,
+            storage: Optional[BaseStorage] = None,
+            imputer: Optional[BaseImputer] = None,
+    ):
+        """
+        Args:
+            model_function (Callable): The Model function to be explained.
+            loss_function (Union[Callable, Metric]): The loss function for which the importance values are calculated.
+                This can either be a callable function or a predefined river.metric.base.Metric.
+                - callable function: The loss_function needs to follow the signature of loss_function(y_true, y_pred)
+                    and handle the output dimensions of the model function. Smaller values are interpreted as being
+                    better. y_pred needs to be an array / list with the shape corresponding to the output dimension
+                    (e.g. single-value outputs (e.g. regression, classification): y_pred_1 = [0], y_pred_2 = [1],
+                    etc.; for multi-label outputs (e.g. probability scores) y_pred_1 = [0.72, 0.28],
+                    y_pred_2 = [0.01, 0.99]).
+                - river.metric.base.Metric: Any Metric implemented in river (e.g. river.metrics.CrossEntropy() for
+                    classification or river.metrics.MSE() for regression).
+            feature_names (list): List of feature names to be explained for the model.
+            storage (BaseStorage): Optional incremental data storage mechanism. Defaults to 
+                `BatchStorage()`.
+            imputer (BaseImputer): Incremental imputing strategy to be used. Defaults to 
+                `MarginalImputer(sampling_strategy='joint')`.
+            n_inner_samples (int): Number of model evaluation per feature and explanation step (observation).
+                Defaults to 1.
+        """
+        self.feature_names = feature_names
+        self.n_inner_samples = n_inner_samples
+        self._model_function = model_function
+        self._loss_function = validate_loss_function(loss_function, model_function)
+        self._storage: BaseStorage = storage
+        if self._storage is None:
+            self._storage = BatchStorage(store_targets=True)
+        self._imputer: BaseImputer = imputer
+        if self._imputer is None:
+            self._imputer = MarginalImputer(
+                sampling_strategy='joint', model_function=self._model_function, storage_object=self._storage)
+        self.importance_values = {feature_name: 0. for feature_name in self.feature_names}
+
+    def update_storage(self, x_i, y_i):
+        self._storage.update(x=x_i, y=y_i)
+
+    def explain_one(self, x_i, y_i, original_sage: bool = False):
+        self._storage.update(x_i, y_i)
+        x_data, y_data = self._storage.get_data()
+        if original_sage:
+            self.explain_many_original(x_data=x_data, y_data=y_data)
+        else:
+            self.explain_many(x_data=x_data, y_data=y_data)
+        return self.importance_values
+
+    def explain_many(
+            self,
+            x_data: list[dict],
+            y_data: list[Any],
+    ) -> dict[str, float]:
+        sage_values = {feature: 0. for feature in self.feature_names}
+        n_data = len(x_data)
+        marginal_prediction = sum(self._model_function(x_data)) / n_data
+        for n, (x_i, y_i) in tqdm(enumerate(zip(x_data, y_data), start=1), total=n_data):
+            permutation_chain = np.random.permutation(self.feature_names)
+            loss_previous = self._loss_function(y_true=y_i, y_prediction=marginal_prediction)
+            features_not_in_S = set(self.feature_names)
+            for feature in permutation_chain:
+                features_not_in_S.remove(feature)
+                predictions = self._imputer.impute(
+                    feature_subset=features_not_in_S,
+                    x_i=x_i,
+                    n_samples=self.n_inner_samples
+                )
+                y = np.mean(np.asarray(predictions), axis=0)[0]
+                feature_loss = self._loss_function(y_true=y_i, y_prediction=y)
+                marginal_contribution = loss_previous - feature_loss
+                sage_values[feature] += marginal_contribution
+                loss_previous = feature_loss
+            n_data = n
+        self.importance_values = {feature: sage_value / n_data for feature, sage_value in sage_values.items()}
+        return self.importance_values
+
+    def explain_many_original(
+            self,
+            x_data: list[dict[str, Any]],
+            y_data: list[Any],
+    ) -> dict[str, float]:
+        sage_values = {feature: 0. for feature in self.feature_names}
+        n_data = len(x_data)
+        marginal_prediction = sum(self._model_function(x_data)) / n_data
+        for n, (x_i, y_i) in tqdm(enumerate(zip(x_data, y_data), start=1), total=n_data):
+            permutation_chain = np.random.permutation(self.feature_names)
+            x_S = {}
+            loss_previous = self._loss_function(y_true=y_i, y_prediction=marginal_prediction)
+            for feature in permutation_chain:
+                x_S[feature] = x_i[feature]
+                y = 0
+                for k in range(1, self.n_inner_samples + 1):
+                    x_marginal = x_data[random.randint(0, n_data - 1)]
+                    x_marginal = {**x_marginal, **x_S}
+                    y += self._model_function(x_marginal)[0]
+                y /= k
+                feature_loss = self._loss_function(y_true=y_i, y_prediction=y)
+                marginal_contribution = loss_previous - feature_loss
+                sage_values[feature] += marginal_contribution
+                loss_previous = feature_loss
+            n_data = n
+        self.importance_values = {feature: sage_value / n_data for feature, sage_value in sage_values.items()}
+        return self.importance_values
+
+
+class IntervalSage(BatchSage):
+
+    def __init__(
+            self,
+            model_function: Callable,
+            feature_names: list,
+            loss_function: Union[Callable, Metric],
+            n_inner_samples: int = 1,
+            interval_length: int = 100,
+            storage: BaseStorage = None,
+            imputer: BaseImputer = None,
+    ):
+
+        if storage is None:
+            storage = IntervalStorage(store_targets=True, size=interval_length)
+        assert isinstance(storage, IntervalStorage), f"Only 'IntervalStorage' allowed not {type(storage)}."
+
+        if imputer is None:
+            imputer = MarginalImputer(model_function=model_function, sampling_strategy='joint', storage_object=storage)
+
+        super().__init__(
+            model_function=model_function,
+            feature_names=feature_names,
+            loss_function=loss_function,
+            n_inner_samples=n_inner_samples,
+            storage=storage,
+            imputer=imputer
+        )
+        self._interval_length = interval_length
+        self.seen_samples = 0
+
+    def explain_one(
+            self,
+            x_i: dict[str, Any],
+            y_i: Any,
+            sub_sample_size: Optional[int] = None,
+            force_explain: bool = False
+    ) -> dict[str, float]:
+        self._storage.update(x=x_i, y=y_i)
+        self.seen_samples += 1
+        if not force_explain and self.seen_samples % self._interval_length != 0:
+            return self.importance_values
+        x_data, y_data = self._storage.get_data()
+        super().explain_many(x_data=x_data, y_data=y_data)
+        return self.importance_values
diff --git a/explainer/shap.py b/increment_explain/explainer/shap.py
similarity index 98%
rename from explainer/shap.py
rename to increment_explain/explainer/shap.py
index eb838f5..e6ec121 100644
--- a/explainer/shap.py
+++ b/increment_explain/explainer/shap.py
@@ -7,17 +7,16 @@
 
 import copy
 import random
-import time
 from abc import ABCMeta
 from abc import abstractmethod
-from typing import Optional, Union, Iterator, Any, Sequence, Callable
+from typing import Optional, Iterator, Any, Sequence, Callable
 from collections import Counter
 import itertools
 
 import numpy as np
 from scipy.special import binom
 
-from utils.permutations import get_n_feature_masks, get_all_feature_masks
+from increment_explain.utils.permutations import get_n_feature_masks, get_all_feature_masks
 
 __all__ = [
     "KernelSHAP",
diff --git a/increment_explain/imputer/__init__.py b/increment_explain/imputer/__init__.py
new file mode 100644
index 0000000..47ce924
--- /dev/null
+++ b/increment_explain/imputer/__init__.py
@@ -0,0 +1,11 @@
+from .default_imputer import DefaultImputer
+from .marginal_imputer import MarginalImputer
+from .base import BaseImputer
+from .tree_imputer import TreeImputer
+
+__all__ = [
+    "DefaultImputer",
+    "MarginalImputer",
+    "BaseImputer",
+    "TreeImputer"
+]
diff --git a/increment_explain/imputer/base.py b/increment_explain/imputer/base.py
new file mode 100644
index 0000000..c4f6f68
--- /dev/null
+++ b/increment_explain/imputer/base.py
@@ -0,0 +1,21 @@
+from abc import abstractmethod, ABC
+import typing
+
+
+class BaseImputer(ABC):
+    """Base class for sampling algorithms.
+
+    Warning: This class should not be used directly.
+    Use derived classes instead.
+    """
+
+    @abstractmethod
+    def __init__(
+            self,
+            model_function: typing.Callable
+    ):
+        self.model_function = model_function
+
+    @abstractmethod
+    def impute(self, feature_subset, x_i, n_samples=None):
+        raise NotImplementedError
diff --git a/increment_explain/imputer/default_imputer.py b/increment_explain/imputer/default_imputer.py
new file mode 100644
index 0000000..c47e0a5
--- /dev/null
+++ b/increment_explain/imputer/default_imputer.py
@@ -0,0 +1,17 @@
+from typing import Any, Callable
+
+from .base import BaseImputer
+
+
+class DefaultImputer(BaseImputer):
+    def __init__(self, model_function: Callable, values: dict[str, Any]):
+        self.values = values
+        super().__init__(
+            model_function=model_function
+        )
+
+    def impute(self, feature_subset: list[str], x_i: dict, n_samples: int = 1):
+        sampled_values = {feature: self.values[feature] for feature in feature_subset}
+        prediction = self.model_function({**x_i, **sampled_values})
+        prediction = [prediction for _ in range(n_samples)]
+        return prediction
diff --git a/increment_explain/imputer/marginal_imputer.py b/increment_explain/imputer/marginal_imputer.py
new file mode 100644
index 0000000..903e257
--- /dev/null
+++ b/increment_explain/imputer/marginal_imputer.py
@@ -0,0 +1,45 @@
+from .base import BaseImputer
+import random
+
+
+class MarginalImputer(BaseImputer):
+    def __init__(self, model_function, sampling_strategy, storage_object):
+        self.sampling_strategy = sampling_strategy
+        self.storage_object = storage_object
+        super().__init__(
+            model_function=model_function
+        )
+        # TODO - random seed - create separate issue
+
+    def _sample(self, storage_object, feature_subset):
+        features, _ = storage_object.get_data()
+        if self.sampling_strategy == 'joint':
+            sampled_features = self._sample_marginals(features, feature_subset)
+        else:
+            sampled_features = self._sample_product_marginals(features, feature_subset)
+        return sampled_features
+
+    def impute(self, feature_subset, x_i, n_samples=1):
+        predictions = []
+        for _ in range(n_samples):
+            sampled_values = self._sample(self.storage_object, feature_subset)
+            prediction = self.model_function({**x_i, **sampled_values})
+            predictions.append(prediction)
+        return predictions
+
+    @staticmethod
+    def _sample_marginals(features, feature_subset):
+        rand_idx = random.randrange(len(features))
+        sampled_instance = features[rand_idx].copy()
+        sampled_features = {feature_name: sampled_instance[feature_name]
+                            for feature_name in feature_subset}
+        return sampled_features
+
+    @staticmethod
+    def _sample_product_marginals(features, feature_subset):
+        sampled_features = {}
+        for feature_name in feature_subset:
+            rand_idx = random.randrange(len(features))
+            sampled_features[feature_name] = features[
+                            rand_idx].copy()[feature_name]
+        return sampled_features
diff --git a/increment_explain/imputer/tree_imputer.py b/increment_explain/imputer/tree_imputer.py
new file mode 100644
index 0000000..feeaf22
--- /dev/null
+++ b/increment_explain/imputer/tree_imputer.py
@@ -0,0 +1,90 @@
+import math
+from typing import Callable
+from increment_explain.storage.tree_storage import TreeStorage
+from .base import BaseImputer
+from river.base import Classifier, Regressor
+from river.tree import HoeffdingTreeRegressor
+import numpy as np
+import random
+
+EPS = 0.0000001
+
+class TreeImputer(BaseImputer):
+    def __init__(
+            self,
+            model_function: Callable,
+            storage_object: TreeStorage,
+            direct_predict_numeric: bool = False,
+            use_storage: bool = False
+    ):
+        self.storage_object = storage_object
+        self.direct_predict_numeric = direct_predict_numeric
+        self.use_storage = use_storage
+        super().__init__(
+            model_function=model_function
+        )
+
+    def _sample_cat_feature(self, feature_name: str, feature_model: Classifier, x_i: dict, n_samples: int = 1):
+        x_i_pred = {**x_i}
+        _ = x_i_pred.pop(feature_name)
+        probas = feature_model.predict_proba_one(x_i_pred)
+        feature_values = list(probas.keys())
+        feature_weights = list(probas.values())
+        feature_value = random.choices(population=feature_values, weights=feature_weights, k=n_samples)[0]
+        return feature_value
+
+    def _sample_num_feature(self, feature_name: str, feature_model: HoeffdingTreeRegressor, x_i: dict, n_samples: int = 1):
+        x_i_pred = {**x_i}
+        _ = x_i_pred.pop(feature_name)
+        if not self.direct_predict_numeric:
+            if hasattr(feature_model._root, "traverse"):
+                leaf_nodes = feature_model._root.traverse(x_i_pred)
+            else:
+                leaf_nodes = [feature_model._root]
+            leaf_node = leaf_nodes[0]
+            if len(leaf_nodes) > 1:
+                weights = [leaf_node.stats.n for leaf_node in leaf_nodes]
+                leaf_node = random.choices(leaf_nodes, weights=weights)[0]
+            mean = leaf_node.stats.mean.get()
+            std = math.sqrt(abs(leaf_node.stats.get()))
+            feature_value = np.random.normal(loc=mean, scale=std)
+        else:
+            feature_value = feature_model.predict_one(x_i_pred)
+        return feature_value
+
+    def _sample(self, feature_name, x_i, n_samples: int = 1):
+        feature_model, feature_type = self.storage_object(feature_name)
+        if feature_type == 'cat':
+            feature_value = self._sample_cat_feature(feature_name, feature_model, x_i, n_samples=n_samples)
+        else:  # feature_type == 'num'
+            feature_value = self._sample_num_feature(feature_name, feature_model, x_i, n_samples=n_samples)
+        return feature_value
+
+    def _sample_from_storages(self, feature_name, x_i, n_samples: int = 1):
+        feature_model, feature_type = self.storage_object(feature_name)
+        data_reservoir = self.storage_object.data_reservoirs[feature_name]
+        leaf_id = self.storage_object.get_path_through_tree(feature_model._root, x_i)
+        try:
+            storage = data_reservoir[leaf_id]
+            x_storage, _ = storage.get_data()
+            random_index = random.randint(0, len(x_storage) - 1)
+            x_sampled = x_storage[random_index]
+            sampled_feature_value = x_sampled[feature_name]
+        except KeyError:
+            sampled_feature_value = self._sample(feature_name=feature_name, x_i=x_i)
+        return sampled_feature_value
+
+    def impute(self, feature_subset, x_i, n_samples: int = 1):
+        x_original_data = {feature_name: x_i[feature_name] for feature_name in x_i.keys() if feature_name not in feature_subset}
+        predictions = []
+        for _ in range(n_samples):
+            sampled_values = {}
+            for feature_name in feature_subset:
+                if self.use_storage:
+                    sampled_value = self._sample_from_storages(feature_name, x_i, n_samples=n_samples)
+                else:
+                    sampled_value = self._sample(feature_name, x_i, n_samples=n_samples)
+                sampled_values[feature_name] = sampled_value
+            prediction = self.model_function({**x_i, **sampled_values})
+            predictions.append(prediction)
+        return predictions
diff --git a/increment_explain/storage/__init__.py b/increment_explain/storage/__init__.py
new file mode 100644
index 0000000..78e529b
--- /dev/null
+++ b/increment_explain/storage/__init__.py
@@ -0,0 +1,17 @@
+from .batch_storage import BatchStorage
+from .interval_storage import IntervalStorage
+from .reservoir_storage import ReservoirStorage
+from .uniform_reservoir_storage import UniformReservoirStorage
+from .geometric_reservoir_storage import GeometricReservoirStorage
+from .sequence_storage import SequenceStorage
+from .tree_storage import TreeStorage
+
+__all__ = [
+    "UniformReservoirStorage",
+    "GeometricReservoirStorage",
+    "ReservoirStorage",
+    "BatchStorage",
+    "IntervalStorage",
+    "SequenceStorage",
+    "TreeStorage"
+]
diff --git a/increment_explain/storage/base.py b/increment_explain/storage/base.py
new file mode 100644
index 0000000..ec0747a
--- /dev/null
+++ b/increment_explain/storage/base.py
@@ -0,0 +1,43 @@
+from typing import List, Dict, Optional, Any
+from abc import abstractmethod, ABC
+
+
+class BaseStorage(ABC):
+    """Base class for sampling algorithms.
+
+    Warning: This class should not be used directly.
+    Use derived classes instead.
+    """
+    @abstractmethod
+    def __init__(self):
+        self._storage_x: List[Dict[str, Any]] = []
+        self._storage_y: List = []
+
+    @abstractmethod
+    def update(self, x: Dict[str, Any], y: Optional[Any]):
+        """Given a data point, it updates the storage.
+
+        Args:
+            x: Features as Dict of feature names (keys) and feature values (values)
+            y: Target as float or integer
+            
+        Returns:
+            None
+        """
+        raise NotImplementedError
+
+    def __len__(self):
+        """Returns size of storage object
+
+        Returns:
+            Number of objects in storage.
+        """
+        return len(self._storage_x)
+
+    def get_data(self):
+        """Fetches data from storage.
+
+        Returns:
+            List of Features and targets in storage as tuple.
+        """
+        return (self._storage_x, self._storage_y)
diff --git a/increment_explain/storage/batch_storage.py b/increment_explain/storage/batch_storage.py
new file mode 100644
index 0000000..08df30c
--- /dev/null
+++ b/increment_explain/storage/batch_storage.py
@@ -0,0 +1,23 @@
+from .base import BaseStorage
+from typing import Dict, Any, Optional
+
+
+class BatchStorage(BaseStorage):
+    """ A Batch Storage storing all seen samples.
+    """
+
+    def __init__(self, store_targets=True):
+        self.store_targets = store_targets
+        super().__init__()
+
+    def update(self, x: Dict, y: Optional[Any] = None):
+        """Given a data point, it updates the storage.
+        Args:
+            x: Features as List of Dicts
+            y: Target as float or integer
+        Returns:
+            None
+        """
+        self._storage_x.append(x)
+        if self.store_targets:
+            self._storage_y.append(y)
diff --git a/increment_explain/storage/geometric_reservoir_storage.py b/increment_explain/storage/geometric_reservoir_storage.py
new file mode 100644
index 0000000..1829e4b
--- /dev/null
+++ b/increment_explain/storage/geometric_reservoir_storage.py
@@ -0,0 +1,37 @@
+from .reservoir_storage import ReservoirStorage
+from typing import Dict, Optional, Any
+import random
+
+
+class GeometricReservoirStorage(ReservoirStorage):
+    """ Geometric Reservoir Storage
+    """
+
+    def __init__(
+            self,
+            size: int,
+            constant_probability: float = None,
+            store_targets: bool = False
+    ):
+        super().__init__(
+            size=size,
+            store_targets=store_targets
+        )
+        super(ReservoirStorage, self).__init__()
+        if constant_probability is not None:
+            self.constant_probability = constant_probability
+        else:
+            self.constant_probability = 1 / self.size
+
+    def update(self, x: Dict, y: Optional[Any] = None):
+        if len(self._storage_x) < self.size:
+            self._storage_x.append(x)
+            if self.store_targets:
+                self._storage_y.append(y)
+        else:
+            random_float = random.random()
+            if random_float <= self.constant_probability:
+                rand_idx = random.randrange(self.size)
+                self._storage_x[rand_idx] = x
+                if self.store_targets:
+                    self._storage_y[rand_idx] = y
diff --git a/increment_explain/storage/interval_storage.py b/increment_explain/storage/interval_storage.py
new file mode 100644
index 0000000..42c443f
--- /dev/null
+++ b/increment_explain/storage/interval_storage.py
@@ -0,0 +1,33 @@
+from .base import BaseStorage
+from collections import deque
+from typing import Dict, Optional, Any
+
+
+class IntervalStorage(BaseStorage):
+    """ An Interval Storage storing last k samples.
+    """
+
+    def __init__(
+            self,
+            size: int,
+            store_targets: bool = True
+    ):
+        self.size = size
+        self.store_targets = store_targets
+        self._storage_x = deque()
+        self._storage_y = deque()
+
+    def update(self, x: Dict, y: Optional[Any] = None):
+        if len(self._storage_x) < self.size:
+            self._storage_x.append(x)
+            if self.store_targets:
+                self._storage_y.append(y)
+        else:
+            self._storage_x.popleft()
+            self._storage_x.append(x)
+            if self.store_targets:
+                self._storage_y.popleft()
+                self._storage_y.append(y)
+
+    def get_data(self):
+        return list(self._storage_x), list(self._storage_y)
diff --git a/increment_explain/storage/reservoir_storage.py b/increment_explain/storage/reservoir_storage.py
new file mode 100644
index 0000000..b80a57b
--- /dev/null
+++ b/increment_explain/storage/reservoir_storage.py
@@ -0,0 +1,19 @@
+from abc import ABC
+from .base import BaseStorage
+
+
+class ReservoirStorage(BaseStorage, ABC):
+    """Reservoir Storage - base class
+
+    Attributes:
+        size int: Size of the reservoir.
+        store_targets bool: Flag if the target values are also stored.
+    """
+
+    def __init__(
+            self,
+            size: int,
+            store_targets: bool = False
+    ):
+        self.size = size
+        self.store_targets = store_targets
diff --git a/increment_explain/storage/sequence_storage.py b/increment_explain/storage/sequence_storage.py
new file mode 100644
index 0000000..209e30a
--- /dev/null
+++ b/increment_explain/storage/sequence_storage.py
@@ -0,0 +1,15 @@
+from .interval_storage import IntervalStorage
+
+
+class SequenceStorage(IntervalStorage):
+    """ An Interval Storage storing the last sample.
+    """
+
+    def __init__(
+            self,
+            store_targets: bool = True
+    ):
+        super().__init__(
+            store_targets=store_targets,
+            size=1
+        )
diff --git a/increment_explain/storage/tree_storage.py b/increment_explain/storage/tree_storage.py
new file mode 100644
index 0000000..f565e94
--- /dev/null
+++ b/increment_explain/storage/tree_storage.py
@@ -0,0 +1,214 @@
+import typing
+from typing import Dict, Any, Optional, List, Union, Tuple
+import random
+import numpy as np
+
+from river import base
+from river.tree import HoeffdingAdaptiveTreeClassifier, HoeffdingAdaptiveTreeRegressor, \
+    HoeffdingTreeRegressor, HoeffdingTreeClassifier
+from river.metrics import R2, Accuracy
+from river.utils import Rolling
+
+
+from .geometric_reservoir_storage import GeometricReservoirStorage
+from ..utils.trackers import WelfordTracker
+from .base import BaseStorage
+
+
+def walk_through_tree(node, x_i, until_leaf=True) -> typing.Iterable[typing.Union["Branch", "Leaf"]]:
+    """Traverses a decision tree given a data point, and a starting node.
+
+    Args:
+        node: Target as float or integer
+        x_i (dict[str, Any]): Data point as Dicts.
+        until_leaf (bool): Flag weather to traverse the tree until a leaf node (``True``) or
+            just the next node (``False``).
+
+    Yields:
+        The next node in the tree.
+    """
+    yield node
+    try:
+        yield from walk_through_tree(node.next(x_i), x_i, until_leaf)
+    except KeyError:
+        if until_leaf:
+            children = node.children
+            ratios = [child.total_weight for child in children]
+            node = random.choices(children, weights=ratios, k=1)[0]
+            yield node
+            yield from walk_through_tree(node, x_i, until_leaf)
+    except AttributeError:  # we are at a leaf node -> which was already returned
+        pass
+
+
+class MeanVarRegressor(base.Regressor):
+    """A simple regressor model intended to be used as a leaf model in Decision Tree Regressors.
+
+    The Regressor keeps track of the mean and standard deviation of the incoming numerical labels and samples
+    prediction values from a normal distribution according to the current mean and standard deviation.
+    """
+
+    def __init__(self):
+        self._stat_object = WelfordTracker()
+
+    def predict_one(self, x=None) -> float:
+        """Predicts a value based on the current summary statistics.
+
+        Args:
+            x (Any): input features (that are not used for prediction)
+
+        Returns:
+            float: The predicted value.
+        """
+        mean = self._stat_object.mean
+        std = self._stat_object.std
+        prediction = np.random.normal(loc=mean, scale=abs(std), size=1)[0]
+        return prediction
+
+    def learn_one(self, x: Any, y: base.typing.RegTarget) -> "base.Regressor":
+        """Updates the summary statistics based on the target labels.
+
+        Args:
+            x (Any): input features (that are not used for prediction)
+            y (base.typing.RegTarget): A number that is transformable into a float.
+
+        Returns:
+            base.Regressor: The Regressor itself.
+        """
+        y_i = float(y)
+        self._stat_object.update(value_i=y_i)
+        return self
+
+
+class TreeStorage(BaseStorage):
+    """ A Tree Storage that trains incremental decision trees for each feature.
+
+    Attributes:
+        feature_names (list[str]): List of features stored.
+        cat_feature_names (list[str]): List of categorical features stored.
+        num_feature_names (list[str]): List of numerical features stored.
+        performances (dict[str, Union[R2, Accuracy]]): Dictionary of performance metrics per incremental decision tree
+            for each feature stored.
+        data_reservoirs (dict[str, dict]): Dictionary of data reservoirs for each feature and leaf nodes.
+    """
+
+    def __init__(self, cat_feature_names: List[str], num_feature_names: List[str],
+                 max_depth: int = 5, leaf_reservoir_length: int = 10):
+        """ A Tree Storage that trains incremental decision trees for each feature.
+
+        Args:
+            cat_feature_names (list[str]): List of categorical features to be stored.
+            num_feature_names (list[str]): List of numerical features to be stored.
+            max_depth (int): Maximum tree depth for the incremental decision trees. Defaults to 5.
+            leaf_reservoir_length (int): Size of the reservoir stored at each leaf node of each feature's incremental
+                decision tree. Defaults to 10.
+        """
+        self.feature_names = cat_feature_names + num_feature_names
+        self.cat_feature_names = cat_feature_names
+        self.num_feature_names = num_feature_names
+        self._leaf_reservoir_length = leaf_reservoir_length
+
+        self._storage_x = {cat_feature: HoeffdingAdaptiveTreeClassifier(
+            max_depth=max_depth, leaf_prediction='nba', binary_split=True)
+            for cat_feature in self.cat_feature_names}
+        self._storage_x.update(
+            {num_feature: HoeffdingAdaptiveTreeRegressor(
+                max_depth=max_depth, leaf_prediction='adaptive', binary_split=True)
+                for num_feature in self.num_feature_names})
+
+        self.performances = {num_feature: Rolling(R2(), window_size=1000) for num_feature in self.num_feature_names}
+        self.performances.update(
+            {cat_feature: Rolling(Accuracy(), window_size=1000) for cat_feature in self.cat_feature_names})
+
+        self.data_reservoirs = {feature: {} for feature in self.feature_names}
+
+    def update(self, x: Dict, y: Optional[Any] = None):
+        """Given a data point, it updates the storage.
+
+        Args:
+            x: Features as List of Dicts
+            y: Target as float or integer
+
+        Returns:
+            None
+        """
+        for feature_name, feature_value in x.items():
+            if feature_name in self.feature_names:
+                feature_model = self._storage_x[feature_name]
+                x_i = {**x}
+                y_i = x_i.pop(feature_name)
+                feature_model.learn_one(x_i, y_i)
+                self._update_data_reservoirs(feature_name, x_i, x)
+                pred_i = feature_model.predict_one(x_i)
+                self.performances[feature_name].update(y_i, pred_i)
+
+    @staticmethod
+    def get_path_through_tree(node, x_i) -> str:
+        """Given a data point and a starting node, traverses the decision tree.
+
+        Args:
+            node: Root node of the model.
+            x_i: Data point to traverse the tree with.
+
+        Returns:
+            str: The walked path through the decision tree.
+        """
+        walked_path = ''
+        path = iter(walk_through_tree(node, x_i, until_leaf=True))
+        for stop in path:
+            walked_path += str(stop)
+            if hasattr(stop, 'repr_split'):
+                walked_path += "|" + str(stop.repr_split) + "|" + str(stop.branch_no(x_i))
+            walked_path += "|STOP|"
+        return walked_path
+
+    def _update_data_reservoirs(self, feature_name, x_i, x):
+        """Update the data reservoir at a leaf node with a new sample.
+
+        Args:
+            feature_name str: The feature for which to update the stored data reservoirs.
+            x_i dict: The data point to find the leaf node with.
+            x dict: The data point to be inserted in the leaf node's reservoir.
+        """
+        root_node = self._storage_x[feature_name]._root
+        data_reservoir = self.data_reservoirs[feature_name]
+        leaf_id = self.get_path_through_tree(root_node, x_i)
+        if leaf_id not in data_reservoir:
+            data_reservoir[leaf_id] = GeometricReservoirStorage(
+                size=self._leaf_reservoir_length, store_targets=False, constant_probability=1.0)
+            self._delete_old_reservoir_if_necessary(leaf_id, feature_name)
+        data_reservoir[leaf_id].update(x)
+
+    def __call__(self, feature_name: str) -> Tuple[Union[HoeffdingTreeRegressor, HoeffdingTreeClassifier], str]:
+        """Given a feature name, returns the associated data reservoirs.
+
+        Args:
+            feature_name (str): The feature name for which to return the data reservoirs.
+
+        Returns:
+            (dict, str): Tuple of data reservoir and flag if it is stored as a numerical feature or categorical.
+
+        Raises:
+            ValueError: If `feature_name` is not stored as a categorical feature nor a numerical feature.
+        """
+        if feature_name in self.cat_feature_names:
+            return self._storage_x[feature_name], 'cat'
+        elif feature_name in self.num_feature_names:
+            return self._storage_x[feature_name], 'num'
+        else:
+            raise ValueError(f"The {feature_name} is not stored.")
+
+    def _delete_old_reservoir_if_necessary(self, leaf_id: str, feature_name: str):
+        """Delete the reservoir that is outdated if a new path is added.
+
+        Args:
+            leaf_id (str): The str path through the tree where each nodes are seperated with a `|STOP|`.
+            feature_name (str): The feature name for which outdated leafs might be deleted.
+        """
+
+        steps = leaf_id.split('|STOP|')[0:-1]
+        previous_reservoir_id = '|STOP|'.join([*steps[0:-2], steps[-1]]) + '|STOP|'
+        if len(steps) > 1:
+            reservoir_ids = set(self.data_reservoirs[feature_name].keys())
+            if previous_reservoir_id in reservoir_ids:
+                del self.data_reservoirs[feature_name][previous_reservoir_id]
diff --git a/increment_explain/storage/uniform_reservoir_storage.py b/increment_explain/storage/uniform_reservoir_storage.py
new file mode 100644
index 0000000..aefa692
--- /dev/null
+++ b/increment_explain/storage/uniform_reservoir_storage.py
@@ -0,0 +1,37 @@
+from .reservoir_storage import ReservoirStorage
+from typing import Dict, Optional, Any
+import random
+
+
+class UniformReservoirStorage(ReservoirStorage):
+    """ Uniform Reservoir Storage
+
+    Attributes:
+        stored_samples int: Number of samples seen in the stream.
+    """
+
+    def __init__(
+            self,
+            store_targets: bool,
+            size: int
+    ):
+        super().__init__(
+            size=size,
+            store_targets=store_targets
+        )
+        super(ReservoirStorage, self).__init__()
+        self.stored_samples: int = 0
+
+    def update(self, x: Dict, y: Optional[Any] = None):
+        self.stored_samples += 1
+        if len(self._storage_x) < self.size:
+            self._storage_x.append(x)
+            if self.store_targets:
+                self._storage_y.append(y)
+        else:
+            random_integer = random.randint(1, self.stored_samples)
+            if random_integer <= self.size:
+                rand_idx = random.randrange(self.size)
+                self._storage_x[rand_idx] = x
+                if self.store_targets:
+                    self._storage_y[rand_idx] = y
diff --git a/utils/__init__.py b/increment_explain/utils/__init__.py
similarity index 100%
rename from utils/__init__.py
rename to increment_explain/utils/__init__.py
diff --git a/utils/permutations.py b/increment_explain/utils/permutations.py
similarity index 100%
rename from utils/permutations.py
rename to increment_explain/utils/permutations.py
diff --git a/utils/trackers.py b/increment_explain/utils/trackers.py
similarity index 66%
rename from utils/trackers.py
rename to increment_explain/utils/trackers.py
index 8ced8eb..785f3b1 100644
--- a/utils/trackers.py
+++ b/increment_explain/utils/trackers.py
@@ -4,7 +4,10 @@
 
 # Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
 #          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
+#          Rohit Jagtani
 
+import copy
+import abc
 from typing import Union
 import numpy as np
 
@@ -13,6 +16,8 @@
     "WelfordTracker",
     "ExponentialSmoothingTracker",
     "SlidingWindowTracker",
+    "MultiValueTracker",
+    "Tracker"
 ]
 
 
@@ -21,25 +26,71 @@
 # =============================================================================
 
 
-class BaseTracker:
+class Tracker(abc.ABC):
 
+    @abc.abstractmethod
     def __init__(self):
         self.tracked_value = 0
+        self.N = 0
 
     def __repr__(self):
-        return f"{round(self.tracked_value, 2)}"
+        return f"{np.round(self.tracked_value, 2)}"
 
     def __call__(self, *args, **kwargs):
         return self.tracked_value
 
+    def get(self):
+        return self.tracked_value
+
+    @abc.abstractmethod
+    def update(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def get_normalized(self):
+        return self.get()
+
 
 # =============================================================================
 # Public Tracker Class
 # =============================================================================
 
 
-# TODO write tests for all Trackers
-class WelfordTracker(BaseTracker):
+class MultiValueTracker(Tracker):
+
+    def __init__(self, base_tracker: Tracker):
+        super().__init__()
+        self.tracked_value: list[Tracker] = [0]
+        self._tracked_indices = 0
+        self._base_tracker = base_tracker
+
+    def update(self, values: list[float]):
+        if len(values) > self._tracked_indices:
+            if self._tracked_indices == 0:
+                self.tracked_value: list[Tracker] = []
+            for _ in range(len(values) - self._tracked_indices):
+                self.tracked_value.append(copy.deepcopy(self._base_tracker))
+                self._tracked_indices += 1
+        for i in range(self._tracked_indices):
+            self.tracked_value[i].update(values[i])
+        self.N += 1
+        return self
+
+    def __call__(self):
+        return np.asarray([self.tracked_value[i].get() for i in range(self._tracked_indices)])
+
+    def get(self):
+        return self()
+
+    def get_normalized(self):
+        if self._tracked_indices <= 1:
+            return self.get()
+        try:
+            return self.get() / sum(self.get())
+        except ZeroDivisionError:
+            return 0.0
+
+
+class WelfordTracker(Tracker):
     """ A Tracker that applies Welford's Algorithm to estimate the mean and variance of a sequence.
 
     Notes
@@ -48,7 +99,6 @@ class WelfordTracker(BaseTracker):
     """
     def __init__(self):
         super().__init__()
-        self.N = 0
         self.sum_squares = 0
 
     def update(self, value_i: Union[int, float]):
@@ -64,6 +114,7 @@ def update(self, value_i: Union[int, float]):
         self.tracked_value += difference_1 / self.N
         difference_2 = value_i - self.tracked_value
         self.sum_squares += difference_1 * difference_2
+        return self
 
     @property
     def var(self):
@@ -77,11 +128,11 @@ def std(self):
 
     @property
     def mean(self):
-        """Returns the mean the stream"""
+        """Returns the mean of the stream"""
         return self.tracked_value
 
 
-class ExponentialSmoothingTracker(BaseTracker):
+class ExponentialSmoothingTracker(Tracker):
     """A Tracker that applies Exponential Smoothing on the numeric input values"""
 
     def __init__(self, alpha: float):
@@ -98,6 +149,8 @@ def update(self, value_i: Union[int, float]):
             numeric value to be added to the tracker
         """
         self.tracked_value = (1 - self.alpha) * self.tracked_value + self.alpha * value_i
+        self.N += 1
+        return self
 
 
 class SlidingWindowTracker:
@@ -108,7 +161,7 @@ def __init__(self, k: int):
         assert 0 < k, "The 'window_size' must be greater than zero."
         self.window_k = 0
         self.k = k
-        self.sliding_window = np.empty(shape=self.k)
+        self.sliding_window = np.array([np.NaN for _ in range(self.k)])
 
     def update(self, value_i: Union[int, float]):
         """Adds one value to the Tracker
@@ -124,6 +177,7 @@ def update(self, value_i: Union[int, float]):
         else:
             self.window_k = 0
             self.sliding_window[self.window_k] = value_i
+        return self
 
     def __call__(self, *args, **kwargs):
         """Returns the current mean of the sliding window"""
@@ -135,14 +189,14 @@ def __repr__(self):
     @property
     def mean(self):
         """Returns the current mean of the sliding window"""
-        return float(np.mean(self.sliding_window, axis=0))
+        return float(np.nanmean(self.sliding_window, axis=0))
 
     @property
     def var(self):
         """Returns the variance of the sliding window"""
-        return float(np.var(self.sliding_window, axis=0))
+        return float(np.nanvar(self.sliding_window, axis=0))
 
     @property
     def std(self):
         """Returns the standard deviation of the sliding window"""
-        return float(np.std(self.sliding_window, axis=0))
+        return float(np.nanstd(self.sliding_window, axis=0))
diff --git a/utils/tests/__init__.py b/increment_explain/utils/validators/__init__.py
similarity index 100%
rename from utils/tests/__init__.py
rename to increment_explain/utils/validators/__init__.py
diff --git a/increment_explain/utils/validators/loss.py b/increment_explain/utils/validators/loss.py
new file mode 100644
index 0000000..3deadda
--- /dev/null
+++ b/increment_explain/utils/validators/loss.py
@@ -0,0 +1,71 @@
+import typing
+import abc
+import river
+from river.metrics.base import Metric
+
+
+class RiverMetricToLossFunction:
+
+    def __init__(self, river_metric: river.metrics.base.Metric):
+        self._river_metric = river_metric
+        self._sign = 1.
+        if hasattr(self._river_metric, "bigger_is_better") and self._river_metric.bigger_is_better:
+            self._sign = -1.
+
+    @abc.abstractmethod
+    def __call__(self, y_true, y_prediction):
+        pass
+
+
+class RiverDictLabelMetricToLossFunction(RiverMetricToLossFunction):
+
+    def __init__(self, river_metric: river.metrics.base.Metric):
+        super().__init__(river_metric)
+
+    def __call__(self, y_true, y_prediction):
+        y_prediction = {i: y_prediction[i] for i in range(0, len(y_prediction))}
+        loss_i = self._river_metric.update(y_true=y_true, y_pred=y_prediction).get()
+        self._river_metric.revert(y_true=y_true, y_pred=y_prediction)
+        return loss_i * self._sign
+
+
+class RiverSingletonMetricToLossFunction(RiverMetricToLossFunction):
+
+    def __init__(self, river_metric: river.metrics.base.Metric):
+        super().__init__(river_metric)
+
+    def __call__(self, y_true, y_prediction):
+        try:
+            y_prediction = float(y_prediction[0])
+        except TypeError:
+            y_prediction = float(y_prediction)
+        loss_i = self._river_metric.update(y_true=y_true, y_pred=y_prediction).get()
+        self._river_metric.revert(y_true=y_true, y_pred=y_prediction)
+        return loss_i * self._sign
+
+
+def _get_loss_function_from_river_metric(river_metric: Metric):
+    try:
+        _ = river_metric.update(y_true=0, y_pred={0: 1}).revert(y_true=0, y_pred={0: 1})
+        validated_loss_function = RiverDictLabelMetricToLossFunction(river_metric=river_metric)
+        _ = validated_loss_function(0, [0])
+        _ = validated_loss_function(0, [0, 0])
+        return validated_loss_function
+    except:
+        pass
+    try:
+        _ = river_metric.update(y_true=0, y_pred=0).revert(y_true=0, y_pred=0)
+        validated_loss_function = RiverSingletonMetricToLossFunction(river_metric=river_metric)
+        _ = validated_loss_function(0, 0)
+        _ = validated_loss_function(0, [0])
+        return validated_loss_function
+    except:
+        pass
+    raise ValueError(f"Provided river metric cannot be transformed into a viable loss function.")
+
+
+def validate_loss_function(loss_function: typing.Union[typing.Callable, Metric], model_function: typing.Callable):
+    validated_loss_function = loss_function
+    if isinstance(loss_function, river.metrics.base.Metric):  # loss function is a river metric object
+        validated_loss_function = _get_loss_function_from_river_metric(river_metric=loss_function)
+    return validated_loss_function
diff --git a/increment_explain/utils/wrappers/__init__.py b/increment_explain/utils/wrappers/__init__.py
new file mode 100644
index 0000000..da68e73
--- /dev/null
+++ b/increment_explain/utils/wrappers/__init__.py
@@ -0,0 +1,8 @@
+from increment_explain.utils.wrappers.torch import TorchSupervisedLearningWrapper
+from increment_explain.utils.wrappers.river import RiverToPredictionFunction, PredictionFunctionToRiverInput
+
+__all__ = [
+    "TorchSupervisedLearningWrapper",
+    "RiverToPredictionFunction",
+    "PredictionFunctionToRiverInput"
+]
diff --git a/increment_explain/utils/wrappers/river.py b/increment_explain/utils/wrappers/river.py
new file mode 100644
index 0000000..8ecf36e
--- /dev/null
+++ b/increment_explain/utils/wrappers/river.py
@@ -0,0 +1,43 @@
+import copy
+import typing
+
+import numpy as np
+
+
+class RiverToPredictionFunction:
+
+    def __init__(self, river_predict_function: typing.Callable):
+        self._river_predict_function = river_predict_function
+        self.max_labels = 1
+
+    def _reduce_dict(self, y_prediction):
+        if not isinstance(y_prediction, dict):
+            return np.asarray([y_prediction])
+        self.max_labels = max(self.max_labels, max(y_prediction))
+        y_arr = np.zeros(shape=self.max_labels + 1)
+        for key, value in y_prediction.items():
+            y_arr[key] = value
+        return y_arr
+
+    def __call__(self, x: list[dict]):
+        x_input = copy.copy(x)
+        if isinstance(x_input, dict):
+            x_input = [x_input]
+        predictions = [self._reduce_dict(self._river_predict_function(x_i)) for x_i in x_input]
+        return predictions
+
+
+class PredictionFunctionToRiverInput:
+
+    def __init__(self, prediction_function: typing.Callable) :
+        self.prediction_function = prediction_function
+
+    def __call__(self, x: typing.Union[dict[str], np.ndarray]):
+        if isinstance(x, dict):
+            x_list = list(x.values())
+            y_pred = np.asarray([self.prediction_function(x_list)])
+        else:
+            y_pred = np.empty(shape=len(x))
+            for i in range(len(x)):
+                y_pred[i] = self.prediction_function(x[i])
+        return y_pred
diff --git a/increment_explain/utils/wrappers/torch.py b/increment_explain/utils/wrappers/torch.py
new file mode 100644
index 0000000..dde969a
--- /dev/null
+++ b/increment_explain/utils/wrappers/torch.py
@@ -0,0 +1,47 @@
+import torch
+
+
+class TorchSupervisedLearningWrapper:
+
+    def __init__(self, model, optimizer, loss_function, n_classes: int = 2, class_labels: list[str] = None):
+        self.model = model
+        self.optimizer = optimizer
+        self.loss_function = loss_function
+        self._supervised = True
+        self.n_classes = n_classes
+        if class_labels is not None:
+            self.n_classes = len(class_labels)
+            self.class_labels = class_labels
+        else:
+            self.class_labels = [i for i in range(self.n_classes)]
+        self.device = 'cpu'
+
+    def _dict_to_tensor(self, x_i):
+        return torch.tensor([list(x_i.values())], dtype=torch.float32, device=self.device)
+
+    def _tensor_to_probas(self, prediction_tensor):
+        probas_tensor = torch.softmax(prediction_tensor, dim=-1)
+        probas_dict = {class_label: float(probas_tensor[0][i])
+                       for class_label, i in zip(self.class_labels, range(self.n_classes))}
+        return probas_dict
+
+    def predict_one(self, x_i):
+        x_tensor = self._dict_to_tensor(x_i)
+        return int(torch.argmax(self.model(x_tensor)))
+
+    def predict_proba_one(self, x_i):
+        x_tensor = self._dict_to_tensor(x_i)
+        y_prediction = self.model(x_tensor)
+        y_prediction = self._tensor_to_probas(y_prediction)
+        return y_prediction
+
+    def learn_one(self, x, y):
+        x_tensor = torch.tensor([list(x.values())], dtype=torch.float32)
+        self.model.train()
+        self.optimizer.zero_grad()
+        y_pred = self.model(x_tensor)
+        y_tensor = torch.zeros(1, self.n_classes, dtype=torch.float32)
+        y_tensor[0, int(y)] = 1.
+        loss = self.loss_function(y_pred, y_tensor)
+        loss.backward()
+        self.optimizer.step()
diff --git a/increment_explain/visualization/__init__.py b/increment_explain/visualization/__init__.py
new file mode 100644
index 0000000..8877090
--- /dev/null
+++ b/increment_explain/visualization/__init__.py
@@ -0,0 +1,5 @@
+from .plotting import FeatureImportancePlotter
+
+__all__ = [
+    "FeatureImportancePlotter",
+]
\ No newline at end of file
diff --git a/increment_explain/visualization/color.py b/increment_explain/visualization/color.py
new file mode 100644
index 0000000..953d7bc
--- /dev/null
+++ b/increment_explain/visualization/color.py
@@ -0,0 +1,33 @@
+from typing import Optional, Generator, Any, Iterable
+
+BASE_COLOR = '#a6a7a9'  # base color for elements in a plot that should not be highlighted / colored
+BACKGROUND_COLOR = '#f8f8f8'  # background color of plot objects (canvas)
+
+DEFAULT_COLOR_LIST = ['#ef27a6', '#4ea5d9', '#7d53de', '#44cfcb', '#44001A']  # default list of colors for highlighting elements
+# in a plot
+
+STD_ALPHA = 0.25  # alpha-channel value for std tunnel around an element (line) for a plot
+
+
+def get_color_with_generator(
+        color_generator: Generator[str, None, None],
+        item_id: Optional[Any] = None,
+        color_item_ids: Optional[list[Any]] = None,
+        base_color: Optional[str] = None
+) -> str:
+    if base_color is None:
+        base_color = BASE_COLOR
+    if color_item_ids is not None and item_id not in color_item_ids:
+        return base_color
+    else:
+        return next(color_generator)
+
+
+def color_list_generator(
+        color_list: Optional[list[str]] = None
+) -> Iterable[str]:
+    if color_list is None:
+        color_list = DEFAULT_COLOR_LIST
+    color_list = color_list * 100  # upper bound
+    for color in color_list:
+        yield color
diff --git a/increment_explain/visualization/config.py b/increment_explain/visualization/config.py
new file mode 100644
index 0000000..a259594
--- /dev/null
+++ b/increment_explain/visualization/config.py
@@ -0,0 +1,1 @@
+SAVE_DPI = 300
diff --git a/visualization/line_plots.py b/increment_explain/visualization/line_plots.py
similarity index 54%
rename from visualization/line_plots.py
rename to increment_explain/visualization/line_plots.py
index beddbe8..bbaa176 100644
--- a/visualization/line_plots.py
+++ b/increment_explain/visualization/line_plots.py
@@ -4,8 +4,7 @@
 import pandas as pd
 import matplotlib.pyplot as plt
 
-
-from visualization.color import color_list_generator, get_color_with_generator, STD_ALPHA, BACKGROUND_COLOR, BASE_COLOR
+from increment_explain.visualization.color import color_list_generator, get_color_with_generator, BACKGROUND_COLOR, BASE_COLOR, STD_ALPHA
 
 
 def _validate_y_x_data(y_data: Union[dict, Sequence],
@@ -26,6 +25,7 @@ def _validate_y_x_data(y_data: Union[dict, Sequence],
                 line_names = ['X' + str(i) for i in range(len(y_data_facet))]
             y_data[y_key] = pd.DataFrame(y_data_facet, columns=line_names)
 
+
     if x_data is not None and isinstance(x_data, dict):
         x_keys = list(x_data.keys())
     else:
@@ -62,30 +62,39 @@ def _validate_std(std: Optional[dict],
     return std
 
 
-def plot_multi_line_graph(axis: plt.axis,
-                          y_data: Union[dict, Sequence],
-                          show: bool = False,
-                          x_data: Optional[Union[dict, Sequence]] = None,
-                          line_names: Optional[Union[dict, Sequence]] = None,
-                          names_to_highlight: Optional[list[str]] = None,
-                          line_styles: Optional[dict[str, str]] = None,
-                          std: Optional[Union[dict, Sequence]] = None,
-                          title: Optional[str] = None,
-                          y_label: Optional[str] = None,
-                          x_label: Optional[str] = None,
-                          y_ticks: Optional[list[Union[int, float]]] = None,
-                          x_ticks: Optional[list[Union[int, float]]] = None,
-                          y_min: Optional[Union[int, float]] = None,
-                          y_max: Optional[Union[int, float]] = None,
-                          x_min: Optional[Union[int, float]] = None,
-                          x_max: Optional[Union[int, float]] = None,
-                          legend_style: Optional[dict] = None,
-                          legend: Optional[dict[str, dict]] = None,
-                          base_color: Optional[str] = None,
-                          color_list: Optional[list[str]] = None,
-                          h_lines: Optional[list[dict]] = None,
-                          v_lines: Optional[list[dict]] = None
-                          ) -> Optional[plt.axis]:
+def plot_multi_line_graph(
+        axis: plt.axis,
+        y_data: Union[dict, Sequence],
+        *,
+        x_data: Optional[Union[dict, Sequence]] = None,
+        line_names: Optional[Union[dict, Sequence]] = None,
+        names_to_highlight: Optional[list[str]] = None,
+        facet_not_to_highlight: Optional[list[str]] = None,
+        line_styles: Optional[dict[str, str]] = None,
+        std: Optional[Union[dict, Sequence]] = None,
+        title: Optional[str] = None,
+        y_label: Optional[str] = None,
+        x_label: Optional[str] = None,
+        y_ticks: Optional[list[Union[int, float]]] = None,
+        x_ticks: Optional[list[Union[int, float]]] = None,
+        y_min: Optional[Union[int, float]] = None,
+        y_max: Optional[Union[int, float]] = None,
+        x_min: Optional[Union[int, float]] = None,
+        x_max: Optional[Union[int, float]] = None,
+        legend_style: Optional[dict] = None,
+        secondary_legends: Optional[list[dict[str, dict]]] = None,
+        base_color: Optional[str] = None,
+        color_list: Optional[list[str]] = None,
+        h_lines: Optional[list[dict]] = None,
+        v_lines: Optional[list[dict]] = None,
+        markevery: Optional[dict[str, int]] = None,
+        tick_right: bool = False,
+        fill_between_props: Optional[list[dict]] = None
+) -> plt.axis:
+
+    if facet_not_to_highlight is None:
+        facet_not_to_highlight = []
+
     if names_to_highlight is None:
         names_to_highlight = line_names if line_names is not None else []
 
@@ -101,6 +110,9 @@ def plot_multi_line_graph(axis: plt.axis,
     if line_styles is None:
         line_styles = {facet: '-' for facet in y_data.keys()}
 
+    if markevery is None:
+        markevery = {facet: 1 for facet in y_data.keys()}
+
     line_colors = {}
 
     for facet, y_facet in y_data.items():
@@ -112,38 +124,76 @@ def plot_multi_line_graph(axis: plt.axis,
         for line_name in line_names:
             color_line = get_color_with_generator(color_generator=color_gens[facet],
                                                   base_color=base_color,
-                                                  item=line_name,
-                                                  item_selection=names_to_highlight)
+                                                  item_id=line_name,
+                                                  color_item_ids=names_to_highlight)
             line_colors[line_name] = color_line
-            alpha = 1. if line_name in names_to_highlight else 0.6
-            axis.plot(x_facet,
-                      y_facet[line_name],
+            alpha = 1.
+            if line_name not in names_to_highlight or facet in facet_not_to_highlight:
+                alpha = 0.6
+            axis.plot(x_facet[::markevery[facet]],
+                      y_facet[line_name][::markevery[facet]],
                       ls=line_styles[facet],
                       c=color_line,
                       alpha=alpha,
                       linewidth=1)
             if std_facet is not None:
-                axis.fill_between(x_facet,
-                                  y_facet[line_name] - std_facet[line_name],
-                                  y_facet[line_name] + std_facet[line_name],
+                std_alpha = STD_ALPHA
+                if line_name not in names_to_highlight:
+                    std_alpha = 0.
+                axis.fill_between(x_facet[::markevery[facet]],
+                                  y_facet[line_name][::markevery[facet]] - std_facet[line_name][::markevery[facet]],
+                                  y_facet[line_name][::markevery[facet]] + std_facet[line_name][::markevery[facet]],
                                   color=color_line,
-                                  alpha=STD_ALPHA,
+                                  alpha=std_alpha,
                                   linewidth=0.)
 
+    if fill_between_props is not None:
+        for fill_between_prop in fill_between_props:
+            facet_1 = fill_between_prop['facet_1']
+            facet_2 = fill_between_prop['facet_2']
+            line_name_1 = fill_between_prop['line_name_1']
+            line_name_2 = fill_between_prop['line_name_2']
+            y_data_line_1 = y_data[facet_1][line_name_1][::markevery[facet_1]]
+            y_data_line_2 = y_data[facet_2][line_name_2][::markevery[facet_2]]
+            x_data_line = x_data[facet_1][::markevery[facet_1]]
+
+            color = None if 'color' not in fill_between_prop.keys() else fill_between_prop['color']
+            alpha = None if 'alpha' not in fill_between_prop.keys() else fill_between_prop['alpha']
+            hatch = None if 'hatch' not in fill_between_prop.keys() else fill_between_prop['hatch']
+
+            axis.fill_between(x_data_line, y_data_line_1, y_data_line_2,
+                              color=color,
+                              alpha=alpha,
+                              hatch=hatch,
+                              linewidth=0.)
+
     if legend_style is not None:
         for line_name in names_to_highlight:
             axis.plot([], label=line_name, color=line_colors[line_name])
-        axis.plot([], label='others', color=BASE_COLOR)
+        if len(names_to_highlight) < len(line_names):
+            axis.plot([], label='others', color=BASE_COLOR)
         axis.legend(edgecolor="0.8", fancybox=False, **legend_style)
 
-    if legend is not None:
-        for legend_item in legend['legend_items']:
-            axis.plot([], label=legend_item[0], color=legend_item[2], ls=legend_item[1])
-            axis.legend(edgecolor="0.8", fancybox=False, **legend['legend_props'])
+    if secondary_legends is not None:
+        for seconday_legend in secondary_legends:
+            ax2 = axis.twinx()
+            legend_lines = []
+            legend_labels = []
+            for legend_item in seconday_legend['legend_items']:
+                legend_labels.append(legend_item[0])
+                legend_line, = ax2.plot([], label=legend_item[0], color=legend_item[2], ls=legend_item[1])
+                legend_lines.append(legend_line)
+            ax2.get_yaxis().set_visible(False)
+            ax2.legend(legend_lines, legend_labels, edgecolor="0.8", fancybox=False, **seconday_legend['legend_props'])
+
 
     # name the y-axis
     if y_label is not None:
-        axis.set_ylabel(y_label)
+        if tick_right:
+            axis.set_ylabel(y_label, rotation=-90)
+            axis.yaxis.set_label_position("right")
+        else:
+            axis.set_ylabel(y_label)
 
     # name the x-axis
     if x_label is not None:
@@ -172,6 +222,8 @@ def plot_multi_line_graph(axis: plt.axis,
     # specify ticks of the y-axis
     if y_ticks is not None:
         axis.set_yticks(y_ticks)
+    if tick_right:
+        axis.yaxis.tick_right()
 
     # specify ticks of the x-axis
     if x_ticks is not None:
@@ -189,10 +241,6 @@ def plot_multi_line_graph(axis: plt.axis,
 
     axis.set_facecolor(BACKGROUND_COLOR)
 
-    if show:
-        plt.show()
-        return None
-
     return axis
 
 
@@ -201,15 +249,16 @@ def plot_multi_line_graph(axis: plt.axis,
 ########################################################################################################################
 
 
-def stacked_plots(axes_function: Union[callable, list[callable]],
-                  data: list[list[dict]],  # TODO think about indexing
-                  ncols: int = 1,
-                  nrows: int = 1,
-                  height_ratios: Optional[list[float]] = None,
-                  width_ratios: Optional[list[float]] = None,
-                  figsize=(10, 5),
-                  title: Optional[str] = None,
-                  show: bool = False):
+def stacked_plots(
+        axes_function: Union[callable, list[callable]],
+        data: dict[int, dict[int, dict]],  # TODO think about indexing
+        ncols: int = 1,
+        nrows: int = 1,
+        height_ratios: Optional[list[float]] = None,
+        width_ratios: Optional[list[float]] = None,
+        figsize=(10, 5),
+        title: Optional[str] = None
+) -> tuple[plt.figure, list[plt.axis]]:
     fig, axes = plt.subplots(nrows=nrows, ncols=ncols, sharex='col', sharey='row',
                              figsize=figsize,
                              gridspec_kw={'height_ratios': height_ratios,
@@ -222,8 +271,5 @@ def stacked_plots(axes_function: Union[callable, list[callable]],
 
     if title is not None:
         fig.suptitle(title)
-    if show:
-        plt.show()
-        return None
     else:
         return fig, axes
diff --git a/increment_explain/visualization/plotting.py b/increment_explain/visualization/plotting.py
new file mode 100644
index 0000000..31405be
--- /dev/null
+++ b/increment_explain/visualization/plotting.py
@@ -0,0 +1,115 @@
+import copy
+from abc import ABCMeta, abstractmethod
+from typing import Optional, Union, Sequence
+
+import matplotlib.pyplot as plt
+
+from increment_explain.visualization.config import SAVE_DPI
+from increment_explain.visualization.line_plots import plot_multi_line_graph
+
+__all__ = [
+    "FeatureImportancePlotter"
+]
+
+
+class BasePlotter(metaclass=ABCMeta):
+
+    def __init__(self) -> None:
+        self.seen_timesteps = 0
+
+    def __repr__(self):
+        return f"Plotter with {self.seen_timesteps} items."
+
+    def save(self,
+             save_file_path: Optional[str] = None,
+             save_dpi: Optional[int] = SAVE_DPI) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def update(self, *args, **kwargs) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def plot(self, *args, **kwargs) -> None:
+        raise NotImplementedError
+
+
+class FeatureImportancePlotter(BasePlotter):
+
+    def __init__(self,
+                 feature_names: Optional[list[str]] = None):
+        super().__init__()
+        self.feature_names = feature_names
+        self.y_data = {}
+        self.x_data = {}
+        self.stored_facets = set()
+
+    def _create_new_facet(self, facet_name: str):
+        self.stored_facets.add(facet_name)
+        self.y_data[facet_name] = []
+        self.x_data[facet_name] = []
+
+    def update(self,
+               importance_values: dict[str, Union[int, float]],
+               facet_name: Optional[str] = None,
+               timestep: Optional[Union[int, float]] = None):
+        if timestep is None:
+            self.seen_timesteps = self.seen_timesteps + 1
+        if facet_name is None:
+            facet_name = 'importance_values'
+        if facet_name not in self.stored_facets:
+            self._create_new_facet(facet_name)
+        self.y_data[facet_name].append(importance_values)
+        self.x_data[facet_name].append(timestep)
+
+    def plot(
+            self,
+            figsize: Optional[tuple[int, int]] = None,
+            model_performances: Optional[dict[str, Sequence]] = None,
+            performance_kw: Optional[dict] = None,
+            save_name: Optional[str] = None,
+            **line_kw,
+    ) -> None:
+
+        if 'names_to_highlight' not in line_kw:
+            line_kw['names_to_highlight'] = self.feature_names
+        if 'line_names' not in line_kw:
+            line_kw['line_names'] = self.feature_names
+
+        if model_performances is not None:
+            fig, (performance_axis, fi_axis) = plt.subplots(
+                nrows=2, ncols=1, sharex='col', sharey='row',
+                figsize=figsize, gridspec_kw={'height_ratios': [1, 4]}
+            )
+            plt.subplots_adjust(hspace=0.000, wspace=0.000)
+            title = None
+            if 'title' in line_kw:
+                title = copy.copy(line_kw['title'])
+                del line_kw['title']
+            v_lines = None
+            if 'v_lines' in line_kw:
+                v_lines = line_kw['v_lines']
+            performance_kw = {} if performance_kw is None else performance_kw
+            performance_axis = plot_multi_line_graph(
+                axis=performance_axis,
+                y_data=model_performances,
+                title=title,
+                v_lines=v_lines,
+                **performance_kw
+            )
+        else:
+            fig, fi_axis = plt.subplots()
+
+        if figsize is not None:
+            fig.set_figheight(figsize[0])
+            fig.set_figwidth(figsize[1])
+
+        fi_axis = plot_multi_line_graph(
+            axis=fi_axis,
+            y_data=self.y_data,
+            #x_data=self.x_data, TODO fix bug it's not plotting
+            **line_kw
+        )
+        if save_name is not None:
+            plt.savefig(save_name, dpi=200)
+        plt.show()
diff --git a/neural_network.py b/neural_network.py
new file mode 100644
index 0000000..7622151
--- /dev/null
+++ b/neural_network.py
@@ -0,0 +1,127 @@
+import river.metrics
+from river.utils import Rolling
+from river import preprocessing, compose
+from river.datasets import Elec2
+
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IncrementalSage, IntervalSage
+from increment_explain.imputer import MarginalImputer
+from increment_explain.storage import GeometricReservoirStorage
+from increment_explain.utils.wrappers.river import RiverToPredictionFunction
+from increment_explain.utils.wrappers.torch import TorchSupervisedLearningWrapper
+
+N_SAMPLES = 10_000
+
+
+class Net(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.layer_1 = nn.Linear(N_INPUT, 100)
+        self.layer_2 = nn.Linear(100, 10)
+        self.layer_3 = nn.Linear(10, N_CLASSES)
+
+    def forward(self, x):
+        x = F.relu(self.layer_1(x))
+        x = F.relu(self.layer_2(x))
+        x = self.layer_3(x)
+        return x
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    stream = Elec2()#Agrawal(classification_function=1, seed=42)
+    feature_names = list([x_0 for x_0, _ in stream.take(1)][0].keys())
+
+    N_INPUT = len(feature_names)
+    N_CLASSES = 3
+    network = Net()
+    network_loss_function = nn.CrossEntropyLoss()
+    network_optimizer = optim.SGD(network.parameters(), lr=0.001, momentum=0.9)
+
+    loss_metric = river.metrics.Accuracy()
+    training_metric = Rolling(river.metrics.Accuracy(), window_size=1000)
+
+    model = TorchSupervisedLearningWrapper(
+        model=network, loss_function=network_loss_function, optimizer=network_optimizer, n_classes=N_CLASSES)
+    model = compose.Pipeline(preprocessing.StandardScaler(), model)
+    model_function = RiverToPredictionFunction(model.predict_one)
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    storage = GeometricReservoirStorage(
+        size=200,
+        store_targets=False
+    )
+
+    imputer = MarginalImputer(
+        model_function=model_function,
+        storage_object=storage,
+        sampling_strategy="joint"
+    )
+
+    incremental_sage = IncrementalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=5,
+        loss_bigger_is_better=True
+    )
+
+    interval_sage = IntervalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        feature_names=feature_names,
+        interval_length=1000,
+        n_inner_samples=5
+    )
+
+    incremental_pfi = IncrementalPFI(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=5
+    )
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+
+        # here we manually 'extend' the binary classification problem to a multi-class classification problem
+        # by artificially introducing another class whenever the 'period' feature exceeds 0.5 -> this does not make
+        # any sense and is just for illustrative purposes
+        if x_i['period'] >= 0.5:
+            y_i = 2
+
+        # predicting
+        y_i_pred = model.predict_one(x_i)
+        training_metric.update(y_true=y_i, y_pred=y_i_pred)
+
+        # sage inc
+        _ = incremental_sage.explain_one(x_i, y_i)
+        _ = interval_sage.explain_one(x_i, y_i)
+        _ = incremental_pfi.explain_one(x_i, y_i, update_storage=False)
+
+        # learning
+        model.learn_one(x_i, y_i)
+
+        if n % 1000 == 0:
+            print(f"{n}: perf                {training_metric.get()}\n"
+                  f"{n}: x_i                 {x_i}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: int-sage            {interval_sage.importance_values}\n"
+                  f"{n}: pfi-sage            {incremental_pfi.importance_values}\n"
+                  f"{n}: diff                {incremental_sage.marginal_loss - incremental_sage.model_loss}\n"
+                  f"{n}: marginal-loss       {incremental_sage.marginal_loss}\n"
+                  f"{n}: model-loss          {incremental_sage.model_loss}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n")
+
+        if n >= N_SAMPLES:
+            break
diff --git a/pandas_example.py b/pandas_example.py
new file mode 100644
index 0000000..09432ea
--- /dev/null
+++ b/pandas_example.py
@@ -0,0 +1,98 @@
+import pandas as pd
+
+from river.stream import iter_pandas
+from river.ensemble import AdaptiveRandomForestRegressor
+from river.metrics import MSE
+from river.utils import Rolling
+from river import preprocessing, compose
+
+from sklearn.datasets import fetch_california_housing
+from sklearn.utils import shuffle
+
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IncrementalSage, IntervalSage
+from increment_explain.imputer import MarginalImputer
+from increment_explain.storage import GeometricReservoirStorage
+from increment_explain.utils.wrappers import RiverToPredictionFunction
+
+if __name__ == "__main__":
+
+    data = fetch_california_housing()
+    data_x = pd.DataFrame(data['data'], columns=data['feature_names'])
+    data_y = pd.Series(data['target'], name='target')
+    data_y = data_y - data_y.mean() / (data_y.max() - data_y.min())
+    data_x, data_y = shuffle(data_x, data_y)  # of course don't shuffle real stream data
+
+    feature_names = data['feature_names']
+
+    # model
+    model = AdaptiveRandomForestRegressor(n_models=15)  # most important hyperparameter is n_models
+    model = compose.Pipeline(preprocessing.StandardScaler(), model)
+    model_function = RiverToPredictionFunction(model.predict_one)
+
+    # performance metric and loss
+    loss_metric = MSE()  # real metric can be everything from river.metrics used for explanations
+    training_metric = Rolling(MSE(), window_size=1000) # metric wrapper to summarize individual prediction losses
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    storage = GeometricReservoirStorage(
+        size=200,
+        store_targets=False
+    )
+
+    imputer = MarginalImputer(
+        model_function=model_function,
+        storage_object=storage,
+        sampling_strategy="joint"
+    )
+
+    incremental_sage = IncrementalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=1
+    )
+
+    interval_sage = IntervalSage(
+        model_function=model_function,
+        loss_function=loss_metric,
+        feature_names=feature_names,
+        interval_length=2000,
+        n_inner_samples=1
+    )
+
+    incremental_pfi = IncrementalPFI(
+        model_function=model_function,
+        loss_function=loss_metric,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=0.001,
+        n_inner_samples=1
+    )
+
+    # main training loop data_x and data_y needs to be pd.DataFrames and / or pd.Series
+    for n, (x_i, y_i) in enumerate(iter_pandas(data_x, data_y), start=1):
+        y_i_pred = model.predict_one(x_i)
+        training_metric.update(y_true=y_i, y_pred=y_i_pred)
+
+        # sage inc
+        _ = incremental_sage.explain_one(x_i, y_i)
+        _ = interval_sage.explain_one(x_i, y_i)
+        _ = incremental_pfi.explain_one(x_i, y_i, update_storage=False)
+
+        model.learn_one(x_i, y_i)
+
+        if n % 1000 == 0:
+            print(f"{n}: perf                {training_metric.get()}\n"
+                  f"{n}: x_i, y_i            {x_i, y_i}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: int-sage            {interval_sage.importance_values}\n"
+                  f"{n}: pfi                 {incremental_pfi.importance_values}\n"
+                  f"{n}: diff                {incremental_sage.marginal_loss - incremental_sage.model_loss}\n"
+                  f"{n}: marginal-loss       {incremental_sage.marginal_loss}\n"
+                  f"{n}: model-loss          {incremental_sage.model_loss}\n"
+                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n")
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..c2cf558
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,15 @@
+# Increment Explain
+
+This is the first iteration of our incremental explainer package.
+
+Currently, it includes two explanation methods: PFI and SAGE.
+
+Please look at `pandas_example.py`, `neural_network.py`, `agrawal_cross_entropy_loss.py`, and `agrawal_accuracy_loss.py`
+for some example usages.
+
+Please help us in improving our work by contributing or pointing to issues. We will update this iteration soon with further information.
+
+## Installation
+Currently, no pip installation is possible. Please clone this repository and install the required packages as they are needed. 
+`torch` for example is only required if torch models are to be explained (see the wrappers for this). Because of the new typing module this code will only work for rather new python versions. It is tested on python 3.9.7.
+We will soon update this for better compatibility with existing code.
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..39d9d75
Binary files /dev/null and b/requirements.txt differ
diff --git a/sampling/sampler.py b/sampling/sampler.py
deleted file mode 100644
index c7c1383..0000000
--- a/sampling/sampler.py
+++ /dev/null
@@ -1,382 +0,0 @@
-"""
-This module gathers Sampling Methods that are meant for Incremental Learning
-"""
-
-# Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
-#          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
-import copy
-from abc import ABCMeta
-from abc import abstractmethod
-import random
-from collections import Counter, defaultdict
-import numpy as np
-import pandas as pd
-
-__all__ = [
-    "HistogramSampler",
-    "ReservoirSampler",
-    "BatchSampler",
-]
-
-# =============================================================================
-# Types and constants
-# =============================================================================
-
-EPS = 1e-10
-
-# =============================================================================
-# Base Sampler Class
-# =============================================================================
-
-
-class BaseSampler(metaclass=ABCMeta):
-    """Base class for sampling algorithms.
-
-    Warning: This class should not be used directly.
-    Use derived classes instead.
-    """
-
-    @abstractmethod
-    def __init__(
-            self,
-            *,
-            feature_names,
-            store_targets,
-            random_state
-    ):
-        self.feature_names = feature_names
-        self.store_targets = store_targets
-        self.random_state = random_state
-        self.stored_samples = 0
-        self.number_of_features = len(feature_names)
-
-    @abstractmethod
-    def update(self, x):
-        pass
-
-    @abstractmethod
-    def sample(self, k):
-        pass
-
-# =============================================================================
-# Public Samplers
-# =============================================================================
-
-
-class BatchSampler(BaseSampler):
-    """ A Batch Sampler storing all seen samples.
-
-    Parameters
-    ----------
-    feature_names : list(str)
-        The names of the features.
-
-    store_targets : bool, default=True
-        Controls if target values (labels) should be stored and sampled.
-        ``True`` denotes that target values are stored and sampled.
-        ``False`` denotes that target values are neither stored nor sampled.
-
-    sampling_strategy : {"joint", "product"}, default="joint"
-    The strategy used to sample data points from the storage.
-    ``"joint"`` joint marginal distribution is sampled. Features are sampled together.
-    ``"product"`` sampling from the product of the marginal distribution is performed. Features are sampled
-    independently of each other.
-
-    random_state : int, or None, default=None
-        Sets the random seed of the histogram
-
-    Attributes
-    ----------
-    seen_samples : int
-        number of samples stored in the batch sampler
-
-    Notes
-    -----
-    This should be used for working with small incrementalized 'datasets' instead of real 'data streams'.
-    Sampling with replacement is performed.
-    """
-
-    def __init__(
-            self,
-            *,
-            feature_names,
-            store_targets=True,
-            sampling_strategy='joint',
-            random_state=None
-    ):
-        super().__init__(
-            feature_names=feature_names,
-            store_targets=store_targets,
-            random_state=random_state
-        )
-        assert sampling_strategy in ['joint', 'product']
-        if sampling_strategy == 'product':
-            # TODO implement product and conditional sampling
-            raise NotImplementedError("'product' sampling is not implemented yet.")
-        self.sampling_strategy = sampling_strategy
-        self._storage_x = []
-        self._storage_y = []
-
-    def update(self, x, y=None):
-        self._storage_x.append(x)
-        self._storage_y.append(y)
-        self.stored_samples += 1
-
-    def sample(self, k=None, last_k=True, sampling_strategy=None):
-        if sampling_strategy is None:
-            sampling_strategy = self.sampling_strategy
-        if k is None:
-            k = self.stored_samples
-        if last_k:
-            return self._sample_last_k_jointly(k)
-
-        if sampling_strategy == 'joint':
-            return self._sample_k_jointly(k)
-        return self._sample_k_product(k)
-
-    def _sample_last_k_jointly(self, k):
-        return self._storage_x[-k:], self._storage_y[-k:]
-
-    def _sample_k_jointly(self, k):
-        sample_indices = random.sample(list(range(0, self.stored_samples)), k)
-        sampled_x = [self._storage_x[index] for index in sample_indices]
-        sampled_y = [self._storage_y[index] for index in sample_indices]
-        return sampled_x, sampled_y
-
-    def _sample_k_product(self, k):
-        sampled_x_dict = {}
-        for feature in self.feature_names:
-            sample_indices = random.sample(list(range(0, self.stored_samples)), k)
-            sampled_features = [self._storage_x[index][feature] for index in sample_indices]
-            sampled_x_dict[feature] = copy.deepcopy(sampled_features)
-        sampled_x = []
-        for sample_i in range(k):
-            sampled_x.append({feature: sampled_x_dict[feature][sample_i] for feature in self.feature_names})
-        return sampled_x, None
-
-
-class HistogramSampler(BaseSampler):
-    """ A Histogram Sampling Method
-
-    Parameters
-    ----------
-    feature_names : list(str)
-        The names of the feature names.
-
-    sample_with_replacement : bool
-        Controls what sampling strategy to use.
-        ``True`` denotes sampling with replacement.
-        ``False`` denotes sampling without replacement.
-
-    random_state : int, or None
-        Sets the random seed of the histogram
-
-    Attributes
-    ----------
-    histograms_ : dict
-        The histogram storing the frequency of the observed feature values
-
-    seen_samples : int
-        number of samples stored in the histogram
-
-    Notes
-    -----
-    Sampling with histograms essentially is sampling with the product of the marginal distribution, as samples for the
-    individual features are drawn independently of each other and from the features.
-    """
-
-    def __init__(
-            self,
-            *,
-            feature_names,
-            sample_with_replacement=True,
-            random_state=None,
-    ):
-        super().__init__(
-            feature_names=feature_names,
-            store_targets=False,
-            random_state=random_state  # doesn't do anything atm
-        )
-        self.histograms_ = {}
-        for feature in self.feature_names:
-            self.histograms_[feature] = Counter()
-        self.sample_with_replacement = sample_with_replacement
-
-    def update(self, x):
-        """Adds the feature values of the sample to the histogram sampler.
-
-        Parameters
-        ----------
-        x : dict of feature-value pairs
-            individual sample of features to store in the histogram
-        """
-        # TODO Add binning support for numeric features
-        self.stored_samples += 1
-        for feature in self.feature_names:
-            self.histograms_[feature].update([x[feature]])
-
-    def sample(self, k):
-        """Sample k features values independently of each other.
-
-        Parameters
-        ----------
-        k : int greater or equal to 1
-            Number of samples to draw.
-
-        Returns
-        -------
-        x_samples : ndarray of dicts of feature value pairs
-            sampled features value pairs
-        """
-        x_samples = []
-        for _ in range(k):
-            sampled_features = {}
-            for feature in self.feature_names:
-                sampled_feature_value = self._sample_from_feature(feature_name=feature)
-                sampled_features[feature] = sampled_feature_value
-            x_samples.append(sampled_features)
-            self.stored_samples -= 1
-        x_samples = np.asarray(x_samples)
-        return x_samples
-
-    def _sample_from_feature(self, feature_name):
-        """Samples one value for a given feature.
-
-        Parameters
-        ----------
-        feature_name : str
-            name of the feature to draw a sample from
-
-        Returns
-        -------
-        sampled_feature_value : object
-            marginalized feature value
-        """
-        feature_values, sample_weights = zip(*self.histograms_[feature_name].items())
-        sampled_feature_value = random.sample(feature_values, k=1, counts=sample_weights)[0]
-        if self.sample_with_replacement:
-            self.histograms_[feature_name].subtract([sampled_feature_value])
-        return sampled_feature_value
-
-
-class ReservoirSampler(BaseSampler):
-    """An incremental Reservoir Sampler.
-
-    """
-
-    def __init__(
-            self,
-            *,
-            feature_names,
-            reservoir_length=1000,
-            reservoir_mode='normal',
-            sample_features_independently=False,
-            constant_probability=None,
-            sample_with_replacement=True,
-            store_targets=True,
-            random_state=None
-    ):
-        super().__init__(
-            feature_names=feature_names,
-            store_targets=store_targets,
-            random_state=random_state
-        )
-        self.reservoir_length = reservoir_length
-        assert reservoir_mode in ['normal', 'constant', 'skip']
-        self.reservoir_mode = reservoir_mode
-        self.sample_features_independently = sample_features_independently
-        self.sample_with_replacement = sample_with_replacement
-        if constant_probability is not None:
-            self.constant_probability = constant_probability
-        else:
-            self.constant_probability = 1 / reservoir_length
-        if self.sample_with_replacement and self.constant_probability < 1:
-            raise ValueError(
-                "The probability of adding a new sample must be set to 1, "
-                "if the sampling strategy is sampling with replacement.")
-        self.reservoir_x = {feature_name: [] for feature_name in self.feature_names}
-        self.reservoir_y = []
-
-    @property
-    def n_reservoir_samples(self):
-        return min(self.stored_samples, self.reservoir_length)
-
-    def update(self, x, y=None):
-        self.stored_samples += 1
-        x = dict((feature, x[feature]) for feature in self.feature_names)
-        if self.stored_samples <= self.reservoir_length:
-            self._add_x_to_reservoir(x)
-            if self.store_targets:
-                self.reservoir_y.append(y)
-        else:
-            if self.reservoir_mode == 'constant':
-                self._update_constant(x, y)
-            else:
-                self._update_normally(x, y)
-
-    def sample(self, k=1):
-        k = min(self.n_reservoir_samples, k)
-        if self.sample_features_independently:
-            x_samples = self._sample_independently(k)
-            y_samples = None
-        else:
-            x_samples, y_samples = self._sample_jointly(k)
-        return x_samples, y_samples
-
-    def _add_x_to_reservoir(self, x, insertion_position=None):
-        if not insertion_position:
-            for feature in x.keys():
-                self.reservoir_x[feature].append(x[feature])
-        else:
-            for feature in x.keys():
-                self.reservoir_x[feature][insertion_position] = x[feature]
-
-    def _add_sample_to_full_reservoir(self, x, y=None):
-        if self.sample_with_replacement:
-            self._add_x_to_reservoir(x)
-            if self.store_targets:
-                self.reservoir_y.append(y)
-        else:
-            random_insertion_position = random.randint(0, self.reservoir_length - 1)
-            self._add_x_to_reservoir(x, insertion_position=random_insertion_position)
-            if self.store_targets:
-                self.reservoir_y[random_insertion_position] = y
-
-    def _update_constant(self, x, y=None):
-        random_float = random.random()
-        if random_float <= self.constant_probability:
-            self._add_sample_to_full_reservoir(x, y)
-
-    def _update_normally(self, x, y=None):
-        random_integer = random.randint(1, self.stored_samples)
-        if random_integer <= self.reservoir_length:
-            self._add_sample_to_full_reservoir(x, y)
-
-    def _sample_jointly(self, k):
-        sample_indices = random.sample(list(range(0, self.n_reservoir_samples)), k)
-        x_samples = []
-        y_samples = []
-        if self.sample_with_replacement:
-            for sample_index in sample_indices:
-                x_samples.append({feature: self.reservoir_x[feature].pop(sample_index) for feature in self.feature_names})
-        else:
-            for sample_index in sample_indices:
-                x_samples.append({feature: self.reservoir_x[feature][sample_index] for feature in self.feature_names})
-        if self.store_targets:
-            if self.sample_with_replacement:
-                y_samples = [self.reservoir_y.pop(sub_sample_index) for sub_sample_index in sample_indices]
-            else:
-                y_samples = [self.reservoir_y[sub_sample_index] for sub_sample_index in sample_indices]
-        return x_samples, y_samples
-
-    def _sample_independently(self, k):
-        x_samples = []
-        if self.sample_with_replacement:
-            for feature in self.feature_names:
-                sample_indices = random.sample(list(range(0, self.stored_samples)), k)
-                x_samples.append({feature: self.reservoir_x[feature].pop(sample_index) for sample_index in sample_indices})
-        else:
-            for feature in self.feature_names:
-                sample_indices = random.sample(list(range(0, self.stored_samples)), k)
-                x_samples.append({feature: self.reservoir_x[feature][sample_index] for sample_index in sample_indices})
-        return x_samples
diff --git a/utils/converters.py b/utils/converters.py
deleted file mode 100644
index 05a4738..0000000
--- a/utils/converters.py
+++ /dev/null
@@ -1,54 +0,0 @@
-"""
-This module gathers converter classes and functions to transform river models into sklearn-like functions and
-vice versa.
-"""
-
-# Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
-#          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
-
-import numpy as np
-from typing import Union, Sequence, Callable
-from river.base import Regressor, Classifier
-from river import stream
-
-
-class RiverToPredictionFunction:
-
-    def __init__(self,
-                 river_model: Union[Regressor, Classifier],
-                 feature_names: list[str]
-                 ) -> None:
-        self.river_model = river_model
-        self.feature_names = feature_names
-
-    def predict(self, x: Sequence) -> np.ndarray:
-        if isinstance(x, np.ndarray):
-            y_pred = np.empty(shape=len(x))
-            for i, (x_i, _) in enumerate(stream.iter_array(x, feature_names=self.feature_names)):
-                if i > len(x):
-                    break
-                y_pred[i] = self.river_model.predict_one(x_i)
-        elif isinstance(x, dict):
-            y_pred = np.asarray([self.river_model.predict_one(x)])
-        else:
-            # TODO implement for non-numpy x inputs also a fast prediction
-            raise NotImplementedError("Currently only numpy arrays can be used for prediction.")
-        return y_pred
-
-
-class PredictionFunctionToRiverInput:
-
-    def __init__(self,
-                 prediction_function: Callable,
-                 ) -> None:
-        self.prediction_function = prediction_function
-
-    def __call__(self, x: Union[dict[str], np.ndarray]):
-        if isinstance(x, dict):
-            x_list = list(x.values())
-            y_pred = np.asarray([self.prediction_function(x_list)])
-        else:
-            y_pred = np.empty(shape=len(x))
-            for i in range(len(x)):
-                y_pred[i] = self.prediction_function(x[i])
-        return y_pred
diff --git a/visualization/color.py b/visualization/color.py
deleted file mode 100644
index f59f038..0000000
--- a/visualization/color.py
+++ /dev/null
@@ -1,22 +0,0 @@
-COLOR_LIST = ['#ef27a6', '#4ea5d9', '#7d53de', '#44cfcb']
-BASE_COLOR = '#a6a7a9'
-BACKGROUND_COLOR = '#f8f8f8'
-STD_ALPHA = 0.2
-SAVE_DPI = 300
-
-
-def get_color_with_generator(color_generator, item=None, item_selection=None, base_color=None):
-    if base_color is None:
-        base_color = BASE_COLOR
-    if item_selection is not None and item not in item_selection:
-        return base_color
-    else:
-        return next(color_generator)
-
-
-def color_list_generator(color_list=None):
-    if color_list is None:
-        color_list = COLOR_LIST
-    color_list = color_list * 100  # upper bound
-    for color in color_list:
-        yield color
diff --git a/visualization/plotting.py b/visualization/plotting.py
deleted file mode 100644
index 57eaddb..0000000
--- a/visualization/plotting.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from abc import ABCMeta
-from typing import Optional, Union
-
-import matplotlib.pyplot as plt
-
-from color import SAVE_DPI
-from visualization.line_plots import plot_multi_line_graph
-
-
-class BasePlotter(metaclass=ABCMeta):
-
-    def __init__(self) -> None:
-        self.seen_timesteps = 0
-
-    def save(self,
-             save_file_path: Optional[str] = None,
-             save_dpi: Optional[int] = SAVE_DPI) -> None:
-        pass
-
-
-class FeatureImportancePlotter(BasePlotter):
-
-    def __init__(self,
-                 feature_names: Optional[list[str]] = None):
-        super().__init__()
-        self.feature_names = feature_names
-        self.y_data = {}
-        self.x_data = {}
-        self.stored_facets = {}
-
-    def _create_new_facet(self, facet_name: str):
-        self.stored_facets.update(facet_name)
-        self.y_data[facet_name] = []
-        self.x_data[facet_name] = []
-
-
-    def update(self,
-               importance_values: dict[str, Union[int, float]],
-               facet_name: Optional[str] = None,
-               timestep: Optional[Union[int, float]] = None):
-        self.seen_timesteps += 1
-        if facet_name is not None and facet_name not in self.stored_facets:
-            self._create_new_facet(facet_name)
-        if timestep is None:
-            timestep = self.seen_timesteps
-        self.y_data[facet_name].append(importance_values)
-        self.x_data[facet_name].append(timestep)
-
-    def plot(self,
-             figsize: Optional[tuple[int]] = None
-             ) -> None:
-        fig, axis = plt.subplot(figsize=figsize)
-        axis = plot_multi_line_graph(axis=axis,
-                                     y_data=self.y_data,
-                                     names_to_highlight=self.feature_names,
-                                     line_names=self.feature_names)
-        plt.show()
-


Test patch:
diff --git a/data/tests/test_synthetic.py b/data/tests/test_synthetic.py
deleted file mode 100644
index c8d7929..0000000
--- a/data/tests/test_synthetic.py
+++ /dev/null
@@ -1,61 +0,0 @@
-"""
-This file gathers Tests for the Shap Implementations
-"""
-
-# Authors: Maximilian Muschalik <maximilian.muschalik@lmu.de>
-#          Fabian Fumagalli <ffumagalli@techfak.uni-bielefeld.de>
-
-import time
-
-import numpy as np
-import random
-from pytest import approx
-
-from data.synthetic import SyntheticDataset
-
-
-class TestSyntheticDataset:
-    # TODO add test for target_function and noise levels
-
-    def test_create_dataset(self):
-        n_samples_test = 1000
-        n_features_test = 20
-        n_numeric_test = 10
-
-        dataset = SyntheticDataset(n_features=n_features_test, n_numeric=n_numeric_test, n_samples=n_samples_test)
-
-        x_data = dataset.x_data
-        y_data = dataset.y_data
-
-        assert len(dataset) == n_samples_test
-        assert x_data.shape[0] == n_samples_test and len(y_data) == n_samples_test
-        assert x_data.shape[1] == n_features_test
-
-    def test_to_stream(self):
-        n_samples_test = 1000
-        n_features_test = 10
-        n_numeric_test = 8
-
-        dataset = SyntheticDataset(n_features=n_features_test, n_numeric=n_numeric_test, n_samples=n_samples_test)
-        stream = dataset.to_stream()
-
-        counter = 0
-        for i, x_i in enumerate(stream):
-            counter += 1
-            if i >= n_samples_test - 1:
-                break
-
-        assert counter == n_samples_test
-
-    def test_feature_names(self):
-        n_samples_test = 1000
-        n_features_test = 10
-        n_numeric_test = 8
-
-        dataset = SyntheticDataset(n_features=n_features_test, n_numeric=n_numeric_test, n_samples=n_samples_test)
-
-        feature_names = dataset.feature_names
-
-        assert len(feature_names) == n_features_test
-        assert feature_names[3] == "N_4"
-        assert feature_names[-1] == "C_2"
diff --git a/experiments/iSAGE_paper/test.py b/experiments/iSAGE_paper/test.py
new file mode 100644
index 0000000..c1623cb
--- /dev/null
+++ b/experiments/iSAGE_paper/test.py
@@ -0,0 +1,199 @@
+import math
+import time
+
+import numpy as np
+import pandas as pd
+from river.datasets.base import REG
+
+from experiments.setup.data import get_dataset, get_concept_drift_dataset
+from experiments.setup.explainer import get_incremental_sage_explainer, \
+    get_interval_sage_explainer, get_incremental_pfi_explainer, get_imputer_and_storage
+from experiments.setup.loss import get_loss_function, get_training_metric
+from experiments.setup.model import get_model
+from increment_explain.explainer import IncrementalPFI
+from increment_explain.explainer.sage import IntervalSageExplainer, IncrementalSageExplainer
+from increment_explain.utils.converters import RiverToPredictionFunction
+from increment_explain.utils.trackers import ExponentialSmoothingTracker
+from increment_explain.visualization import FeatureImportancePlotter
+
+DEBUG = True
+
+DATASET_1_NAME = 'stream_bike'
+
+DATASET_RANDOM_SEED = 1
+SHUFFLE_DATA = False
+N_SAMPLES_1 = None
+
+MODEL_NAME = 'ARF'
+
+N_INNER_SAMPLES: int = 1
+SMOOTHING_ALPHA: float = 0.001
+FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
+RESERVOIR_LENGTH: int = 100
+RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
+
+TRAINING_METRIC_WINDOW = 500
+
+MODEL_PARAMS = {
+    'ARF': {
+        'n_models': 25,
+    }
+}
+
+
+if __name__ == "__main__":
+
+    # Get Data ---------------------------------------------------------------------------------------------------------
+    dataset = get_dataset(
+        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
+    feature_names = dataset.feature_names
+    feature_names.remove('moment')
+    cat_feature_names = dataset.cat_feature_names
+    num_feature_names = dataset.num_feature_names
+    n_samples = dataset.n_samples
+    stream = dataset.stream
+
+    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
+    task = dataset.task
+    loss_function = get_loss_function(task=task)
+    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
+    model, model_function = get_model(
+        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
+
+    # Get imputer and explainers ---------------------------------------------------------------------------------------
+    imputer, storage = get_imputer_and_storage(
+        model_function=model_function,
+        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
+        reservoir_kind=RESERVOIR_KIND,
+        reservoir_length=RESERVOIR_LENGTH,
+        cat_feature_names=cat_feature_names,
+        num_feature_names=num_feature_names
+    )
+
+    incremental_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        imputer=imputer,
+        storage=storage,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    incremental_welford_sage = IncrementalSageExplainer(
+        model_function=model_function,
+        loss_function=loss_function,
+        dynamic_setting=False,
+        feature_names=feature_names,
+        smoothing_alpha=SMOOTHING_ALPHA,
+        n_inner_samples=N_INNER_SAMPLES
+    )
+
+    # warm-up because of errors ----------------------------------------------------------------------------------------
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        del x_i['moment']
+        print(x_i)
+        model.learn_one(x_i, y_i)
+        if n > 10:
+            break
+
+    plotter = FeatureImportancePlotter(
+        feature_names=feature_names
+    )
+
+    time_sage = 0.
+    time_interval = 0.
+    time_learning = 0.
+
+    sage_fi_values = []
+    sage_welford_fi_values = []
+
+    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
+    model_loss = []
+    marginal_loss = []
+
+    for (n, (x_i, y_i)) in enumerate(stream, start=1):
+        del x_i['moment']
+        # predicting
+        y_i_pred = model_function(x_i)
+        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
+        model_loss_tracker.update(loss_i)
+        model_loss.append({"loss": model_loss_tracker.get()})
+        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
+        marginal_loss_tracker.update(loss_marginal_i)
+        marginal_loss.append({"loss": marginal_loss_tracker.get()})
+
+        # sage inc
+        #start_time = time.time()
+        #fi_values = incremental_welford_sage.explain_one(x_i, y_i)
+        #time_sage += time.time() - start_time
+        #sage_welford_fi_values.append(fi_values)
+        #plotter.update(fi_values, facet_name='inc-w')
+
+        # sage inc
+        #start_time = time.time()
+        #fi_values = incremental_sage.explain_one(x_i, y_i)
+        #time_sage += time.time() - start_time
+        #sage_fi_values.append(fi_values)
+        #plotter.update(fi_values, facet_name='inc')
+
+        # learning
+        start_time = time.time()
+        model.learn_one(x_i, y_i)
+        time_learning += time.time() - start_time
+
+        if DEBUG and n % 1000 == 0:
+            print(f"{n}: x_i                 {x_i}\n"
+                  f"{n}: x_i                 {model_loss_tracker.get()}\n"
+                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
+                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
+                  )
+
+        if n >= n_samples:
+            break
+
+    # Store the results in a database ----------------------------------------------------------------------------------
+
+    facet = "left.csv"
+    data_folder = "static_sage_in_dynamic_setting_insects"
+
+    name = "inc_fi_values"
+    df = pd.DataFrame(sage_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "inc_welford_fi_values"
+    df = pd.DataFrame(sage_welford_fi_values)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "model_loss"
+    df = pd.DataFrame(model_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    name = "marginal_loss"
+    df = pd.DataFrame(marginal_loss)
+    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
+
+    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
+                      "line_names": ["loss"], "names_to_highlight": ['loss'],
+                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
+                      "markevery": {"model_loss": 100, "marginal_loss": 100},
+                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
+                                 "legend_items": [("marginal loss", "dashed", "red"),
+                                                  ("model loss", "solid", "red")]}}
+
+    plotter.plot(
+        figsize=(5, 10),
+        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
+        performance_kw=performance_kw,
+        title=f'Agrawal Stream with Concept Drift',
+        y_label='SAGE values',
+        x_label='Samples',
+        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
+        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
+        line_styles={'inc': '-', 'inc-w': '--'},
+        legend_style={"fontsize": "small"},
+        markevery={'inc': 10, 'inc-w': 10},
+        y_min=-0.05,
+        y_max=0.42,
+    )
diff --git a/sampling/__init__.py b/increment_explain/explainer/tests/__init__.py
similarity index 100%
rename from sampling/__init__.py
rename to increment_explain/explainer/tests/__init__.py
diff --git a/explainer/tests/test_shap.py b/increment_explain/explainer/tests/test_shap.py
similarity index 100%
rename from explainer/tests/test_shap.py
rename to increment_explain/explainer/tests/test_shap.py
diff --git a/visualization/__init__.py b/tests/__init__.py
similarity index 100%
rename from visualization/__init__.py
rename to tests/__init__.py
diff --git a/tests/test_storage.py b/tests/test_storage.py
new file mode 100644
index 0000000..db4b71d
--- /dev/null
+++ b/tests/test_storage.py
@@ -0,0 +1,85 @@
+import pytest
+from increment_explain.storage import (BatchStorage, IntervalStorage,
+                                       SequenceStorage,
+                                       UniformReservoirStorage,
+                                       GeometricReservoirStorage)
+
+
+@pytest.fixture
+def dummy_stream():
+    # TODO - make stream an iterator
+    stream = []
+    for i in range(10):
+        stream.append(({'t': i}, i))
+    return stream
+
+
+@pytest.mark.parametrize("store_targets", [True, False])
+def test_batch_storage(dummy_stream, store_targets):
+    storage = BatchStorage(store_targets=store_targets)
+    for x_i, y_i in dummy_stream:
+        storage.update(x_i, y_i)
+    assert len(storage) == 10
+    if store_targets:
+        assert len(storage.get_data()[1]) == 10
+    else:
+        assert len(storage.get_data()[1]) == 0
+
+
+@pytest.mark.parametrize("store_targets", [True, False])
+@pytest.mark.parametrize("size", [5, 8, 1])
+def test_interval_storage(dummy_stream, store_targets, size):
+    storage = IntervalStorage(store_targets=store_targets, size=size)
+    for x_i, y_i in dummy_stream:
+        storage.update(x_i, y_i)
+    assert len(storage) == size
+    assert storage.get_data()[0][size-1]['t'] == 9
+    assert storage.get_data()[0][0]['t'] == 10 - size
+    if store_targets:
+        assert len(storage.get_data()[1]) == size
+        assert storage.get_data()[1][size-1] == 9
+        assert storage.get_data()[1][0] == 10 - size
+    else:
+        assert len(storage.get_data()[1]) == 0
+
+
+@pytest.mark.parametrize("store_targets", [True, False])
+def test_sequence_storage(dummy_stream, store_targets):
+    storage = SequenceStorage(store_targets=store_targets)
+    for x_i, y_i in dummy_stream:
+        storage.update(x_i, y_i)
+    assert len(storage) == 1
+    assert storage.get_data()[0][0]['t'] == 9
+    if store_targets:
+        assert len(storage.get_data()[1]) == 1
+        assert storage.get_data()[1][0] == 9
+    else:
+        assert len(storage.get_data()[1]) == 0
+
+
+@pytest.mark.parametrize("store_targets", [True, False])
+@pytest.mark.parametrize("size", [3, 5])
+def test_uniform_reservoir_storage(dummy_stream, store_targets, size):
+    storage = UniformReservoirStorage(store_targets=store_targets,
+                                      size=size)
+    for x_i, y_i in dummy_stream:
+        storage.update(x_i, y_i)
+    assert len(storage) == size
+    if store_targets:
+        assert len(storage.get_data()[1]) == size
+    else:
+        assert len(storage.get_data()[1]) == 0
+
+
+@pytest.mark.parametrize("store_targets", [True, False])
+@pytest.mark.parametrize("size", [3, 5])
+def test_geometric_reservoir_storage(dummy_stream, store_targets, size):
+    storage = GeometricReservoirStorage(store_targets=store_targets,
+                                        size=size)
+    for x_i, y_i in dummy_stream:
+        storage.update(x_i, y_i)
+    assert len(storage) == size
+    if store_targets:
+        assert len(storage.get_data()[1]) == size
+    else:
+        assert len(storage.get_data()[1]) == 0
diff --git a/utils/tests/test_converters.py b/utils/tests/test_converters.py
deleted file mode 100644
index e69de29..0000000