Repository: mmschlk/iXAI (language: python)

Issue:
Refactor Experiments out of the package
The experiments module with all data and setup code needs to be refactored into separate repositories to reduce clutter.


Discussion / hints:

Fix patch:
diff --git a/data/__init__.py b/data/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/data/batch/__init__.py b/data/batch/__init__.py
deleted file mode 100644
index b4f363a..0000000
--- a/data/batch/__init__.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from .adult import Adult
-from .bank import BankMarketing
-from .bike import BikeSharing
-
-__all__ = [
-    "Adult",
-    "BankMarketing",
-    "BikeSharing"
-]
diff --git a/data/batch/_base.py b/data/batch/_base.py
deleted file mode 100644
index 4e16bc4..0000000
--- a/data/batch/_base.py
+++ /dev/null
@@ -1,115 +0,0 @@
-from abc import ABCMeta, abstractmethod
-from typing import Optional, Union
-import openml
-from sklearn.utils import shuffle
-from river.stream import iter_pandas
-from river.datasets import base
-from sklearn.preprocessing import LabelEncoder
-import numpy as np
-
-from data.stream._base import BatchStream, ConceptDriftStream
-
-
-__all__ = [
-    "get_open_ml_dataset",
-    "BaseBatchDataset",
-]
-
-
-def get_open_ml_dataset(open_ml_id, version=1):
-    dataset = openml.datasets.get_dataset(open_ml_id, version=version, download_data=True)
-    class_label = dataset.default_target_attribute
-    x_data = dataset.get_data()[0]
-    return x_data, class_label
-
-
-class BaseBatchDataset(metaclass=ABCMeta):
-    """Base class for creating data sets.
-
-    Warning: This class should not be used directly.
-    Use derived classes instead.
-    """
-
-    @abstractmethod
-    def __init__(
-            self,
-            *,
-            dataset,
-            class_label,
-            classification: bool,
-            random_seed: Optional[int] = None,
-            drop_na: bool = True,
-            shuffle_dataset: bool = True,
-            n_samples: Optional[int] = None
-    ):
-        if drop_na:
-            dataset.dropna(inplace=True)
-        if shuffle_dataset:
-            dataset = shuffle(dataset, random_state=random_seed)
-        if n_samples is not None:
-            dataset = dataset[0:n_samples]
-        self.x_data = dataset
-        self.y_data = dataset.pop(class_label)
-        self.feature_names = list(self.x_data.columns)
-        self.n_features = len(self.feature_names)
-        self.n_samples = len(self.x_data)
-        self.classification = classification
-        if self.classification:
-            self.task = base.BINARY_CLF
-            self._label_encoder = LabelEncoder()
-            self._transform_label_column()
-        else:
-            self.task = base.REG
-        self.n_outputs = 1
-
-    @property
-    def stream(self, start_pos: int = 0, end_pos: Optional[int] = None):
-        end_pos = self.n_samples + 1 if end_pos is None else end_pos
-        start_pos = start_pos * self.n_samples if type(start_pos) == float else start_pos
-        end_pos = end_pos * self.n_samples if type(end_pos) == float else end_pos
-        if (end_pos - start_pos - 1) != self.n_samples:
-            self.n_samples = end_pos - start_pos - 1
-        return BatchStream(
-            stream=iter_pandas(X=self.x_data[start_pos:end_pos], y=self.y_data[start_pos:end_pos]),
-            task=self.task, n_features=self.n_features, n_outputs=self.n_outputs
-        )
-
-    def to_concept_drift_stream(
-            self,
-            feature_remapping: dict[str, str],
-            position: Union[int, float] = 0.5,
-            width: Union[int, float] = 0.05
-    ):
-        feature_remapping.update({v: k for k, v in feature_remapping.items()})
-        n_stream_1 = int(np.floor(position * self.n_samples)) if position < 1 else int(position)
-        width = int(width * self.n_samples) if width < 1 else int(width)
-
-        x_data_1, y_data_1 = self.x_data[:n_stream_1], self.y_data[:n_stream_1]
-        x_data_2, y_data_2 = self.x_data[n_stream_1:], self.y_data[n_stream_1:]
-        x_data_2 = x_data_2.rename(columns=feature_remapping, inplace=False)
-        stream_1 = BatchStream(
-            stream=iter_pandas(X=x_data_1, y=y_data_1),
-            task=self.task, n_features=self.n_features, n_outputs=self.n_outputs
-        )
-        stream_2 = BatchStream(
-            stream=iter_pandas(X=x_data_2, y=y_data_2),
-            task=self.task, n_features=self.n_features, n_outputs=self.n_outputs
-        )
-
-        concept_drift_stream = ConceptDriftStream(stream=stream_1, drift_stream=stream_2, width=width)
-        return concept_drift_stream
-
-    def transform_features(self, feature_names, transformer):
-        for feature in feature_names:
-            self.x_data[feature] = transformer.fit_transform(self.x_data[feature].values.reshape(-1, 1))
-
-    def _transform_label_column(self):
-        self.y_data[:] = self._label_encoder.fit_transform(self.y_data)
-
-    @property
-    def label_encoding(self):
-        return self._label_encoder.get_params()
-
-
-if __name__ == "__main__":
-    pass
diff --git a/data/batch/adult.py b/data/batch/adult.py
deleted file mode 100644
index ff80ffd..0000000
--- a/data/batch/adult.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import pandas as pd
-from sklearn.compose import ColumnTransformer
-from sklearn.impute import SimpleImputer
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import OrdinalEncoder, StandardScaler
-
-from data.batch._base import BaseBatchDataset, get_open_ml_dataset
-
-
-class Adult(BaseBatchDataset):
-
-    def __init__(
-            self,
-            version=2,
-            random_seed=None,
-            shuffle_dataset=False,
-            n_samples=None
-    ):
-        assert version in [1, 2], "OpenML census dataset version must be '1' or '2'."
-        dataset, class_label = get_open_ml_dataset("adult", version=version)
-        self.num_feature_names = ['age', 'capital-gain', 'capital-loss', 'hours-per-week', 'fnlwgt']
-        self.cat_feature_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race', 'sex', 'native-country', 'education-num']
-        dataset[self.num_feature_names] = dataset[self.num_feature_names].apply(pd.to_numeric)
-        num_pipeline = Pipeline([
-            ('imputer', SimpleImputer(strategy='median')),
-            ('std_scaler', StandardScaler())
-        ])
-        cat_pipeline = Pipeline([
-            ('ordinal_encoder', OrdinalEncoder()),
-        ])
-        column_transformer = ColumnTransformer([
-            ('numerical', num_pipeline, self.num_feature_names),
-            ('categorical', cat_pipeline, self.cat_feature_names),
-        ], remainder='passthrough')
-        col_names = self.num_feature_names + self.cat_feature_names
-        col_names += [feature for feature in dataset.columns if feature not in col_names]
-        dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
-        dataset.dropna(inplace=True)
-        super(Adult, self).__init__(
-            dataset=dataset,
-            class_label=class_label,
-            classification=True,
-            random_seed=random_seed,
-            shuffle_dataset=shuffle_dataset,
-            n_samples=n_samples
-        )
-
-
-if __name__ == "__main__":
-    test_dataset = Adult(random_seed=42, shuffle_dataset=False)
-    print(f"n_samples:     {test_dataset.n_samples}")
-    print(f"n_features:    {test_dataset.n_features}")
-    print(f"feature_names: {test_dataset.feature_names}")
-    stream = test_dataset.stream
-    for i, sample in enumerate(stream):
-        print(i, sample)
-        if i > 3:
-            break
\ No newline at end of file
diff --git a/data/batch/bank.py b/data/batch/bank.py
deleted file mode 100644
index 3f961de..0000000
--- a/data/batch/bank.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import pandas as pd
-from sklearn.compose import ColumnTransformer
-from sklearn.impute import SimpleImputer
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import OrdinalEncoder, StandardScaler
-
-from data.batch._base import BaseBatchDataset, get_open_ml_dataset
-
-
-OPEN_ML_BANK_MARKETING_RENAME_MAPPER = {
-    'V1': 'age',
-    'V2': 'job',
-    'V3': 'marital',
-    'V4': 'education',
-    'V5': 'default',
-    'V6': 'balance',
-    'V7': 'housing',
-    'V8': 'loan',
-    'V9': 'contact',
-    'V10': 'day',
-    'V11': 'month',
-    'V12': 'duration',
-    'V13': 'campaign',
-    'V14': 'pdays',
-    'V15': 'previous',
-    'V16': 'poutcome'
-}
-
-
-class BankMarketing(BaseBatchDataset):
-
-    def __init__(
-            self,
-            random_seed=None,
-            shuffle_dataset=False,
-            n_samples=None
-    ):
-        dataset, class_label = get_open_ml_dataset("bank-marketing", version=1)
-        dataset = dataset.rename(columns=OPEN_ML_BANK_MARKETING_RENAME_MAPPER)
-        self.num_feature_names = ['age', 'balance', 'duration', 'pdays', 'previous']
-        self.cat_feature_names = ['job', 'marital', 'education', 'default', 'housing', 'loan', 'contact', 'day', 'month', 'campaign', 'poutcome']
-        dataset[self.num_feature_names] = dataset[self.num_feature_names].apply(pd.to_numeric)
-        num_pipeline = Pipeline([
-            ('imputer', SimpleImputer(strategy='median')),
-            ('std_scaler', StandardScaler())
-        ])
-        cat_pipeline = Pipeline([
-            ('ordinal_encoder', OrdinalEncoder()),
-        ])
-        column_transformer = ColumnTransformer([
-            ('numerical', num_pipeline, self.num_feature_names),
-            ('categorical', cat_pipeline, self.cat_feature_names),
-        ], remainder='passthrough')
-        col_names = self.num_feature_names + self.cat_feature_names
-        col_names += [feature for feature in dataset.columns if feature not in col_names]
-        dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
-        dataset.dropna(inplace=True)
-        super(BankMarketing, self).__init__(
-            dataset=dataset,
-            class_label=class_label,
-            classification=True,
-            random_seed=random_seed,
-            shuffle_dataset=shuffle_dataset,
-            n_samples=n_samples
-        )
-
-
-if __name__ == "__main__":
-    test_dataset = BankMarketing(random_seed=42, shuffle_dataset=False)
-    print(f"n_samples:     {test_dataset.n_samples}")
-    print(f"n_features:    {test_dataset.n_features}")
-    print(f"feature_names: {test_dataset.feature_names}")
-    stream = test_dataset.stream
-    for i, sample in enumerate(stream):
-        print(i, sample)
-        if i > 3:
-            break
\ No newline at end of file
diff --git a/data/batch/bike.py b/data/batch/bike.py
deleted file mode 100644
index 47b118b..0000000
--- a/data/batch/bike.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import pandas as pd
-from sklearn.compose import ColumnTransformer
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import OrdinalEncoder, RobustScaler
-
-from data.batch._base import BaseBatchDataset, get_open_ml_dataset
-
-
-class BikeSharing(BaseBatchDataset):
-
-    def __init__(
-            self,
-            random_seed=None,
-            shuffle_dataset=False,
-            n_samples=None
-    ):
-        dataset, class_label = get_open_ml_dataset(42713, version=1)
-        self.num_feature_names = ['hour', 'temp', 'feel_temp', 'humidity', 'windspeed']
-        self.cat_feature_names = ['season', 'year', 'month', 'holiday', 'weekday', 'workingday', 'weather']
-        dataset[self.num_feature_names] = dataset[self.num_feature_names].apply(pd.to_numeric)
-        num_pipeline = Pipeline([
-            ('scaler', RobustScaler())
-        ])
-        cat_pipeline = Pipeline([
-            ('ordinal_encoder', OrdinalEncoder()),
-        ])
-        column_transformer = ColumnTransformer([
-            ('numerical', num_pipeline, self.num_feature_names),
-            ('categorical', cat_pipeline, self.cat_feature_names),
-        ], remainder='passthrough')
-        col_names = self.num_feature_names + self.cat_feature_names
-        col_names += [feature for feature in dataset.columns if feature not in col_names]
-        dataset = pd.DataFrame(column_transformer.fit_transform(dataset), columns=col_names)
-        dataset.dropna(inplace=True)
-        super(BikeSharing, self).__init__(
-            dataset=dataset,
-            class_label=class_label,
-            classification=False,
-            random_seed=random_seed,
-            shuffle_dataset=shuffle_dataset,
-            n_samples=n_samples
-        )
-
-
-if __name__ == "__main__":
-    test_dataset = BikeSharing(random_seed=42, shuffle_dataset=False)
-    print(f"n_samples:     {test_dataset.n_samples}")
-    print(f"n_features:    {test_dataset.n_features}")
-    print(f"feature_names: {test_dataset.feature_names}")
-    stream = test_dataset.stream
-    for i, sample in enumerate(stream):
-        print(i, sample)
-        if i > 3:
-            break
diff --git a/data/stream/__init__.py b/data/stream/__init__.py
deleted file mode 100644
index c64e568..0000000
--- a/data/stream/__init__.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from .elec2 import Elec2
-from .insects import Insects
-from .credit_card import CreditCard
-from .bike_stream import Bike
-
-__all__ = [
-    "Elec2",
-    "Insects",
-    "CreditCard",
-    "Bike"
-]
diff --git a/data/stream/_base.py b/data/stream/_base.py
deleted file mode 100644
index 1ad7dcf..0000000
--- a/data/stream/_base.py
+++ /dev/null
@@ -1,167 +0,0 @@
-from random import random
-from typing import Optional, Generator, Union, Dict
-import numpy as np
-from river.datasets.base import Dataset
-
-
-__all__ = [
-    "BatchStream",
-    "FeatureSwitchStream",
-    "ConceptDriftStream",
-    "StreamDataset"
-]
-
-
-class BatchStream(Dataset):
-
-    def __init__(self, stream, task, n_features, n_classes=None, n_outputs=None):
-        super().__init__(task, n_features, n_classes=n_classes, n_outputs=n_outputs)
-        self.stream_gen = stream
-
-    def __iter__(self):
-        for x_i, y_i in self.stream_gen:
-            yield x_i, y_i
-
-
-class ConceptDriftStream(Dataset):
-
-    def __init__(
-            self,
-            stream,
-            drift_stream,
-            position: int = 5000,
-            width: int = 1000,
-            alpha: float = None,
-    ):
-        super().__init__(
-            n_features=stream.n_features,
-            n_classes=stream.n_classes,
-            n_outputs=stream.n_outputs,
-            task=stream.task,
-        )
-        self.alpha = alpha
-        if self.alpha is not None:
-            if 0 < self.alpha <= 90.0:
-                w = int(1 / np.tan(self.alpha * np.pi / 180))
-                self.width = w if w > 0 else 1
-            else:
-                raise ValueError(
-                    f"Invalid alpha value: {alpha}. " f"Valid values are in the range (0.0, 90.0]"
-                )
-        else:
-            self.width = width
-        self.position = position
-        self.stream = stream
-        self.drift_stream = drift_stream
-
-    def __iter__(self):
-        stream_generator = iter(self.stream)
-        drift_stream_generator = iter(self.drift_stream)
-        sample_idx = 0
-
-        while True:
-            sample_idx += 1
-            v = -4.0 * float(sample_idx - self.position) / float(self.width)
-            probability_drift = 1.0 / (1.0 + np.exp(v))
-            try:
-                if random() > probability_drift:
-                    try:
-                        x, y = next(stream_generator)
-                    except StopIteration:
-                        x, y = next(drift_stream_generator)
-                else:
-                    try:
-                        x, y = next(drift_stream_generator)
-                    except StopIteration:
-                        x, y = next(stream_generator)
-            except StopIteration:
-                break
-            yield x, y
-
-
-class FeatureSwitchStream(Dataset):
-
-    def __init__(self, feature_switch, stream, task, n_features, n_classes=None, n_outputs=None):
-        super().__init__(task, n_features, n_classes=n_classes, n_outputs=n_outputs)
-        self.stream = stream
-        self.feature_switch = feature_switch
-
-    def __iter__(self):
-        for x, y in self.stream.__iter__():
-            for feature_1, feature_2 in self.feature_switch.items():
-                x[feature_1], x[feature_2] = x[feature_2], x[feature_1]
-            yield x, y
-
-
-class StreamDataset(Dataset):
-
-    def __init__(self, stream, n_samples, task, n_features, n_outputs, feature_names, cat_feature_names, num_feature_names):
-        super().__init__(task, n_features)
-        self.stream = stream
-        self.n_outputs = n_outputs
-        self.feature_names = feature_names
-        self.cat_feature_names = cat_feature_names
-        self.num_feature_names = num_feature_names
-        self.n_samples = n_samples
-
-    def __iter__(self):
-        n = 0
-        stream_iter = iter(self.stream)
-        while n < self.n_samples:
-            yield next(stream_iter)
-            n += 1
-
-
-def slice_stream(stream: Generator, position: int):
-    first_stream = []
-    second_stream = []
-    for n, stream_i in enumerate(stream):
-        if n < position:
-            first_stream.append(stream_i)
-        else:
-            second_stream.append(stream_i)
-    first_stream = BatchStream(
-        stream=first_stream,
-        task=stream.task, n_features=stream.n_features, n_outputs=stream.n_outputs, n_classes=stream.n_classes)
-    second_stream = BatchStream(
-        stream=second_stream,
-        task=stream.task, n_features=stream.n_features, n_outputs=stream.n_outputs, n_classes=stream.n_classes)
-    return first_stream, second_stream
-
-
-def _get_feature_remapping_dict(feature_remapping_str: str):
-    feature_remapping = {}
-    for features in feature_remapping_str.split(' '):
-        feature_1, feature_2 = features.split('_')
-        feature_remapping[feature_1] = feature_2
-        feature_remapping[feature_2] = feature_1
-    return feature_remapping
-
-
-def get_concept_drift_stream(
-        stream,
-        position: int,
-        drift_stream: Optional[Generator] = None,
-        feature_remapping: Optional[Union[Dict[str, str], str]] = None,
-        width: int = 1
-):
-
-    if drift_stream is None:
-        stream, drift_stream = slice_stream(stream=stream, position=position)
-
-    if feature_remapping is not None:
-        if type(feature_remapping) == str:
-            feature_remapping = _get_feature_remapping_dict(feature_remapping)
-        drift_stream = FeatureSwitchStream(
-            feature_switch=feature_remapping, stream=drift_stream,
-            task=drift_stream.task, n_features=drift_stream.n_features,
-            n_classes=drift_stream.n_classes, n_outputs=drift_stream.n_outputs)
-
-    concept_drift_stream = ConceptDriftStream(
-        stream=stream, drift_stream=drift_stream, position=position, width=width)
-
-    return concept_drift_stream
-
-
-if __name__ == "__main__":
-    pass
diff --git a/data/stream/bike_stream.py b/data/stream/bike_stream.py
deleted file mode 100644
index 8f41055..0000000
--- a/data/stream/bike_stream.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from typing import Optional
-
-from river.datasets import Bikes as RiverDataset
-from data.stream._base import StreamDataset
-
-
-class Bike(StreamDataset):
-
-    def __init__(
-            self,
-            n_samples: Optional[int] = 182470
-    ):
-        if n_samples is None:
-            n_samples = 182470
-        assert n_samples <= 182470, f"Bike contains at maximum  182470 samples."
-        stream = RiverDataset()
-        feature_names = ['moment', 'station', 'clouds', 'description', 'humidity', 'pressure', 'temperature', 'wind']
-        cat_feature_names = ['station', 'description']
-        num_feature_names = ['clouds', 'humidity', 'pressure', 'temperature', 'wind']
-        super().__init__(
-            stream=stream,
-            n_samples=n_samples,
-            feature_names=feature_names,
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names,
-            task=stream.task,
-            n_features=len(feature_names),
-            n_outputs=stream.n_outputs,
-        )
-
-
-if __name__ == "__main__":
-    dataset = Bike()
-    stream = dataset.stream
-    print(stream.n_samples)
-    for n, (x_i, y_i) in enumerate(stream):
-        print(n, x_i, y_i)
-        if n > 3:
-            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
-            break
diff --git a/data/stream/credit_card.py b/data/stream/credit_card.py
deleted file mode 100644
index 12ddbb9..0000000
--- a/data/stream/credit_card.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from typing import Optional
-
-from river.datasets import CreditCard as RiverDataset
-from data.stream._base import StreamDataset
-
-
-class CreditCard(StreamDataset):
-
-    def __init__(
-            self,
-            n_samples: Optional[int] = 284_807
-    ):
-        if n_samples is None:
-            n_samples = 284_807
-        assert n_samples <= 284_807, f"CreditCard contains at maximum  284_807 samples."
-        stream = RiverDataset()
-        feature_names = ['Time', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28', 'Amount']
-        cat_feature_names = []
-        num_feature_names = feature_names
-        super().__init__(
-            stream=stream,
-            n_samples=n_samples,
-            feature_names=feature_names,
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names,
-            task=stream.task,
-            n_features=len(feature_names),
-            n_outputs=stream.n_outputs,
-        )
-
-
-if __name__ == "__main__":
-    dataset = CreditCard()
-    stream = dataset.stream
-    print(stream.n_samples)
-    for n, (x_i, y_i) in enumerate(stream):
-        print(n, x_i, y_i)
-        if n > 3:
-            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
-            break
diff --git a/data/stream/elec2.py b/data/stream/elec2.py
deleted file mode 100644
index 15d48c8..0000000
--- a/data/stream/elec2.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from typing import Optional
-
-from river.datasets import Elec2 as RiverDataset
-from data.stream._base import StreamDataset
-
-
-class Elec2(StreamDataset):
-
-    def __init__(
-            self,
-            n_samples: Optional[int] = 45312
-    ):
-        if n_samples is None:
-            n_samples = 45312
-        assert n_samples <= 45312, f"Elec2 contains at maximum  45312 samples."
-        stream = RiverDataset()
-        feature_names = ['date', 'day', 'period', 'nswprice', 'nswdemand', 'vicprice', 'vicdemand', 'transfer']
-        cat_feature_names = []
-        num_feature_names = feature_names
-        super().__init__(
-            stream=stream,
-            n_samples=n_samples,
-            feature_names=feature_names,
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names,
-            task=stream.task,
-            n_features=len(feature_names),
-            n_outputs=stream.n_outputs,
-        )
-
-
-if __name__ == "__main__":
-    dataset = Elec2()
-    stream = dataset.stream
-    print(stream.n_samples)
-    for n, (x_i, y_i) in enumerate(stream):
-        print(n, x_i, y_i)
-        if n > 3:
-            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
-            break
diff --git a/data/stream/insects.py b/data/stream/insects.py
deleted file mode 100644
index c9c8ffd..0000000
--- a/data/stream/insects.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from typing import Optional
-from river.datasets import Insects as RiverDataset
-from data.stream._base import StreamDataset
-
-
-class Insects(StreamDataset):
-
-    def __init__(
-            self,
-            variant="abrupt_balanced",
-            n_samples: Optional[int] = None
-    ):
-        stream = RiverDataset()
-        feature_names = ['Att1', 'Att2', 'Att3', 'Att4', 'Att5', 'Att6', 'Att7', 'Att8', 'Att9', 'Att10', 'Att11',
-                              'Att12', 'Att13', 'Att14', 'Att15', 'Att16', 'Att17', 'Att18', 'Att19', 'Att20', 'Att21',
-                              'Att22', 'Att23', 'Att24', 'Att25', 'Att26', 'Att27', 'Att28', 'Att29', 'Att30', 'Att31',
-                              'Att32', 'Att33']
-        cat_feature_names = []
-        num_feature_names = ['Att1', 'Att2', 'Att3', 'Att4', 'Att5', 'Att6', 'Att7', 'Att8', 'Att9', 'Att10',
-                                  'Att11', 'Att12', 'Att13', 'Att14', 'Att15', 'Att16', 'Att17', 'Att18', 'Att19',
-                                  'Att20', 'Att21', 'Att22', 'Att23', 'Att24', 'Att25', 'Att26', 'Att27', 'Att28',
-                                  'Att29', 'Att30', 'Att31', 'Att32', 'Att33']
-        variant_sizes = {
-            "abrupt_balanced": (52_848, 16_419_025),
-            "abrupt_imbalanced": (355_275, 110_043_637),
-            "gradual_balanced": (24_150, 7_503_750),
-            "gradual_imbalanced": (143_323, 44_371_501),
-            "incremental-abrupt_balanced": (79_986, 24_849_436),
-            "incremental-abrupt_imbalanced": (452_044, 140_004_225),
-            "incremental-reoccurring_balanced": (79_986, 24_849_092),
-            "incremental-reoccurring_imbalanced": (452_044, 140_004_230),
-            "incremental_balanced": (57_018, 17_713_574),
-            "incremental_imbalanced": (452_044, 140_004_218),
-            "out-of-control": (905_145, 277_777_854),
-        }
-        n_samples_variant = variant_sizes[variant][0]
-        if n_samples is not None and n_samples > n_samples_variant:
-            raise ValueError(f"The dataset with variant {variant} contains less samples, {n_samples_variant} "
-                             f"than expected which was {n_samples}. Select less samples.")
-        super().__init__(
-            stream=stream,
-            n_samples=n_samples,
-            feature_names=feature_names,
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names,
-            task=stream.task,
-            n_features=len(feature_names),
-            n_outputs=stream.n_outputs,
-        )
-
-
-if __name__ == "__main__":
-    dataset = Insects()
-    stream = dataset.stream
-    print(stream.n_samples)
-    for n, (x_i, y_i) in enumerate(stream):
-        print(n, x_i, y_i)
-        if n > 3:
-            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
-            break
diff --git a/data/stream/synth/__init__.py b/data/stream/synth/__init__.py
deleted file mode 100644
index a406d66..0000000
--- a/data/stream/synth/__init__.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from ._base import SyntheticDataset
-from .agrawal import Agrawal
-from .planes import Planes2D
-from .stagger import Stagger
-
-__all__ = [
-    "Stagger",
-    "Agrawal",
-    "Planes2D",
-    "SyntheticDataset"
-]
diff --git a/data/stream/synth/_base.py b/data/stream/synth/_base.py
deleted file mode 100644
index 668fa83..0000000
--- a/data/stream/synth/_base.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from abc import ABCMeta, abstractmethod
-from typing import Callable, Optional, List
-
-from numpy import ndarray
-import river
-import numpy as np
-
-
-__all__ = [
-    "SyntheticDataset",
-]
-
-
-def _default_target_function(x):
-    weights = [1 for _ in range(len(x))]
-    return np.dot(x, weights)
-
-
-# =============================================================================
-# Base Datasets
-# =============================================================================
-
-
-class BaseSyntheticDataset(metaclass=ABCMeta):
-    """Base class for creating synthetic data sets.
-
-    Warning: This class should not be used directly.
-    Use derived classes instead.
-    """
-
-    @abstractmethod
-    def __init__(
-            self,
-            *,
-            x_data: ndarray,
-            y_data: ndarray,
-            feature_names: List[str]
-    ):
-        self.x_data = x_data
-        self.y_data = y_data
-        self.feature_names = feature_names
-
-    def to_stream(self):
-        """ Generates a data stream from the dataset
-
-        :return: data stream as an iterator of dictionary data samples
-        :rtype: river.base.typing.Stream
-        """
-        return river.stream.iter_array(self.x_data, self.y_data, feature_names=self.feature_names)
-
-
-# =============================================================================
-# Public Datasets
-# =============================================================================
-
-
-class SyntheticDataset(BaseSyntheticDataset):
-    """ A synthetic dataset with numeric and categorical values.
-
-    Parameters
-    ----------
-    target_function : Callable, default=None
-        The function used to create the target labels (i.e. the mapping function / relationship from X to y).
-        If no target function is provided the default function is a simple dot-product of the input data x with a
-        weight-vector containing only ones (i.e. sum of the feature values) is used.
-
-    n_samples : int, default=1000
-        Number of samples (i.e. rows) of the dataset.
-
-    n_features : int, default=15
-        Total Number of features (i.e. columns) of the dataset.
-
-    n_numeric : int default=10
-        Number of numerical features of the dataset.
-
-    n_categorical_values : int default=5
-        Number of categorical features of the dataset. If "n_numeric" + "n_categorical_values" is larger than
-        "n_features", then "n_categorical" is the remaining difference between "n_features" - "n_numeric".
-
-    noise_mean : float default=0.
-        Mean of the artificial noise added to the label function. This mean can create a bias in the data.
-        If set to 0. then no bias is added.
-
-    noise_std : float default=0.
-        Standard deviation of the artificial noise added to the label function. This adds some random deviation around
-        the "target_function" to induce more randomness into the dataset.
-        If set to 0. then no deviation is added.
-    """
-
-    def __init__(
-            self,
-            classification: bool = False,
-            target_function: Callable = None,
-            n_samples: int = 10000,
-            n_features: int = 15,
-            n_numeric: int = 10,
-            n_categorical_values: int = 5,
-            noise_mean: float = 0.,
-            noise_std: float = 0.,
-            random_seed: Optional[int] = None
-    ):
-        self.classification = classification
-        np.random.seed(random_seed)
-        self.n_samples = n_samples
-        self.n_features = n_features
-        self.n_numeric = n_numeric if n_numeric <= self.n_features else self.n_features
-        self.n_categorical = self.n_features - self.n_numeric
-        self.n_categorical_values = n_categorical_values
-        self.target_function = target_function
-        if self.target_function is None:
-            self.target_function = _default_target_function
-
-        self._noise_mean = noise_mean
-        self._noise_std = noise_std
-
-        x_data, feature_names = self._create_x_data()
-        y_data = self._create_y_data(x_data)
-
-        super(SyntheticDataset, self).__init__(
-            x_data=x_data,
-            y_data=y_data,
-            feature_names=feature_names
-        )
-
-    def __len__(self):
-        return self.n_samples
-
-    def _create_x_data(self):
-        x_data = np.random.rand(self.n_samples, self.n_numeric)
-        feature_names = ["N_"+str(i + 1) for i in range(self.n_numeric)]
-        if self.n_categorical > 0:
-            x_categorical = np.random.randint(low=0, high=self.n_categorical_values,
-                                              size=(self.n_samples, self.n_categorical))
-            x_data = np.concatenate((x_data, x_categorical), axis=1)
-            feature_names.extend(["C_" + str(i + 1) for i in range(self.n_categorical)])
-        return x_data, feature_names
-
-    def _create_y_data(self, x_data):
-        y_data = np.apply_along_axis(self.target_function, arr=x_data, axis=1)
-        if self.classification:
-            y_data = y_data.astype(int)
-        else:
-            y_data = y_data + np.random.normal(self._noise_mean, self._noise_std, len(y_data))
-        return y_data
-
-
-if __name__ == "__main__":
-    dataset = SyntheticDataset(n_features=20, n_numeric=10, n_samples=1000)
-    pass
diff --git a/data/stream/synth/agrawal.py b/data/stream/synth/agrawal.py
deleted file mode 100644
index eb66acb..0000000
--- a/data/stream/synth/agrawal.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from typing import Optional
-from river.datasets.synth import Agrawal as RiverDataset
-
-from data.stream._base import StreamDataset
-
-
-class Agrawal(StreamDataset):
-
-    def __init__(
-            self,
-            classification_function,
-            random_seed: Optional[int] = None,
-            n_samples: Optional[int] = None
-    ):
-        if n_samples is None:
-            n_samples = 20000
-        stream = RiverDataset(classification_function=classification_function, seed=random_seed)
-        feature_names = ['salary', 'commission', 'age', 'elevel', 'car', 'zipcode', 'hvalue', 'hyears', 'loan']
-        cat_feature_names = ['elevel', 'car', 'zipcode']
-        num_feature_names = ['salary', 'commission', 'age', 'hvalue', 'hyears', 'loan']
-        super().__init__(
-            stream=stream,
-            n_samples=n_samples,
-            feature_names=feature_names,
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names,
-            task=stream.task,
-            n_features=len(feature_names),
-            n_outputs=stream.n_outputs,
-        )
-
-
-if __name__ == "__main__":
-    dataset = Agrawal(classification_function=1, random_seed=42)
-    stream = dataset.stream
-    print(stream.n_samples)
-    for n, (x_i, y_i) in enumerate(stream):
-        print(n, x_i, y_i)
-        if n > 3:
-            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
-            break
diff --git a/data/stream/synth/planes.py b/data/stream/synth/planes.py
deleted file mode 100644
index 22a6928..0000000
--- a/data/stream/synth/planes.py
+++ /dev/null
@@ -1,39 +0,0 @@
-from typing import Optional
-from river.datasets.synth import Planes2D as RiverDataset
-
-from data.stream._base import StreamDataset
-
-class Planes2D(StreamDataset):
-
-    def __init__(
-            self,
-            random_seed: Optional[int] = None,
-            n_samples:  Optional[int] = None
-    ):
-        if n_samples is None:
-            n_samples = 20000
-        stream = RiverDataset(seed=random_seed)
-        feature_names = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        cat_feature_names = []
-        num_feature_names = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        super().__init__(
-            stream=stream,
-            n_samples=n_samples,
-            feature_names=feature_names,
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names,
-            task=stream.task,
-            n_features=len(feature_names),
-            n_outputs=stream.n_outputs,
-        )
-
-
-if __name__ == "__main__":
-    dataset = Planes2D(random_seed=42)
-    stream = dataset.stream
-    print(stream.n_samples)
-    for n, (x_i, y_i) in enumerate(stream):
-        print(n, x_i, y_i)
-        if n > 3:
-            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
-            break
diff --git a/data/stream/synth/stagger.py b/data/stream/synth/stagger.py
deleted file mode 100644
index 88dd1e4..0000000
--- a/data/stream/synth/stagger.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from typing import Optional
-from river.datasets.synth import STAGGER as RiverDataset
-
-from data.stream._base import StreamDataset
-
-
-class Stagger(StreamDataset):
-
-    def __init__(
-            self,
-            classification_function,
-            random_seed: Optional[int] = None,
-            n_samples: Optional[int] = 10000
-    ):
-        if n_samples is None:
-            n_samples = 10000
-        stream = RiverDataset(classification_function=classification_function, seed=random_seed)
-        feature_names = ['size', 'color', 'shape']
-        cat_feature_names = ['size', 'color', 'shape']
-        num_feature_names = []
-        super().__init__(
-            stream=stream,
-            n_samples=n_samples,
-            feature_names=feature_names,
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names,
-            task=stream.task,
-            n_features=len(feature_names),
-            n_outputs=stream.n_outputs,
-        )
-
-
-if __name__ == "__main__":
-    dataset = Stagger(classification_function=1, random_seed=42)
-    stream = dataset.stream
-    print(stream.n_samples)
-    for n, (x_i, y_i) in enumerate(stream):
-        print(n, x_i, y_i)
-        if n > 3:
-            print("\n", dataset.feature_names, "\n", list(x_i.keys()))
-            break
diff --git a/agrawal_accuracy_loss.py b/examples/agrawal_accuracy_loss.py
similarity index 100%
rename from agrawal_accuracy_loss.py
rename to examples/agrawal_accuracy_loss.py
diff --git a/agrawal_cross_entropy_loss.py b/examples/agrawal_cross_entropy_loss.py
similarity index 100%
rename from agrawal_cross_entropy_loss.py
rename to examples/agrawal_cross_entropy_loss.py
diff --git a/examples/basic_usage.ipynb b/examples/basic_usage.ipynb
new file mode 100644
index 0000000..a94d198
--- /dev/null
+++ b/examples/basic_usage.ipynb
@@ -0,0 +1,280 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# Basic Usage\n",
+    "This notebook illustrates the basic usage to create incremental explanations with our incremental explainers."
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Import Modules\n",
+    "Imports all necessary modules at once."
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from river.metrics import CrossEntropy, Accuracy\n",
+    "from river.utils import Rolling\n",
+    "from river.ensemble import AdaptiveRandomForestClassifier\n",
+    "\n",
+    "from river.datasets.synth import Agrawal\n",
+    "\n",
+    "from increment_explain.explainer import IncrementalPFI\n",
+    "from increment_explain.explainer.sage import IncrementalSage, IntervalSage\n",
+    "from increment_explain.imputer import MarginalImputer\n",
+    "from increment_explain.storage import GeometricReservoirStorage\n",
+    "from increment_explain.utils.wrappers.river import RiverPredictionFunctionWrapper\n",
+    "from increment_explain.visualization import FeatureImportancePlotter"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Setup data stream"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "stream = Agrawal(classification_function=1, seed=42)\n",
+    "feature_names = list([x_0 for x_0, _ in stream.take(1)][0].keys())\n",
+    "n_samples = 25_000"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Setup model and loss function"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "model = AdaptiveRandomForestClassifier(n_models=15, max_depth=10, leaf_prediction='mc')\n",
+    "model_function = RiverPredictionFunctionWrapper(model.predict_proba_one)\n",
+    "\n",
+    "loss_metric = CrossEntropy()\n",
+    "training_metric = Rolling(CrossEntropy(), window_size=1000)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Setup Explainers\n",
+    "The explainer also require an imputer and a storage mechanism."
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "storage = GeometricReservoirStorage(\n",
+    "    size=200,\n",
+    "    store_targets=False\n",
+    ")\n",
+    "\n",
+    "imputer = MarginalImputer(\n",
+    "    model_function=model_function,\n",
+    "    storage_object=storage,\n",
+    "    sampling_strategy=\"joint\"\n",
+    ")\n",
+    "\n",
+    "incremental_sage = IncrementalSage(\n",
+    "    model_function=model_function,\n",
+    "    loss_function=loss_metric,\n",
+    "    imputer=imputer,\n",
+    "    storage=storage,\n",
+    "    feature_names=feature_names,\n",
+    "    smoothing_alpha=0.001,\n",
+    "    n_inner_samples=1\n",
+    ")\n",
+    "\n",
+    "interval_sage = IntervalSage(\n",
+    "    model_function=model_function,\n",
+    "    loss_function=loss_metric,\n",
+    "    feature_names=feature_names,\n",
+    "    interval_length=2000,\n",
+    "    n_inner_samples=1\n",
+    ")\n",
+    "\n",
+    "incremental_pfi = IncrementalPFI(\n",
+    "    model_function=model_function,\n",
+    "    loss_function=loss_metric,\n",
+    "    imputer=imputer,\n",
+    "    storage=storage,\n",
+    "    feature_names=feature_names,\n",
+    "    smoothing_alpha=0.001,\n",
+    "    n_inner_samples=1\n",
+    ")\n",
+    "\n",
+    "sage_plotter = FeatureImportancePlotter(feature_names=feature_names)\n",
+    "pfi_plotter = FeatureImportancePlotter(feature_names=feature_names)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Train and explain the models"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "for (n, (x_i, y_i)) in enumerate(stream, start=1):\n",
+    "    # predicting\n",
+    "    y_i_pred = model.predict_proba_one(x_i)\n",
+    "    training_metric.update(y_true=y_i, y_pred=y_i_pred)\n",
+    "\n",
+    "    # sage inc\n",
+    "    inc_fi_sage = incremental_sage.explain_one(x_i, y_i)\n",
+    "    int_fi_sage = interval_sage.explain_one(x_i, y_i)\n",
+    "    inc_fi_pfi = incremental_pfi.explain_one(x_i, y_i, update_storage=False)\n",
+    "\n",
+    "    # visualize\n",
+    "    sage_plotter.update(inc_fi_sage, facet_name='inc-sage')\n",
+    "    sage_plotter.update(int_fi_sage, facet_name='int-sage')\n",
+    "    pfi_plotter.update(inc_fi_pfi, facet_name='inc-pfi')\n",
+    "\n",
+    "    # learning\n",
+    "    model.learn_one(x_i, y_i)\n",
+    "\n",
+    "    if n % 1000 == 0:\n",
+    "        print(f\"{n}: perf                {training_metric.get()}\\n\"\n",
+    "              f\"{n}: x_i                 {x_i}\\n\"\n",
+    "              f\"{n}: inc-sage            {incremental_sage.importance_values}\\n\"\n",
+    "              f\"{n}: int-sage            {interval_sage.importance_values}\\n\"\n",
+    "              f\"{n}: pfi-sage            {incremental_pfi.importance_values}\\n\"\n",
+    "              f\"{n}: diff                {incremental_sage.marginal_loss - incremental_sage.model_loss}\\n\"\n",
+    "              f\"{n}: marginal-loss       {incremental_sage.marginal_loss}\\n\"\n",
+    "              f\"{n}: model-loss          {incremental_sage.model_loss}\\n\"\n",
+    "              f\"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\\n\")\n",
+    "\n",
+    "    if n >= n_samples:\n",
+    "        break"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Visualize the explanations"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "# TODO add plotter"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/neural_network.py b/examples/neural_network.py
similarity index 100%
rename from neural_network.py
rename to examples/neural_network.py
diff --git a/pandas_example.py b/examples/pandas_example.py
similarity index 100%
rename from pandas_example.py
rename to examples/pandas_example.py
diff --git a/sklearn_example.py b/examples/sklearn_example.py
similarity index 100%
rename from sklearn_example.py
rename to examples/sklearn_example.py
diff --git a/experiments/__init__.py b/experiments/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/experiments/iSAGE_paper/batch_approximation.py b/experiments/iSAGE_paper/batch_approximation.py
deleted file mode 100644
index f4085b1..0000000
--- a/experiments/iSAGE_paper/batch_approximation.py
+++ /dev/null
@@ -1,138 +0,0 @@
-from river.datasets.base import REG
-
-from experiments.setup.data import get_dataset
-from experiments.setup.explainer import get_incremental_sage_explainer, \
-    get_batch_sage_explainer, get_interval_sage_explainer, get_imputer_and_storage
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.explainer.sage import IncrementalSageExplainer, BatchSageExplainer, IntervalSageExplainer
-
-DEBUG = True
-
-DATASET_NAME = 'adult'
-DATASET_RANDOM_SEEDS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-DATASET_SHUFFLE = True
-N_SAMPLES = 20000
-
-MODEL_NAME = 'ARF'
-
-N_INNER_SAMPLES: int = 1
-N_INTERVAL_LENGTH: int = 2000
-SMOOTHING_ALPHA: float = 0.001
-FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product']
-RESERVOIR_LENGTH: int = 100
-RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
-
-TRAINING_METRIC_WINDOW = 100
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 25,
-    }
-}
-
-if __name__ == "__main__":
-
-    inc_final_values = []
-    bat_final_values = []
-
-    for DATASET_RANDOM_SEED in DATASET_RANDOM_SEEDS:
-
-        # Get Training Data --------------------------------------------------------------------------------------------
-        dataset = get_dataset(DATASET_NAME, random_seed=DATASET_RANDOM_SEED, n_samples=N_SAMPLES, shuffle=DATASET_SHUFFLE)
-        feature_names = dataset.feature_names
-        n_samples = dataset.n_samples
-        stream = dataset.stream
-
-        # Get Model / Loss and Training Procedure ----------------------------------------------------------------------
-        task = dataset.task
-        loss_function = get_loss_function(task=task)
-        rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-        model, model_function = get_model(
-            model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-        # Get explainers -----------------------------------------------------------------------------------------------
-        imputer, storage = get_imputer_and_storage(
-            model_function=model_function,
-            feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
-            reservoir_kind=RESERVOIR_KIND,
-            reservoir_length=RESERVOIR_LENGTH,
-            cat_feature_names=None,
-            num_feature_names=None
-        )
-
-        incremental_explainer = IncrementalSageExplainer(
-            model_function=model_function,
-            loss_function=loss_function,
-            imputer=imputer,
-            storage=storage,
-            feature_names=feature_names,
-            smoothing_alpha=SMOOTHING_ALPHA,
-            n_inner_samples=N_INNER_SAMPLES
-        )
-
-        batch_explainer = BatchSageExplainer(
-            model_function=model_function,
-            loss_function=loss_function,
-            feature_names=feature_names,
-            n_inner_samples=N_INNER_SAMPLES
-        )
-
-        interval_explainer = IntervalSageExplainer(
-            model_function=model_function,
-            loss_function=loss_function,
-            feature_names=feature_names,
-            n_inner_samples=N_INNER_SAMPLES,
-            interval_length=N_INTERVAL_LENGTH
-        )
-
-        model_performance = []
-
-        # Training Phase -----------------------------------------------------------------------------------------------
-        if DEBUG:
-            print(f"Starting Training for {n_samples}")
-        for (n, (x_i, y_i)) in enumerate(stream, start=1):
-            if task == REG:
-                y_i_pred = model.predict_one(x_i)
-            else:
-                y_i_pred = model.predict_proba_one(x_i)
-            rolling_training_metric.update(y_true=y_i, y_pred=y_i_pred)
-            model.learn_one(x_i, y_i)
-            if DEBUG and n % 1000 == 0:
-                print(f"{n}: performance {rolling_training_metric.get()}\n")
-            if n > n_samples:
-                break
-
-        # Get Explanation Data -----------------------------------------------------------------------------------------
-        dataset = get_dataset(DATASET_NAME, random_seed=DATASET_RANDOM_SEED, n_samples=N_SAMPLES, shuffle=DATASET_SHUFFLE)
-        n_samples = dataset.n_samples
-        stream = dataset.stream
-
-        # Explanation-Phase --------------------------------------------------------------------------------------------
-        if DEBUG:
-            print(f"Starting Explanation for {n_samples}")
-        for (n, (x_i, y_i)) in enumerate(stream, start=1):
-            inc_fi_values = incremental_explainer.explain_one(x_i, y_i)
-            int_fi_values = interval_explainer.explain_one(x_i, y_i)
-            batch_explainer.update_storage(x_i, y_i)
-            bat_fi_values = batch_explainer.importance_values
-            if DEBUG and n % 1000 == 0:
-                print(f"{n}: inc-sage {incremental_explainer.importance_values}")
-                print(f"{n}: int-sage {interval_explainer.importance_values}")
-                print(f"{n}: bat-sage {batch_explainer.importance_values}\n")
-            if n >= n_samples:
-                bat_fi_values = batch_explainer.explain_one(x_i, y_i)
-                int_fi_values = interval_explainer.explain_one(x_i, y_i, force_explain=True)
-                break
-
-        if DEBUG:
-            print(f"final values: inc-sage {incremental_explainer.importance_values}")
-            print(f"final values: int-sage {interval_explainer.importance_values}")
-            print(f"final values: bat-sage {batch_explainer.importance_values}\n")
-
-        inc_final_values.append(incremental_explainer.importance_values)
-        bat_final_values.append(batch_explainer.importance_values)
-
-    if DEBUG:
-        print("all inc values:", inc_final_values)
-        print("all bat values:", bat_final_values)
diff --git a/experiments/iSAGE_paper/concept_drift.py b/experiments/iSAGE_paper/concept_drift.py
deleted file mode 100644
index 413c4be..0000000
--- a/experiments/iSAGE_paper/concept_drift.py
+++ /dev/null
@@ -1,237 +0,0 @@
-import time
-
-import pandas as pd
-
-from experiments.setup.data import get_dataset, get_concept_drift_dataset
-from experiments.setup.explainer import get_imputer_and_storage
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.explainer import IncrementalPFI
-from increment_explain.explainer.sage import IntervalSageExplainer, IncrementalSageExplainer
-from increment_explain.utils.tracker.exponential_smoothing import ExponentialSmoothingTracker
-from increment_explain.visualization import FeatureImportancePlotter
-
-DEBUG = True
-
-DATASET_1_NAME = 'agrawal 1'
-DATASET_2_NAME = 'agrawal 2'
-DATASET_RANDOM_SEED = 1
-SHUFFLE_DATA = False
-N_SAMPLES_1 = 20000
-N_SAMPLES_2 = 20000
-
-CONCEPT_DRIFT_POSITION = 20000
-CONCEPT_DRIFT_WIDTH = 1
-CONCEPT_DRIFT_SWITCHING_FEATURES = None # {'salary': 'age'}
-
-MODEL_NAME = 'ARF'
-
-N_INNER_SAMPLES: int = 5
-N_INTERVAL_LENGTH: int = 5000
-CONFIDENCE_BOUNDS_DELTA: float = 0.1
-SMOOTHING_ALPHA: float = 0.001
-FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
-RESERVOIR_LENGTH: int = 100
-RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
-
-TRAINING_METRIC_WINDOW = 500
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 25,
-    }
-}
-
-
-if __name__ == "__main__":
-
-    # Get Data ---------------------------------------------------------------------------------------------------------
-    dataset_1 = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
-    dataset_2 = get_dataset(
-        dataset_name=DATASET_2_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_2)
-    dataset = get_concept_drift_dataset(
-        dataset_1_name=DATASET_1_NAME, dataset_2_name=DATASET_2_NAME,
-        dataset_1=dataset_1, dataset_2=dataset_2,
-        position=CONCEPT_DRIFT_POSITION, width=CONCEPT_DRIFT_WIDTH,
-        features_to_switch=CONCEPT_DRIFT_SWITCHING_FEATURES,
-    )
-    feature_names = dataset.feature_names
-    cat_feature_names = dataset.cat_feature_names
-    num_feature_names = dataset.num_feature_names
-    n_samples = dataset.n_samples
-    stream = dataset.stream
-
-    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
-    task = dataset.task
-    loss_function = get_loss_function(task=task)
-    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-    model, model_function = get_model(
-        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-    # Get imputer and explainers ---------------------------------------------------------------------------------------
-    imputer, storage = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
-        reservoir_kind=RESERVOIR_KIND,
-        reservoir_length=RESERVOIR_LENGTH,
-        cat_feature_names=cat_feature_names,
-        num_feature_names=num_feature_names
-    )
-
-    incremental_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    incremental_pfi = IncrementalPFI(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_samples=N_INNER_SAMPLES,
-        storage=storage
-    )
-
-    interval_explainer = IntervalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        feature_names=feature_names,
-        n_inner_samples=N_INNER_SAMPLES,
-        interval_length=N_INTERVAL_LENGTH
-    )
-
-    # warm-up because of errors ----------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        model.learn_one(x_i, y_i)
-        if n > 10:
-            break
-
-    plotter = FeatureImportancePlotter(
-        feature_names=feature_names
-    )
-
-    time_pfi = 0.
-    time_sage = 0.
-    time_interval = 0.
-    time_learning = 0.
-
-    sage_fi_values = []
-    interval_fi_values = []
-    pfi_fi_values = []
-
-    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    model_loss = []
-    marginal_loss = []
-
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        start_time = time.time()
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-        time_sage += time.time() - start_time
-        sage_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='inc')
-
-        # pfi inc
-        start_time = time.time()
-        fi_values = incremental_pfi.explain_one(x_i, y_i, update_storage=False)
-        time_pfi += time.time() - start_time
-        pfi_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='pfi')
-
-        # sage int
-        start_time = time.time()
-        fi_values = interval_explainer.explain_one(x_i, y_i)
-        time_interval += time.time() - start_time
-        interval_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='int')
-
-        # learning
-        start_time = time.time()
-        model.learn_one(x_i, y_i)
-        time_learning += time.time() - start_time
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
-                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
-                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
-                  f"{n}: pfi                 {incremental_pfi.importance_values}\n"
-                  )
-
-        if n >= n_samples:
-            break
-
-    # Store the results in a database ----------------------------------------------------------------------------------
-
-    facet = "left.csv"
-    data_folder = "incremental_setting_5"
-
-    name = "inc_fi_values"
-    df = pd.DataFrame(sage_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "int_fi_values"
-    df = pd.DataFrame(interval_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "pfi_fi_values"
-    df = pd.DataFrame(pfi_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "model_loss"
-    df = pd.DataFrame(model_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "marginal_loss"
-    df = pd.DataFrame(marginal_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
-                      "line_names": ["loss"], "names_to_highlight": ['loss'],
-                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-                      "markevery": {"model_loss": 100, "marginal_loss": 100},
-                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
-                                 "legend_items": [("marginal loss", "dashed", "red"),
-                                                  ("model loss", "solid", "red")]}}
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
-        performance_kw=performance_kw,
-        title=f'Agrawal Stream with Concept Drift',
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
-        v_lines=[{'x': CONCEPT_DRIFT_POSITION, 'ls': '--', 'c': 'black', 'linewidth': 1}],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'inc': '-', 'int': '--', 'pfi': '-'},
-        legend_style={"fontsize": "small"},
-        markevery={'inc': 10, 'int': 1, 'pfi': 10},
-        facet_not_to_highlight=['pfi'],
-        y_min=-0.05,
-        y_max=0.42,
-        save_name="result_data/concept_drift.png"
-    )
diff --git a/experiments/iSAGE_paper/conditional_vs_marginal.py b/experiments/iSAGE_paper/conditional_vs_marginal.py
deleted file mode 100644
index d4b0146..0000000
--- a/experiments/iSAGE_paper/conditional_vs_marginal.py
+++ /dev/null
@@ -1,220 +0,0 @@
-import time
-
-import pandas as pd
-
-from experiments.setup.data import get_dataset
-from experiments.setup.explainer import get_imputer_and_storage
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.explainer.sage import IncrementalSageExplainer
-from increment_explain.utils.tracker.exponential_smoothing import ExponentialSmoothingTracker
-from increment_explain.visualization import FeatureImportancePlotter
-
-DEBUG = True
-
-DATASET_1_NAME = 'elec2'
-
-DATASET_RANDOM_SEED = 1
-SHUFFLE_DATA = False
-N_SAMPLES_1 = None
-
-MODEL_NAME = 'ARF'
-
-N_INNER_SAMPLES: int = 2
-SMOOTHING_ALPHA: float = 0.001
-RESERVOIR_LENGTH: int = 100
-
-TRAINING_METRIC_WINDOW = 500
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 25,
-    }
-}
-
-
-if __name__ == "__main__":
-
-    # Get Data ---------------------------------------------------------------------------------------------------------
-    dataset = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
-    feature_names = dataset.feature_names
-    cat_feature_names = dataset.cat_feature_names
-    num_feature_names = dataset.num_feature_names
-    n_samples = dataset.n_samples
-    stream = dataset.stream
-
-    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
-    task = dataset.task
-    loss_function = get_loss_function(task=task)
-    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-    model, model_function = get_model(
-        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-    # Get imputer and explainers ---------------------------------------------------------------------------------------
-    # conditional
-    imputer_cond, storage_cond = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution="conditional",
-        reservoir_kind=None,
-        reservoir_length=None,
-        cat_feature_names=cat_feature_names,
-        num_feature_names=num_feature_names
-    )
-    incremental_sage_cond = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer_cond,
-        storage=storage_cond,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    # marginal
-    imputer_marg, storage_marg = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution="marginal joint",
-        reservoir_kind="geometric",
-        reservoir_length=RESERVOIR_LENGTH,
-        cat_feature_names=None,
-        num_feature_names=None
-    )
-    incremental_sage_marg = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer_marg,
-        storage=storage_marg,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    # warm-up because of errors ------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        model.learn_one(x_i, y_i)
-        if n > 10:
-            break
-
-    plotter = FeatureImportancePlotter(
-        feature_names=feature_names
-    )
-
-    time_marg = 0.
-    time_cond = 0.
-    time_learning = 0.
-
-    sage_cond_fi_values = []
-    sage_marg_fi_values = []
-
-    model_loss_tracker_cond = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    marginal_loss_tracker_cond = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    model_loss_cond = []
-    marginal_loss_cond = []
-
-    model_loss_tracker_marg = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    marginal_loss_tracker_marg = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    model_loss_marg = []
-    marginal_loss_marg = []
-
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker_cond.update(loss_i)
-        model_loss_cond.append({"loss": model_loss_tracker_cond.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage_cond._marginal_prediction_tracker.get())
-        marginal_loss_tracker_cond.update(loss_marginal_i)
-        marginal_loss_cond.append({"loss": marginal_loss_tracker_cond.get()})
-
-        model_loss_tracker_marg.update(loss_i)
-        model_loss_marg.append({"loss": model_loss_tracker_marg.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage_cond._marginal_prediction_tracker.get())
-        marginal_loss_tracker_marg.update(loss_marginal_i)
-        marginal_loss_marg.append({"loss": marginal_loss_tracker_marg.get()})
-
-        # sage inc cond
-        start_time = time.time()
-        fi_values = incremental_sage_cond.explain_one(x_i, y_i)
-        time_cond += time.time() - start_time
-        sage_cond_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='cond')
-
-        # sage inc marg
-        start_time = time.time()
-        fi_values = incremental_sage_marg.explain_one(x_i, y_i)
-        time_marg += time.time() - start_time
-        sage_marg_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='marg')
-
-        # learning
-        start_time = time.time()
-        model.learn_one(x_i, y_i)
-        time_learning += time.time() - start_time
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: conditional\n"
-                  f"{n}: marginal-prediction {incremental_sage_cond._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker_cond.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker_cond.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker_cond.get() - model_loss_tracker_cond.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage_cond.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage_cond.importance_values}\n"
-                  f"{n}: marginal\n"
-                  f"{n}: marginal-prediction {incremental_sage_marg._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker_marg.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker_marg.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker_marg.get() - model_loss_tracker_marg.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage_marg.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage_marg.importance_values}\n"
-                  )
-
-        if n >= n_samples:
-            break
-
-    # Store the results in a database ----------------------------------------------------------------------------------
-
-    data_folder = "conditional_marginal"
-    facet = "right.csv"
-
-    name = "cond_fi_values"
-    df = pd.DataFrame(sage_cond_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "marg_fi_values"
-    df = pd.DataFrame(sage_marg_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "model_loss"
-    df = pd.DataFrame(model_loss_cond)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "marginal_loss"
-    df = pd.DataFrame(marginal_loss_cond)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "cross\nentropy", "color_list": ["red", "black"],
-                      "line_names": ["loss"], "names_to_highlight": ['loss'],
-                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-                      "markevery": {"model_loss": 100, "marginal_loss": 100},
-                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
-                                 "legend_items": [("marginal loss", "dashed", "red"),
-                                                  ("model loss", "solid", "red")]}}
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"model_loss": model_loss_cond, "marginal_loss": marginal_loss_cond},
-        performance_kw=performance_kw,
-        title=f'Agrawal Stream with Concept Drift',
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'cond': '-', 'marg': '--'},
-        legend_style={"fontsize": "small"},
-        markevery={'cond': 100, 'marg': 100},
-        y_min=-0.049,
-        y_max=0.325,
-        save_name="result_data/conditional_vs_marginal.png"
-    )
diff --git a/experiments/iSAGE_paper/confidence_bounds.py b/experiments/iSAGE_paper/confidence_bounds.py
deleted file mode 100644
index 9ee73b0..0000000
--- a/experiments/iSAGE_paper/confidence_bounds.py
+++ /dev/null
@@ -1,205 +0,0 @@
-import math
-
-import pandas as pd
-
-from experiments.setup.data import get_dataset
-from experiments.setup.explainer import get_imputer_and_storage
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.explainer.sage import IncrementalSageExplainer
-from increment_explain.utils.tracker.exponential_smoothing import ExponentialSmoothingTracker
-from increment_explain.visualization import FeatureImportancePlotter
-
-DEBUG = True
-
-DATASET_1_NAME = 'elec2'
-DATASET_RANDOM_SEED = 1
-SHUFFLE_DATA = False
-N_SAMPLES_1 = None
-
-
-CONCEPT_DRIFT_WIDTH = 1
-CONCEPT_DRIFT_SWITCHING_FEATURES = None # {'salary': 'age'}
-
-TRAINING_METRIC_WINDOW = 500
-MODEL_NAME = 'ARF'
-
-CONFIDENCE_BOUNDS_1_DELTA = 0.1
-CONFIDENCE_BOUNDS_2_DELTA = 0.05
-N_INNER_SAMPLES: int = 1
-N_INTERVAL_LENGTH: int = 2000
-SMOOTHING_ALPHA: float = 0.001
-FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
-RESERVOIR_LENGTH: int = 100
-RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 25,
-    }
-}
-
-
-if __name__ == "__main__":
-
-    # Get Data ---------------------------------------------------------------------------------------------------------
-    dataset = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
-    feature_names = dataset.feature_names
-    cat_feature_names = dataset.cat_feature_names
-    num_feature_names = dataset.num_feature_names
-    n_samples = dataset.n_samples
-    stream = dataset.stream
-
-    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
-    task = dataset.task
-    loss_function = get_loss_function(task=task)
-    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-    model, model_function = get_model(
-        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-    # Get explainers ---------------------------------------------------------------------------------------------------
-    imputer, storage = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
-        reservoir_kind=RESERVOIR_KIND,
-        reservoir_length=RESERVOIR_LENGTH,
-        cat_feature_names=cat_feature_names,
-        num_feature_names=num_feature_names
-    )
-
-    incremental_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    # warm-up because of errors ------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        model.learn_one(x_i, y_i)
-        if n > 10:
-            break
-
-    plotter = FeatureImportancePlotter(
-        feature_names=feature_names
-    )
-
-    sage_fi_values = []
-
-    confidence_bounds_1 = []
-    confidence_bounds_2 = []
-    std = []
-
-    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    model_loss = []
-    marginal_loss = []
-
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-
-        # storing data
-        plotter.update(fi_values, facet_name='inc')
-        sage_fi_values.append(fi_values)
-        confidence_bounds_1.append(incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_1_DELTA))
-        confidence_bounds_2.append(incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_2_DELTA))
-        std.append({feature: math.sqrt(value.get()) for feature, value in incremental_sage.variances.items()})
-
-        # learning
-        model.learn_one(x_i, y_i)
-
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: performance       {rolling_training_metric.get()}\n"
-                  f"{n}: inc-sage          {incremental_sage.importance_values}\n"
-                  f"{n}: sum-sage          {sum(list(incremental_sage.importance_values.values()))}\n"
-                  f"{n}: inc-sage-bnd (1)  {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_1_DELTA)}\n"
-                  f"{n}: inc-sage-bnd (2)  {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_2_DELTA)}\n"
-                  f"{n}: inc-sage-var      {incremental_sage.variances}\n"
-                  )
-
-        if n >= n_samples:
-            break
-
-    # Store the results in a database ----------------------------------------------------------------------------------
-
-    facet = ".csv"
-    data_folder = "confidence_bounds"
-
-    name = "inc_fi_values"
-    df = pd.DataFrame(sage_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "confidence_bounds_1"
-    df = pd.DataFrame(confidence_bounds_1)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "confidence_bounds_2"
-    df = pd.DataFrame(confidence_bounds_2)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "model_loss"
-    df = pd.DataFrame(model_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "marginal_loss"
-    df = pd.DataFrame(marginal_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "cross\nentropy", "color_list": ["red", "black"],
-                      "line_names": ["loss"], "names_to_highlight": ['loss'],
-                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-                      "markevery": {"model_loss": 100, "marginal_loss": 100},
-                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
-                                 "legend_items": [("marginal loss", "dashed", "red"),
-                                                  ("model loss", "solid", "red")]}}
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
-        performance_kw=performance_kw,
-        title=f"{DATASET_1_NAME} data stream, confidence bounds with " + r"$\delta =$ " + f"{CONFIDENCE_BOUNDS_1_DELTA}",
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['nswprice', 'vicprice', 'nswdemand'],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'inc': '-'},
-        legend_style={},
-        markevery={'inc': 10},
-        std={'inc': confidence_bounds_1},
-        y_min=-0.1,
-        y_max=0.75,
-        save_name="result_data/confidence_bounds_delta_1.png"
-    )
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
-        performance_kw=performance_kw,
-        title=f"{DATASET_1_NAME} data stream, confidence bounds with " + r"$\delta =$ " + f"{CONFIDENCE_BOUNDS_2_DELTA}",
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['nswprice', 'vicprice', 'nswdemand'],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'inc': '-'},
-        legend_style={},
-        markevery={'inc': 10},
-        std={'inc': confidence_bounds_2},
-        y_min=-0.1,
-        y_max=0.75,
-        save_name="result_data/confidence_bounds_delta_2.png"
-    )
diff --git a/experiments/iSAGE_paper/explaining_change.py b/experiments/iSAGE_paper/explaining_change.py
deleted file mode 100644
index b778cfa..0000000
--- a/experiments/iSAGE_paper/explaining_change.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import pandas as pd
-from river.datasets.base import REG
-from river.drift import ADWIN
-
-from experiments.setup.data import get_dataset
-from experiments.setup.explainer import get_incremental_sage_explainer
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.visualization import FeatureImportancePlotter
-
-DEBUG = True
-
-DATASET_1_NAME = 'elec2'
-
-DATASET_RANDOM_SEED = 1
-SHUFFLE_DATA = False
-N_SAMPLES_1 = None
-
-CONCEPT_DRIFT_SWITCHING_FEATURES = None  # {'salary': 'age'}
-
-MODEL_NAME = 'ARF'
-
-N_INNER_SAMPLES = 1
-
-TRAINING_METRIC_WINDOW = 500
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 25,
-    }
-}
-
-
-if __name__ == "__main__":
-
-    # Get Data ---------------------------------------------------------------------------------------------------------
-    dataset = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
-    feature_names = dataset.feature_names
-    n_samples = dataset.n_samples
-    stream = dataset.stream
-
-    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
-    task = dataset.task
-    loss_function = get_loss_function(task=task)
-    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-    model, model_function = get_model(
-        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-    # Get explainers ---------------------------------------------------------------------------------------------------
-    incremental_sage = get_incremental_sage_explainer(
-        feature_names=feature_names,
-        model_function=model_function,
-        loss_function=loss_function,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    # Change Detectors -------------------------------------------------------------------------------------------------
-    fi_detectors = {feature_name: ADWIN(delta=0.001) for feature_name in feature_names}
-    change_points = []
-    change_points_fi = []
-
-    # warm-up because of errors ----------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        model.learn_one(x_i, y_i)
-        if n > 10:
-            break
-
-    # storing ----------------------------------------------------------------------------------------------------------
-    plotter = FeatureImportancePlotter(
-        feature_names=feature_names
-    )
-    model_performance = []
-
-    # run --------------------------------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-
-        # predicting
-        if task == REG:
-            y_i_pred = model.predict_one(x_i)
-        else:
-            y_i_pred = model.predict_proba_one(x_i)
-        rolling_training_metric.update(y_true=y_i, y_pred=y_i_pred)
-        model_performance.append(rolling_training_metric.get())
-
-        # sage inc
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-
-        # storing data
-        plotter.update(fi_values, facet_name='inc')
-
-        # change detection
-        detected_drifts = 0
-        for feature_name in feature_names:
-            fi_detector = fi_detectors[feature_name]
-            fi_detector.update(fi_values[feature_name])
-            if fi_detector.drift_detected:
-                detected_drifts += 1
-
-        if detected_drifts >= 1:
-            change_points.append(n)
-            change_points_fi.append({**incremental_sage.importance_values, 'number_of_drifts': detected_drifts})
-
-        # learning
-        model.learn_one(x_i, y_i)
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i               {x_i}\n"
-                  f"{n}: inc-sage          {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-var      {incremental_sage.variances}\n"
-                  )
-
-        if n >= n_samples:
-            break
-
-    # Store the results in a database ----------------------------------------------------------------------------------
-
-    #v_lines = [{'x': CONCEPT_DRIFT_POSITION, 'ls': '--', 'c': 'black', 'linewidth': 1}]
-    v_lines = []
-    v_lines.extend([{'x': fi_drift, 'ls': '--', 'c': 'gray', 'linewidth': 1}
-                    for fi_drift in change_points])
-    plotter.plot(
-        figsize=(5, 10),
-        model_performance={"cross\nentropy": model_performance},
-        title=f'Agrawal Stream with Concept Drift',
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['nswprice', 'vicprice', 'nswdemand'],
-        v_lines=v_lines,
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'inc': '-'},
-        legend_style={},
-        markevery={'inc': 10},
-        y_min=-0.1,
-        y_max=0.75
-    )
-
-
-    change_points_fi = pd.DataFrame(change_points_fi)
-    change_points_fi['t'] = change_points
-    change_points_fi.to_csv("result_data/changes_detected.csv", index=False)
diff --git a/experiments/iSAGE_paper/plots/conditional_marginal.py b/experiments/iSAGE_paper/plots/conditional_marginal.py
deleted file mode 100644
index fd9bfd5..0000000
--- a/experiments/iSAGE_paper/plots/conditional_marginal.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import copy
-import os
-import pandas as pd
-
-from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
-
-if __name__ == "__main__":
-    SAVE_PATH = "conditional_marginal/conditional_marginal.png"
-
-    # load data --------------------------------------------------------------------------------------------------------
-    DATA_DIR = "conditional_marginal"
-    # left data
-    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
-    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
-    cond_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "cond_fi_values_left.csv"))
-    marg_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "marg_fi_values_left.csv"))
-    # right data
-
-    marginal_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_right.csv"))
-    model_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_right.csv"))
-    cond_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "cond_fi_values_right.csv"))
-    marg_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "marg_fi_values_right.csv"))
-
-    # plot data --------------------------------------------------------------------------------------------------------
-    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
-                     'line_names': ['loss']}
-    top_right_data = {'y_data': {"model_loss": model_loss_right_df, "marginal_loss": marginal_loss_right_df},
-                      'line_names': ['loss']}
-    bot_left_data = {'y_data': {'cond': cond_fi_values_left_df,
-                                'marg': marg_fi_values_left_df},
-                     'line_names': list(cond_fi_values_left_df.columns)}
-    bot_right_data = {'y_data': {'cond': cond_fi_values_right_df,
-                                 'marg': marg_fi_values_right_df},
-                      'line_names': list(cond_fi_values_right_df.columns)}
-
-    # plot styles ------------------------------------------------------------------------------------------------------
-
-    # top styles / performance row
-    top_styles = {
-        "y_min": 0, "y_max": 1,
-        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
-        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-        "fill_between_props": [{
-            'facet_1': 'model_loss', 'facet_2': 'marginal_loss',
-            'line_name_1': 'loss', 'line_name_2': 'loss',
-            'hatch': '///',
-            'color': 'red',
-            'alpha': 0.1,
-        }]
-    }
-    top_right_styles = copy.deepcopy(top_styles)
-    top_left_styles = copy.deepcopy(top_styles)
-    top_left_styles["y_label"] = "loss"
-    top_left_styles["title"] = f"agrawal stream"
-    top_right_styles["title"] = f"elec2 stream"
-    #top_right_styles['y_ticks'] = []
-
-    # bottom styles / feature importance row
-    bot_styles = {
-        "x_label": "Samples",
-        "h_lines": [{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        "line_styles": {'cond': 'solid', 'marg': 'dotted'},
-        "markevery": {'cond': 30, 'marg': 30, },
-    }
-    bot_left_styles = copy.deepcopy(bot_styles)
-    bot_left_styles["y_label"] = "SAGE values"
-    bot_left_styles["legend_style"] = {"fontsize": "small", 'title': 'features'}
-    bot_left_styles["names_to_highlight"] = ['salary', 'commission', 'age']
-    bot_left_styles["y_min"] = -0.04
-    bot_left_styles["y_max"] = 0.19
-    bot_left_styles["y_ticks"] = [0., 0.05, 0.1, 0.15]
-    bot_right_styles = copy.deepcopy(bot_styles)
-    bot_right_styles["y_min"] = -0.09
-    bot_right_styles["y_max"] = 0.598
-    bot_right_styles["y_ticks"] = [0., 0.1, 0.2, 0.3, 0.4, 0.5]
-    bot_right_styles["legend_style"] = {"fontsize": "small", 'title': 'features'}
-    bot_right_styles["names_to_highlight"] = ['nswprice', 'vicprice']
-
-    # legends
-    top_right_styles["secondary_legends"] = [{
-        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", 'title': 'loss', "borderaxespad": 0,
-                         "bbox_to_anchor": (1.02, 1), "frameon": False},
-        "legend_items": [("mean prediction", "dashed", "red"),
-                         ("model", "solid", "red")]
-    }]
-    bot_right_styles["secondary_legends"] = [
-        {
-        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", "title": "distribution", "borderaxespad": 0,
-                         "bbox_to_anchor": (1.02, 1), "frameon": False},
-        "legend_items": [("marg.", "solid", "black"),
-                         ("cond.", "dotted", "black")]
-    }]
-
-    plot_double_stacked_line_plots(
-        figsize=(12.5, 5),
-        top_left_data=top_left_data,
-        top_left_styles=top_left_styles,
-        bot_left_data=bot_left_data,
-        bot_left_styles=bot_left_styles,
-        top_right_data=top_right_data,
-        top_right_styles=top_right_styles,
-        bot_right_data=bot_right_data,
-        bot_right_styles=bot_right_styles,
-        top_portion=0.25,
-        show=True,
-        save_path=SAVE_PATH,
-        different_scales=True,
-        wspace=0.08
-    )
\ No newline at end of file
diff --git a/experiments/iSAGE_paper/plots/confidence_bounds.py b/experiments/iSAGE_paper/plots/confidence_bounds.py
deleted file mode 100644
index e19ea46..0000000
--- a/experiments/iSAGE_paper/plots/confidence_bounds.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import copy
-import os
-import pandas as pd
-
-from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots
-
-
-if __name__ == "__main__":
-    DATA_DIR = "confidence_bounds"
-
-    SAVE_PATH = os.path.join(DATA_DIR, "confidence_bounds.png")
-
-    # load data --------------------------------------------------------------------------------------------------------
-    marginal_loss_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_.csv"))
-    model_loss_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_.csv"))
-    sage_fi_values_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_.csv"))
-    confidence_bounds_1_df = pd.read_csv(os.path.join(DATA_DIR, "confidence_bounds_1_.csv"))
-    confidence_bounds_2_df = pd.read_csv(os.path.join(DATA_DIR, "confidence_bounds_2_.csv"))
-
-    # plot data --------------------------------------------------------------------------------------------------------
-    top_left_data = {'y_data': {"model_loss": model_loss_df, "marginal_loss": marginal_loss_df},
-                     'line_names': ['loss']}
-    top_right_data = top_left_data
-    bot_left_data = {'y_data': {'inc': sage_fi_values_df},
-                     'line_names': list(sage_fi_values_df.columns)}
-    bot_right_data = bot_left_data
-
-    # plot styles ------------------------------------------------------------------------------------------------------
-
-    # top styles / performance row
-    top_styles = {
-        "y_min": 0, "y_max": 1,
-        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
-        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-        "fill_between_props": [{'facet_1': 'model_loss', 'facet_2': 'marginal_loss','line_name_1': 'loss', 'line_name_2': 'loss','hatch': '///','color': 'red','alpha': 0.1}]
-    }
-    top_right_styles = copy.copy(top_styles)
-    top_left_styles = copy.copy(top_styles)
-
-    top_left_styles["y_label"] = "loss"
-    top_left_styles["title"] = r"$\delta =$ " + f"{0.1}"
-    top_right_styles["title"] = r"$\delta =$ " + f"{0.05}"
-
-    # bottom styles / feature importance row
-    bot_styles = {
-        "y_min": -0.15, "y_max": 0.79, "x_label": "Samples",
-        "names_to_highlight": ['nswprice', 'vicprice', 'nswdemand'],
-        "h_lines": [{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-    }
-    bot_left_styles = copy.copy(bot_styles)
-    bot_left_styles["std"] = {'inc': confidence_bounds_1_df}
-    bot_left_styles["y_label"] = "SAGE values"
-    bot_right_styles = copy.copy(bot_styles)
-    bot_right_styles["std"] = {'inc': confidence_bounds_2_df}
-
-    # legends
-    top_right_styles["secondary_legends"] = [{
-        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", 'title': 'loss', "borderaxespad": 0,
-                         "bbox_to_anchor": (1.02, 1), "frameon": False},
-        "legend_items": [("mean prediction", "dashed", "red"),
-                         ("model", "solid", "red")]
-    }]
-    bot_right_styles["legend_style"] = {
-        "fontsize": "small", 'title': 'features', "borderaxespad": 0, "frameon": False,
-        "loc": 'upper left', "bbox_to_anchor": (1.02, 1)}
-
-    plot_double_stacked_line_plots(
-        figsize=(12.5, 5),
-        top_left_data=top_left_data,
-        top_left_styles=top_left_styles,
-        bot_left_data=bot_left_data,
-        bot_left_styles=bot_left_styles,
-        top_right_data=top_right_data,
-        top_right_styles=top_right_styles,
-        bot_right_data=bot_right_data,
-        bot_right_styles=bot_right_styles,
-        top_portion=0.25, title="Confidence Bounds for elec2 data stream",
-        show=True,
-        save_path=SAVE_PATH
-    )
diff --git a/experiments/iSAGE_paper/plots/incremental_setting.py b/experiments/iSAGE_paper/plots/incremental_setting.py
deleted file mode 100644
index 11311cf..0000000
--- a/experiments/iSAGE_paper/plots/incremental_setting.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import copy
-import os
-import pandas as pd
-
-from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
-
-
-if __name__ == "__main__":
-    DATA_DIR = "incremental_setting_5"
-
-    SAVE_PATH = os.path.join(DATA_DIR, "incremental_setting.png")
-    SAVE_PATH_LEFT = os.path.join(DATA_DIR, "incremental_setting_left.png")
-    SAVE_PATH_RIGHT = os.path.join(DATA_DIR, "incremental_setting_right.png")
-
-    # load data --------------------------------------------------------------------------------------------------------
-    # left data
-    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
-    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
-    inc_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_left.csv"))
-    int_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_left.csv"))
-    pfi_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "pfi_fi_values_left.csv"))
-    # right data
-    marginal_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_right.csv"))
-    model_loss_right_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_right.csv"))
-    inc_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_right.csv"))
-    int_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_right.csv"))
-    inc_welford_fi_values_right_df = pd.read_csv(os.path.join(DATA_DIR, "inc_welford_fi_values_right.csv"))
-
-    # plot data --------------------------------------------------------------------------------------------------------
-    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
-                     'line_names': ['loss']}
-    top_right_data = {'y_data': {"model_loss": model_loss_right_df, "marginal_loss": marginal_loss_right_df},
-                      'line_names': ['loss']}
-    bot_left_data = {'y_data': {'inc': inc_fi_values_left_df,
-                                'int': int_fi_values_left_df,
-                                },
-                     'line_names': list(inc_fi_values_left_df.columns)}
-    bot_right_data = {'y_data': {'inc': inc_fi_values_right_df,
-                                'bat': inc_welford_fi_values_right_df
-                                },
-                      'line_names': list(inc_fi_values_right_df.columns)}
-
-    # plot styles ------------------------------------------------------------------------------------------------------
-
-    # top styles / performance row
-    top_styles = {
-        "y_min": 0, "y_max": 1,
-        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
-        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-        "fill_between_props": [{
-            'facet_1': 'model_loss', 'facet_2': 'marginal_loss',
-            'line_name_1': 'loss', 'line_name_2': 'loss',
-            'hatch': '///',
-            'color': 'red',
-            'alpha': 0.1,
-        }]
-    }
-    top_right_styles = copy.deepcopy(top_styles)
-    top_left_styles = copy.deepcopy(top_styles)
-    top_left_styles["y_label"] = "loss"
-    top_left_styles["title"] = f"iSAGE vs. interval SAGE (agrawal function 2 to 3)"
-    top_left_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
-    top_right_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
-    top_right_styles["title"] = f"iSAGE vs. SAGE (agrawal function 3 to 2)"
-
-    # bottom styles / feature importance row
-    bot_styles = {
-        "y_min": -0.03, "y_max": 0.24, "x_label": "Samples",
-        "names_to_highlight": ['salary', 'commission', 'age', 'elevel'],
-        "h_lines": [{'y': 0., 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}],
-        "line_styles": {'inc': 'solid', 'int': 'dashed', 'bat': 'dashdot', 'pfi': 'dotted'},
-        "markevery": {'inc': 100, 'int': 1, 'bat': 100, 'pfi': 100},
-        "x_ticks": [0, 10_000, 20_000, 30_000, 40_000],
-        "y_ticks": [0, 0.1, 0.2],
-    }
-    bot_left_styles = copy.deepcopy(bot_styles)
-    bot_left_styles["y_label"] = "SAGE values"
-    bot_left_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
-    bot_right_styles = copy.deepcopy(bot_styles)
-    bot_right_styles["v_lines"] = [{'x': 20_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
-
-
-
-
-    # legends
-    top_right_styles["secondary_legends"] = [{
-        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", 'title': 'loss', "borderaxespad": 0,
-                         "bbox_to_anchor": (1.02, 1), "frameon": False},
-        "legend_items": [("mean prediction", "dashed", "red"),
-                         ("model", "solid", "red")]
-    }]
-    bot_right_styles["legend_style"] = {
-        "fontsize": "small", 'title': 'features', "borderaxespad": 0, "frameon": False,
-        "loc": 'upper left', "bbox_to_anchor": (1.02, 1)}
-    bot_right_styles["secondary_legends"] = [{
-        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", "title": "explainer", "borderaxespad": 0,
-                         "bbox_to_anchor": (1.02, 0.6), "frameon": False},
-        "legend_items": [("iSAGE", "solid", "black"),
-                         ("interval SAGE", "dashed", "black"),
-                         #("iPFI", "dotted", 'black'),
-                         ("SAGE", "dashdot", "black")]
-    }]
-
-    plot_double_stacked_line_plots(
-        figsize=(12.5, 5),
-        top_left_data=top_left_data,
-        top_left_styles=top_left_styles,
-        bot_left_data=bot_left_data,
-        bot_left_styles=bot_left_styles,
-        top_right_data=top_right_data,
-        top_right_styles=top_right_styles,
-        bot_right_data=bot_right_data,
-        bot_right_styles=bot_right_styles,
-        top_portion=0.25,
-        show=True,
-        save_path=SAVE_PATH,
-        wspace=0.03
-        #title=r"iSAGE on agrawal streams ($\alpha: 0.001$)"
-    )
-
-    '''
-    plot_stacked_line_plots(
-        figsize=(12.5, 5),
-        top_data=top_left_data,
-        top_styles=top_left_styles,
-        bot_data=bot_left_data,
-        bot_styles=bot_left_styles,
-        show=True,
-        save_path=SAVE_PATH_LEFT
-    )
-
-    plot_stacked_line_plots(
-        figsize=(12.5, 5),
-        top_data=top_right_data,
-        top_styles=top_right_styles,
-        bot_data=bot_right_data,
-        bot_styles=bot_right_styles,
-        show=True,
-        save_path=SAVE_PATH_RIGHT
-    )
-    '''
diff --git a/experiments/iSAGE_paper/plots/reoccurring_drift.py b/experiments/iSAGE_paper/plots/reoccurring_drift.py
deleted file mode 100644
index 8145492..0000000
--- a/experiments/iSAGE_paper/plots/reoccurring_drift.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import copy
-import os
-import pandas as pd
-
-from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
-
-
-if __name__ == "__main__":
-    DATA_DIR = "reoccurring_drift"
-
-    SAVE_PATH_LEFT = os.path.join(DATA_DIR, "reoccurring_drift.png")
-
-    # load data --------------------------------------------------------------------------------------------------------
-    # left data
-    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
-    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
-    inc_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_left.csv"))
-    int_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_left.csv"))
-    inc_welford_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_welford_fi_values_left.csv"))
-
-
-    # plot data --------------------------------------------------------------------------------------------------------
-    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
-                     'line_names': ['loss']}
-    bot_left_data = {'y_data': {'inc': inc_fi_values_left_df,
-                                'int': int_fi_values_left_df,
-                                'bat': inc_welford_fi_values_left_df
-                                },
-                     'line_names': list(inc_fi_values_left_df.columns)}
-
-    # plot styles ------------------------------------------------------------------------------------------------------
-
-    # top styles / performance row
-    top_styles = {
-        "y_min": 0, "y_max": 1,
-        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
-        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-        "fill_between_props": [{
-            'facet_1': 'model_loss', 'facet_2': 'marginal_loss',
-            'line_name_1': 'loss', 'line_name_2': 'loss',
-            'hatch': '///',
-            'color': 'red',
-            'alpha': 0.1,
-        }]
-    }
-    top_left_styles = copy.deepcopy(top_styles)
-    top_left_styles["y_label"] = "loss"
-    top_left_styles["title"] = f"iSAGE vs. interval SAGE (agrawal function 2 to 3)"
-    top_left_styles["v_lines"] = [{'x': 25_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
-                                  {'x': 50_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
-                                  {'x': 75_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
-
-    # bottom styles / feature importance row
-    bot_styles = {
-        "y_min": -0.03, "y_max": 0.31, "x_label": "Samples",
-        "names_to_highlight": ['salary', 'commission', 'age', 'elevel'],
-        "h_lines": [{'y': 0., 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}],
-        "line_styles": {'inc': 'solid', 'int': 'dashed', 'bat': 'dashdot', 'pfi': 'dotted'},
-        "markevery": {'inc': 100, 'int': 1, 'bat': 100, 'pfi': 100},
-        "x_ticks": [0, 10_000, 20_000, 30_000, 40_000, 50_000, 60_000, 70_000, 80_000, 90_000, 100_000],
-        "y_ticks": [0, 0.1, 0.2],
-    }
-    bot_left_styles = copy.deepcopy(bot_styles)
-    bot_left_styles["y_label"] = "SAGE values"
-    bot_left_styles["v_lines"] = [{'x': 25_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
-                                  {'x': 50_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1},
-                                  {'x': 75_000, 'ls': 'dotted', 'c': 'grey', 'linewidth': 1}]
-
-
-    # legends
-    top_left_styles["secondary_legends"] = [{
-        "legend_props": {"loc": 'center right', "ncol": 1, "fontsize": "small", 'title': 'loss'},
-        "legend_items": [("mean prediction", "dashed", "red"),
-                         ("model", "solid", "red")]
-    }]
-    bot_left_styles["legend_style"] = {
-        "fontsize": "small", 'title': 'features', "ncol": 2,
-        "loc": 'upper left', "bbox_to_anchor": (0, 1)}
-    bot_left_styles["secondary_legends"] = [{
-        "legend_props": {"loc": 'upper left', "ncol": 1, "fontsize": "small", "title": "explainer",
-                         "bbox_to_anchor": (0.18, 1)},
-        "legend_items": [("iSAGE", "solid", "black"),
-                         ("interval SAGE", "dashed", "black"),
-                         #("iPFI", "dotted", 'black'),
-                         ("SAGE", "dashdot", "black")]
-    }]
-
-
-    plot_stacked_line_plots(
-        figsize=(12.5, 5),
-        top_data=top_left_data,
-        top_styles=top_left_styles,
-        bot_data=bot_left_data,
-        bot_styles=bot_left_styles,
-        show=True,
-        save_path=SAVE_PATH_LEFT,
-        top_portion=0.25
-    )
diff --git a/experiments/iSAGE_paper/plots/static_sage_in_dynamic_setting.py b/experiments/iSAGE_paper/plots/static_sage_in_dynamic_setting.py
deleted file mode 100644
index aa553df..0000000
--- a/experiments/iSAGE_paper/plots/static_sage_in_dynamic_setting.py
+++ /dev/null
@@ -1,78 +0,0 @@
-import copy
-import os
-import pandas as pd
-
-from experiments.visualization.plot_feature_importance import plot_double_stacked_line_plots, plot_stacked_line_plots
-
-
-if __name__ == "__main__":
-    DATA_DIR = "static_sage_in_dynamic_setting"
-
-    SAVE_PATH = os.path.join(DATA_DIR, "static_sage_in_dynamic_setting.png")
-
-    # load data --------------------------------------------------------------------------------------------------------
-    # left data
-    marginal_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "marginal_loss_left.csv"))
-    model_loss_left_df = pd.read_csv(os.path.join(DATA_DIR, "model_loss_left.csv"))
-    inc_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_fi_values_left.csv"))
-    int_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "int_fi_values_left.csv"))
-    inc_welford_fi_values_left_df = pd.read_csv(os.path.join(DATA_DIR, "inc_welford_fi_values_left.csv"))
-
-    # plot data --------------------------------------------------------------------------------------------------------
-    top_left_data = {'y_data': {"model_loss": model_loss_left_df, "marginal_loss": marginal_loss_left_df},
-                     'line_names': ['loss']}
-    bot_left_data = {'y_data': {'inc': inc_fi_values_left_df,
-                                #'int': int_fi_values_left_df,
-                                'bat': inc_welford_fi_values_left_df},
-                     'line_names': list(inc_fi_values_left_df.columns)}
-
-    # plot styles ------------------------------------------------------------------------------------------------------
-
-    # top styles / performance row
-    top_styles = {
-        "y_min": 0, "y_max": 1,
-        "color_list": ["red", "black"], "markevery": {"model_loss": 100, "marginal_loss": 100},
-        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-        "fill_between_props": [
-            {'facet_1': 'model_loss', 'facet_2': 'marginal_loss', 'line_name_1': 'loss', 'line_name_2': 'loss',
-             'hatch': '///', 'color': 'red', 'alpha': 0.1}]
-    }
-    top_left_styles = copy.deepcopy(top_styles)
-    top_left_styles["y_label"] = "loss"
-    top_left_styles["v_lines"] = [{'x': 20_000, 'ls': '--', 'c': 'grey', 'linewidth': 1}]
-    top_left_styles["secondary_legends"] = [{
-        "legend_props": {"loc": "upper left", "ncol": 1, "fontsize": "small"},
-        "legend_items": [("mean prediction loss", "dashed", "red"), ("model loss", "solid", "red")]
-    }]
-    top_left_styles["y_ticks"] = [0, 0.5, 1.0]
-
-    # bottom styles / feature importance row
-    bot_styles = {
-        "y_min": -0.0499, "y_max": 0.29, "x_label": "Samples",
-        "names_to_highlight": ['salary', 'commission', 'age', 'elevel'],
-        "h_lines": [{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        "line_styles": {'inc': 'solid', 'int': 'dashed', 'bat': 'dashdot'},
-        "markevery": {'inc': 100, 'int': 1, 'bat': 100},
-        "x_ticks": [0, 5_000, 10_000,  15_000, 20_000, 25_000, 30_000, 35_000, 40_000]
-    }
-    bot_left_styles = copy.deepcopy(bot_styles)
-    bot_left_styles["y_label"] = "SAGE values"
-    bot_left_styles["v_lines"] = [{'x': 20_000, 'ls': '--', 'c': 'grey', 'linewidth': 1}]
-    bot_left_styles["legend_style"] = {"fontsize": "small", 'title': 'features', "loc": "upper left"}
-    bot_left_styles['facet_not_to_highlight'] = ['int']
-    bot_left_styles["y_ticks"] = [0, 0.1, 0.2]
-    bot_left_styles["secondary_legends"] = [{
-        "legend_props": {"loc": "upper left", "ncol": 1, "fontsize": "small", "bbox_to_anchor": (0.1, 1), "title": "explainer"},
-        "legend_items": [("iSAGE", "solid", "black"), ("SAGE", "dashdot", "black")]
-    }]
-
-    plot_stacked_line_plots(
-        figsize=(12.5, 5),
-        top_data=top_left_data,
-        top_styles=top_left_styles,
-        bot_data=bot_left_data,
-        bot_styles=bot_left_styles,
-        show=True, top_portion=0.2,
-        save_path=SAVE_PATH
-    )
-
diff --git a/experiments/iSAGE_paper/reoccurring_concept_drift.py b/experiments/iSAGE_paper/reoccurring_concept_drift.py
deleted file mode 100644
index 432fb67..0000000
--- a/experiments/iSAGE_paper/reoccurring_concept_drift.py
+++ /dev/null
@@ -1,315 +0,0 @@
-import pandas as pd
-
-from experiments.setup.data import get_dataset
-from experiments.setup.explainer import get_imputer_and_storage
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.explainer.sage import IntervalSageExplainer, IncrementalSageExplainer
-from increment_explain.utils.tracker.exponential_smoothing import ExponentialSmoothingTracker
-
-DEBUG = True
-
-DATASET_1_NAME = 'agrawal 1'
-DATASET_2_NAME = 'agrawal 2'
-DATASET_3_NAME = 'agrawal 1'
-DATASET_4_NAME = 'agrawal 2'
-DATASET_RANDOM_SEED = 1
-SHUFFLE_DATA = False
-N_SAMPLES = 25000
-
-MODEL_NAME = 'ARF'
-
-N_INNER_SAMPLES: int = 4
-N_INTERVAL_LENGTH: int = 5000
-CONFIDENCE_BOUNDS_DELTA: float = 0.1
-SMOOTHING_ALPHA: float = 0.001
-FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
-RESERVOIR_LENGTH: int = 100
-RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
-
-TRAINING_METRIC_WINDOW = 500
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 15,
-    }
-}
-
-
-if __name__ == "__main__":
-
-    # Get Data ---------------------------------------------------------------------------------------------------------
-    dataset = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=10)
-    dataset_1 = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
-    dataset_2 = get_dataset(
-        dataset_name=DATASET_2_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
-    dataset_3 = get_dataset(
-        dataset_name=DATASET_3_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
-    dataset_4 = get_dataset(
-        dataset_name=DATASET_4_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES)
-    feature_names = dataset.feature_names
-    cat_feature_names = dataset.cat_feature_names
-    num_feature_names = dataset.num_feature_names
-    stream_0 = dataset
-    n_samples = N_SAMPLES * 4
-    stream_1 = dataset_1.stream
-    stream_2 = dataset_2.stream
-    stream_3 = dataset_3.stream
-    stream_4 = dataset_4.stream
-
-    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
-    task = dataset_1.task
-    loss_function = get_loss_function(task=task)
-    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-    model, model_function = get_model(
-        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-    # Get imputer and explainers ---------------------------------------------------------------------------------------
-    imputer, storage = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
-        reservoir_kind=RESERVOIR_KIND,
-        reservoir_length=RESERVOIR_LENGTH,
-        cat_feature_names=cat_feature_names,
-        num_feature_names=num_feature_names
-    )
-
-    incremental_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    incremental_welford_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        dynamic_setting=False,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    interval_explainer = IntervalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        feature_names=feature_names,
-        n_inner_samples=N_INNER_SAMPLES,
-        interval_length=N_INTERVAL_LENGTH
-    )
-
-    # warm-up because of errors ----------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream_0, start=1):
-        model.learn_one(x_i, y_i)
-
-
-    time_sage = 0.
-    time_interval = 0.
-    time_learning = 0.
-
-    sage_fi_values = []
-    sage_welford_fi_values = []
-    interval_fi_values = []
-
-    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    model_loss = []
-    marginal_loss = []
-
-    for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
-        sage_welford_fi_values.append(fi_values)
-
-        # sage inc
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-        sage_fi_values.append(fi_values)
-
-        # sage int
-        fi_values = interval_explainer.explain_one(x_i, y_i)
-        interval_fi_values.append(fi_values)
-
-        # learning
-        model.learn_one(x_i, y_i)
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
-                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
-                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
-                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
-                  )
-
-        if n >= N_SAMPLES:
-            break
-
-    for (n, (x_i, y_i)) in enumerate(stream_2, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
-        sage_welford_fi_values.append(fi_values)
-
-        # sage inc
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-        sage_fi_values.append(fi_values)
-
-        # sage int
-        fi_values = interval_explainer.explain_one(x_i, y_i)
-        interval_fi_values.append(fi_values)
-
-        # learning
-        model.learn_one(x_i, y_i)
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
-                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
-                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
-                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
-                  )
-
-        if n >= N_SAMPLES:
-            break
-
-    for (n, (x_i, y_i)) in enumerate(stream_3, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
-        sage_welford_fi_values.append(fi_values)
-
-        # sage inc
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-        sage_fi_values.append(fi_values)
-
-        # sage int
-        fi_values = interval_explainer.explain_one(x_i, y_i)
-        interval_fi_values.append(fi_values)
-
-        # learning
-        model.learn_one(x_i, y_i)
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
-                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
-                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
-                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
-                  )
-
-        if n >= N_SAMPLES:
-            break
-
-    for (n, (x_i, y_i)) in enumerate(stream_4, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
-        sage_welford_fi_values.append(fi_values)
-
-        # sage inc
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-        sage_fi_values.append(fi_values)
-
-        # sage int
-        fi_values = interval_explainer.explain_one(x_i, y_i)
-        interval_fi_values.append(fi_values)
-
-        # learning
-        model.learn_one(x_i, y_i)
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
-                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
-                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
-                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
-                  )
-
-        if n >= N_SAMPLES:
-            break
-
-    # Store the results in a database ----------------------------------------------------------------------------------
-
-    facet = "left.csv"
-    data_folder = "reoccurring_drift"
-
-    name = "inc_fi_values"
-    df = pd.DataFrame(sage_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "inc_welford_fi_values"
-    df = pd.DataFrame(sage_welford_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "int_fi_values"
-    df = pd.DataFrame(interval_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "model_loss"
-    df = pd.DataFrame(model_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "marginal_loss"
-    df = pd.DataFrame(marginal_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
\ No newline at end of file
diff --git a/experiments/iSAGE_paper/sampling_procedures_check.py b/experiments/iSAGE_paper/sampling_procedures_check.py
deleted file mode 100644
index ca9f6b5..0000000
--- a/experiments/iSAGE_paper/sampling_procedures_check.py
+++ /dev/null
@@ -1,259 +0,0 @@
-import random
-
-import numpy as np
-import pandas as pd
-import river
-from matplotlib import pyplot as plt
-
-from data.stream.synth import Agrawal
-from experiments.setup.loss import get_loss_function
-from increment_explain.imputer import MarginalImputer
-from increment_explain.imputer.tree_imputer import TreeImputer
-from increment_explain.storage import GeometricReservoirStorage
-from increment_explain.storage.tree_storage import TreeStorage
-from river.datasets.base import REG
-
-from scipy.stats import multivariate_normal as mvn
-
-
-def model_function_placeholder(x_i):
-    return 1.0
-
-
-def get_covariance_stream(size):
-
-    mvn.rvs(mean=[0., 0.], cov=[[1, 0.8], [0.8, 1]], size=5000)
-
-    cov = np.array([[1,0.8,0,0],[0.8,1,0,0],[0,0,1,0],[0,0,0,1]])
-    x_data = mvn.rvs(mean=[0.,0.,0.,0.], cov=cov, size=size)
-    feature_names = ['_'.join(('N', str(i))) for i in range(1, 5)]
-    x_data = pd.DataFrame(x_data, columns=feature_names)
-    x_data['C_1'] = np.array([random.randint(1, 5) for _ in range(size)])
-    y_data = pd.DataFrame(np.array([1 for _ in range(size)]))
-    stream = river.stream.iter_pandas(x_data, y_data, feature_names=feature_names)
-    return stream, feature_names
-
-
-if __name__ == "__main__":
-
-    RANDOM_SEED = 1
-
-    # Model and training setup
-    model_function = model_function_placeholder
-    loss_function = get_loss_function(task=REG)
-
-    sample_from_leaf_reservoirs = True  # let the model predict (False) or sample from leaf reservoirs (True)
-
-    n_samples_agrawal = 5000
-    n_samples_covariance_test = 5000
-
-    # agrawal Test -----------------------------------------------------------------------------------------------------
-    dataset_1 = Agrawal(classification_function=0, random_seed=RANDOM_SEED, n_samples=n_samples_agrawal)
-    n_samples = n_samples_agrawal
-    stream_1 = dataset_1.stream
-    feature_names = dataset_1.feature_names
-    cat_feature_names = dataset_1.cat_feature_names
-    num_feature_names = dataset_1.num_feature_names
-
-    # init imputer and storage
-    storage_tree = TreeStorage(cat_feature_names=cat_feature_names, num_feature_names=num_feature_names)
-    imputer_cond = TreeImputer(model_function, storage_object=storage_tree, direct_predict_numeric=True, use_storage=True)
-    storage_reservoir = GeometricReservoirStorage(store_targets=False, size=100, constant_probability=1.0)
-    imputer_marg = MarginalImputer(model_function, storage_object=storage_reservoir, sampling_strategy='joint')
-
-    x_data_agrawal = []
-
-    commission_sampled_cond = []
-    commission_sampled_marg = []
-
-    age_sampled_cond = []
-    age_sampled_marg = []
-
-    performances_agrawal = []
-
-    for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
-        storage_tree.update(x_i, y_i)
-        storage_reservoir.update(x_i, y_i)
-
-        x_data_agrawal.append(x_i)
-
-        if sample_from_leaf_reservoirs:
-            commission_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='commission', x_i=x_i))
-            age_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='age', x_i=x_i))
-        else:
-            commission_sampled_cond.append(imputer_cond._sample(feature_name='commission', x_i=x_i))
-            age_sampled_cond.append(imputer_cond._sample(feature_name='age', x_i=x_i))
-
-        commission_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['commission'])['commission'])
-        age_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['age'])['age'])
-
-        performance = {}
-        for feature_name, model in storage_tree._storage_x.items():
-            performance[feature_name] = storage_tree.performances[feature_name].get()
-        performances_agrawal.append(performance)
-
-        if n >= n_samples:
-            break
-
-    # Covariance Test --------------------------------------------------------------------------------------------------
-    stream_1, feature_names = get_covariance_stream(size=n_samples_covariance_test)
-    n_samples = n_samples_covariance_test
-    feature_names = feature_names
-    cat_feature_names = ['C_1']
-    num_feature_names = ['N_1', 'N_2', 'N_3', 'N_4']
-
-    # init imputer
-    storage_tree = TreeStorage(cat_feature_names=cat_feature_names, num_feature_names=num_feature_names)
-    imputer_cond = TreeImputer(model_function, storage_object=storage_tree, direct_predict_numeric=True, use_storage=True)
-    storage_reservoir = GeometricReservoirStorage(store_targets=False, size=100, constant_probability=1.0)
-    imputer_marg = MarginalImputer(model_function, storage_object=storage_reservoir, sampling_strategy='joint')
-
-    x_data_covariance_test = []
-
-    C_1_sampled_cond = []
-    C_1_sampled_marg = []
-
-    N_1_sampled_cond = []
-    N_1_sampled_marg = []
-
-    performances_covariance_test = []
-
-    # Concept 1 ----------------------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
-        storage_tree.update(x_i, y_i)
-        storage_reservoir.update(x_i, y_i)
-
-        x_data_covariance_test.append(x_i)
-
-        if sample_from_leaf_reservoirs:
-            C_1_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='C_1', x_i=x_i))
-            N_1_sampled_cond.append(imputer_cond._sample_from_storages(feature_name='N_1', x_i=x_i))
-        else:
-            C_1_sampled_cond.append(imputer_cond._sample(feature_name='C_1', x_i=x_i))
-            N_1_sampled_cond.append(imputer_cond._sample(feature_name='N_1', x_i=x_i))
-
-        C_1_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['C_1'])['C_1'])
-        N_1_sampled_marg.append(imputer_marg._sample(storage_reservoir, feature_subset=['N_1'])['N_1'])
-
-        performance = {}
-        for feature_name, model in storage_tree._storage_x.items():
-            performance[feature_name] = storage_tree.performances[feature_name].get()
-        performances_covariance_test.append(performance)
-
-        if n >= n_samples:
-            break
-
-    # plotting ---------------------------------------------------------------------------------------------------------
-    fig_size_vertical = 3
-    fig_size_horizontal = 9
-
-    # plot agrawal -----------------------------------------------------------------------------------------------------
-    markevery_step = 30
-    markevery_ground_truth_step = 15
-
-    alpha_hue = np.asarray([i for i in range(1, n_samples + 1)]) / n_samples
-    alpha_hue = alpha_hue[::markevery_step]
-
-    performances_agrawal = pd.DataFrame(performances_agrawal)
-    x_data_agrawal = pd.DataFrame(x_data_agrawal)
-
-    fig, axis = plt.subplots(figsize=(fig_size_horizontal*2/3, fig_size_vertical), nrows=1, ncols=2, sharey="all")
-    axis_1 = axis[0]
-    axis_2 = axis[1]
-
-    axis_1.scatter(
-        x_data_agrawal['commission'].loc[::markevery_ground_truth_step],
-        x_data_agrawal['salary'].loc[::markevery_ground_truth_step],
-        marker="o", c="black", alpha=alpha_hue
-    )
-    axis_1.scatter(
-        commission_sampled_marg[::markevery_step],
-        x_data_agrawal['salary'].loc[::markevery_step],
-        marker="+", c="blue", alpha=alpha_hue
-    )
-    axis_1.scatter(
-        commission_sampled_cond[::markevery_step],
-        x_data_agrawal['salary'].loc[::markevery_step],
-        marker="x", c="red", alpha=alpha_hue
-    )
-    axis_1.axis(xmin=-5_000, xmax=90_000, ymin=15_000, ymax=155_000)
-    axis_1.set_xlabel(r"$X_{commission}$")
-    axis_1.set_ylabel(r"$X_{salary}$")
-    axis_1.set_yticks([25_000, 50_000, 75_000, 100_000, 125_000, 150_000])
-    axis_1.set_yticklabels(["25k", "50k", "75k", "100k", "125k", "150k"])
-    axis_1.set_xticks([0, 20_000, 40_000, 60_000, 80_000])
-    axis_1.set_xticklabels(["0", "20k", "40k", "60k", "80k"])
-    axis_1.set_title(r"dependent")
-
-    axis_2.scatter(
-        x_data_agrawal['age'].loc[::markevery_ground_truth_step],
-        x_data_agrawal['salary'].loc[::markevery_ground_truth_step],
-        marker="o", c="black", alpha=alpha_hue
-    )
-    axis_2.scatter(
-        age_sampled_marg[::markevery_step],
-        x_data_agrawal['salary'].loc[::markevery_step],
-        marker="+", c="blue", alpha=alpha_hue
-    )
-    axis_2.scatter(
-        age_sampled_cond[::markevery_step],
-        x_data_agrawal['salary'].loc[::markevery_step],
-        marker="x", c="red", alpha=alpha_hue
-    )
-    axis_2.axis(xmin=5, xmax=85)
-    axis_2.set_xlabel(r"$X_{age}$")
-    axis_2.set_xlim([15, 85])
-    axis_2.set_xticks([20, 30, 40, 50, 60, 70, 80])
-    axis_2.set_title(r"independent")
-
-    plt.tight_layout()
-    plt.subplots_adjust(wspace=0, hspace=0, left=0.12, right=0.995)
-    #plt.savefig("agrawal_conditonal_vs_marginal.png", dpi=200)
-    plt.show()
-
-    # plot covariance test ---------------------------------------------------------------------------------------------
-    performances_covariance_test = pd.DataFrame(performances_covariance_test)
-    x_data_covariance_test = pd.DataFrame(x_data_covariance_test)
-
-    markevery_step = 30
-    markevery_ground_truth_step = 15
-
-    alpha_hue = np.asarray([i for i in range(1, n_samples + 1)]) / n_samples
-    alpha_hue = alpha_hue[::markevery_step]
-
-    fig_2, axes_2 = plt.subplots(figsize=(fig_size_horizontal*1/3, fig_size_vertical), nrows=1, ncols=1)
-    axis_3 = axes_2
-
-    axis_3.scatter(
-        x_data_covariance_test['N_1'].loc[::markevery_ground_truth_step],
-        x_data_covariance_test['N_2'].loc[::markevery_ground_truth_step],
-        marker="o", c="black", alpha=alpha_hue
-    )
-    axis_3.scatter(
-        N_1_sampled_marg[::markevery_step],
-        x_data_covariance_test['N_2'].loc[::markevery_step],
-        marker="+", c="blue", alpha=alpha_hue
-    )
-    axis_3.scatter(
-        N_1_sampled_cond[::markevery_step],
-        x_data_covariance_test['N_2'].loc[::markevery_step],
-        marker="x", c="red", alpha=alpha_hue
-    )
-    axis_3.axis(xmin=-3.8, xmax=3.8, ymin=-4.8, ymax=3.8)
-    axis_3.set_xlabel(r"$X_{2}$")
-    axis_3.set_ylabel(r"$X_{1}$")
-    axis_3.set_xticks([-3, -2, -1, 0, 1, 2, 3])
-    axis_3.set_yticks([-3, -2, -1, 0, 1, 2, 3])
-
-    axis_3.scatter([], [], marker="o", c="black", label="data")
-    axis_3.scatter([], [], marker="+", c="blue", label="marginal")
-    axis_3.scatter([], [], marker="x", c="red", label="conditional")
-    axis_3.legend(loc="lower center", fontsize="small", frameon=True, ncol=3, columnspacing=0.5, handletextpad=0.1)
-
-    plt.title(r"$\rho_{X_{1}, X_{2}} = 0.8$")
-
-    plt.tight_layout()
-    plt.subplots_adjust(wspace=0, hspace=0, left=0.18, right=0.99)
-
-    #plt.savefig("covariance_conditonal_vs_marginal.png", dpi=200)
-    plt.show()
diff --git a/experiments/iSAGE_paper/static_sage_in_dynamic_setting.py b/experiments/iSAGE_paper/static_sage_in_dynamic_setting.py
deleted file mode 100644
index 0d9fd7e..0000000
--- a/experiments/iSAGE_paper/static_sage_in_dynamic_setting.py
+++ /dev/null
@@ -1,232 +0,0 @@
-import time
-
-import pandas as pd
-
-from experiments.setup.data import get_dataset, get_concept_drift_dataset
-from experiments.setup.explainer import get_imputer_and_storage
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.explainer.sage import IntervalSageExplainer, IncrementalSageExplainer
-from increment_explain.utils.tracker.exponential_smoothing import ExponentialSmoothingTracker
-from increment_explain.visualization import FeatureImportancePlotter
-
-DEBUG = True
-
-DATASET_1_NAME = 'agrawal 2'
-DATASET_2_NAME = 'agrawal 1'
-DATASET_RANDOM_SEED = 1
-SHUFFLE_DATA = False
-N_SAMPLES_1 = 20000
-N_SAMPLES_2 = 20000
-
-CONCEPT_DRIFT_POSITION = 20000
-CONCEPT_DRIFT_WIDTH = 1
-CONCEPT_DRIFT_SWITCHING_FEATURES = None # {'salary': 'age'}
-
-MODEL_NAME = 'ARF'
-
-N_INNER_SAMPLES: int = 5
-N_INTERVAL_LENGTH: int = 5000
-CONFIDENCE_BOUNDS_DELTA: float = 0.1
-SMOOTHING_ALPHA: float = 0.001
-FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
-RESERVOIR_LENGTH: int = 100
-RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
-
-TRAINING_METRIC_WINDOW = 500
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 25,
-    }
-}
-
-
-if __name__ == "__main__":
-
-    # Get Data ---------------------------------------------------------------------------------------------------------
-    dataset_1 = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
-    dataset_2 = get_dataset(
-        dataset_name=DATASET_2_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_2)
-    dataset = get_concept_drift_dataset(
-        dataset_1_name=DATASET_1_NAME, dataset_2_name=DATASET_2_NAME,
-        dataset_1=dataset_1, dataset_2=dataset_2,
-        position=CONCEPT_DRIFT_POSITION, width=CONCEPT_DRIFT_WIDTH,
-        features_to_switch=CONCEPT_DRIFT_SWITCHING_FEATURES,
-    )
-    feature_names = dataset.feature_names
-    cat_feature_names = dataset.cat_feature_names
-    num_feature_names = dataset.num_feature_names
-    n_samples = dataset.n_samples
-    stream = dataset.stream
-
-    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
-    task = dataset.task
-    loss_function = get_loss_function(task=task)
-    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-    model, model_function = get_model(
-        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-    # Get imputer and explainers ---------------------------------------------------------------------------------------
-    imputer, storage = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
-        reservoir_kind=RESERVOIR_KIND,
-        reservoir_length=RESERVOIR_LENGTH,
-        cat_feature_names=cat_feature_names,
-        num_feature_names=num_feature_names
-    )
-
-    incremental_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    incremental_welford_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        dynamic_setting=False,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    interval_explainer = IntervalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        feature_names=feature_names,
-        n_inner_samples=N_INNER_SAMPLES,
-        interval_length=N_INTERVAL_LENGTH
-    )
-
-    # warm-up because of errors ----------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        model.learn_one(x_i, y_i)
-        if n > 10:
-            break
-
-    plotter = FeatureImportancePlotter(
-        feature_names=feature_names
-    )
-
-    time_sage = 0.
-    time_interval = 0.
-    time_learning = 0.
-
-    sage_fi_values = []
-    sage_welford_fi_values = []
-    interval_fi_values = []
-
-    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    model_loss = []
-    marginal_loss = []
-
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        start_time = time.time()
-        fi_values = incremental_welford_sage.explain_one(x_i, y_i)
-        time_sage += time.time() - start_time
-        sage_welford_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='inc-w')
-
-        # sage inc
-        start_time = time.time()
-        fi_values = incremental_sage.explain_one(x_i, y_i)
-        time_sage += time.time() - start_time
-        sage_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='inc')
-
-        # sage int
-        start_time = time.time()
-        fi_values = interval_explainer.explain_one(x_i, y_i)
-        time_interval += time.time() - start_time
-        interval_fi_values.append(fi_values)
-        plotter.update(fi_values, facet_name='int')
-
-        # learning
-        start_time = time.time()
-        model.learn_one(x_i, y_i)
-        time_learning += time.time() - start_time
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: marginal-prediction {incremental_sage._marginal_prediction_tracker.get()}\n"
-                  f"{n}: model-loss          {model_loss_tracker.get()}\n"
-                  f"{n}: marginal-loss       {marginal_loss_tracker.get()}\n"
-                  f"{n}: diff                {marginal_loss_tracker.get() - model_loss_tracker.get()}\n"
-                  f"{n}: sum-sage            {sum(list(incremental_sage.importance_values.values()))}\n"
-                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-bnd        {incremental_sage.get_confidence_bound(delta=CONFIDENCE_BOUNDS_DELTA)}\n"
-                  f"{n}: inc-sage-var        {incremental_sage.variances}\n\n"
-                  f"{n}: int-sage            {interval_explainer.importance_values}\n"
-                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
-                  )
-
-        if n >= n_samples:
-            break
-
-    # Store the results in a database ----------------------------------------------------------------------------------
-
-    facet = "right.csv"
-    data_folder = "incremental_setting_5"
-
-    name = "inc_fi_values"
-    df = pd.DataFrame(sage_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "inc_welford_fi_values"
-    df = pd.DataFrame(sage_welford_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "int_fi_values"
-    df = pd.DataFrame(interval_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "model_loss"
-    df = pd.DataFrame(model_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "marginal_loss"
-    df = pd.DataFrame(marginal_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
-                      "line_names": ["loss"], "names_to_highlight": ['loss'],
-                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-                      "markevery": {"model_loss": 100, "marginal_loss": 100},
-                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
-                                 "legend_items": [("marginal loss", "dashed", "red"),
-                                                  ("model loss", "solid", "red")]}}
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
-        performance_kw=performance_kw,
-        title=f'Agrawal Stream with Concept Drift',
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
-        v_lines=[{'x': CONCEPT_DRIFT_POSITION, 'ls': '--', 'c': 'black', 'linewidth': 1}],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'inc': '-', 'int': '--', 'inc-w': 'dotted'},
-        legend_style={"fontsize": "small"},
-        markevery={'inc': 10, 'int': 1, 'inc-w': 10},
-        y_min=-0.05,
-        y_max=0.42,
-    )
diff --git a/experiments/network_ideas/link_prediction.py b/experiments/network_ideas/link_prediction.py
deleted file mode 100644
index da2b81b..0000000
--- a/experiments/network_ideas/link_prediction.py
+++ /dev/null
@@ -1,346 +0,0 @@
-import math
-import random
-
-import river.datasets.synth.concept_drift_stream
-from river.metrics import Accuracy
-from river.utils import Rolling
-from river.ensemble import AdaptiveRandomForestClassifier
-from matplotlib import pyplot as plt
-from river.datasets.base import BINARY_CLF
-
-from data.stream._base import BatchStream
-from experiments.setup.explainer import get_imputer_and_storage
-from experiments.setup.loss import get_loss_function
-from increment_explain.explainer import IncrementalPFI
-from increment_explain.explainer.sage import IncrementalSageExplainer
-from increment_explain.utils.converters import RiverToPredictionFunction
-from increment_explain.utils.tracker.exponential_smoothing import ExponentialSmoothingTracker
-from increment_explain.visualization import FeatureImportancePlotter
-
-N_NODE_FEATURES = 1
-
-
-def _get_label_function_1_old(node_1, node_2):
-    if node_1['node_1_interest_1'] >= 0.5 and node_2['node_2_interest_1'] >= 0.5:
-        return 1
-    if node_1['node_1_interest_2'] >= 0.5 and node_2['node_2_interest_2'] >= 0.5:
-        return 1
-    return 0
-
-
-def _get_label_function_2_old(node_1, node_2):
-    if node_1['node_1_interest_2'] >= 0.6 and node_2['node_2_interest_2'] >= 0.6:
-        return 1
-    if node_1['node_1_interest_3'] >= 0.5 and node_2['node_2_interest_3'] >= 0.5:
-        return 1
-    return 0
-
-
-def _get_label_function_1(node_1, node_2):
-    alpha = 30
-    beta = 25
-    gamma = 10
-    z_1 = alpha * (node_1['node_1_interest_1'] * node_2['node_2_interest_1'])
-    z_2 = beta * (node_1['node_1_interest_2'] * node_2['node_2_interest_2'])
-    z_3 = gamma * node_1['graph_connectivity']
-    p = 1 / (1 + math.exp(-(z_1 + z_2 + z_3)))
-    random_number = random.random()
-    if random_number < p:
-        return 1, p
-    return 0, p
-
-
-def _get_label_function_2(node_1, node_2):
-    alpha = 30
-    beta = 25
-    gamma = 10
-    z_1 = alpha * (node_1['node_1_interest_2'] * node_2['node_2_interest_2'])
-    z_2 = beta * (node_1['node_1_interest_3'] * node_2['node_2_interest_3'])
-    z_3 = gamma * node_1['graph_connectivity']
-    p = 1 / (1 + math.exp(-(z_1 + z_2 + z_3)))
-    random_number = random.random()
-    #print(int(random_number < p), p, node_1['node_1_interest_1'], node_2['node_2_interest_1'])
-    #print(int(random_number < p), p, node_1['node_1_interest_3'], node_2['node_2_interest_3'])
-    if random_number < p:
-        return 1, p
-    return 0, p
-
-
-def get_sample(classification_function: int = 1, n_interests=3):
-    node_1 = {'node_1_interest_' + str(i): random.uniform(-1, 1) for i in range(1, n_interests + 1)}
-    node_1['node_1_att_1'] = random.randint(1, 3)
-    node_1['graph_connectivity'] = random.uniform(-1, 1)
-    node_1['node_1_att_2'] = random.randint(1, 3)
-    node_2 = {'node_2_interest_' + str(i): random.uniform(-1, 1) for i in range(1, n_interests + 1)}
-    node_2['node_2_att_1'] = random.randint(1, 3)
-
-    #node_1.update({'interest_1': node_1['node_1_interest_1'] * node_2['node_2_interest_1'],'interest_2': node_1['node_1_interest_2'] * node_2['node_2_interest_2'],'interest_3': node_1['node_1_interest_3'] * node_2['node_2_interest_3']})
-    #node_1['interest_1_interaction'] = node_1['node_1_interest_1'] * node_2['node_2_interest_1']
-    #node_1['interest_2_interaction'] = node_1['node_1_interest_2'] * node_2['node_2_interest_2']
-    #node_1['interest_2_interaction'] = node_1['node_1_interest_3'] * node_2['node_2_interest_3']
-
-
-    if classification_function == 1:
-        y_i, p_i = _get_label_function_1(node_1, node_2)
-    elif classification_function == 2:
-        y_i, p_i = _get_label_function_2(node_1, node_2)
-    else:
-        raise NotImplementedError("Only classificaiton function 1 and 2 implemented.")
-
-    x_i = {**node_1, **node_2}
-
-    return x_i, y_i
-
-def print_probas(n=10_000, n_interests=3):
-    ys = []
-    ps = []
-    for i in range(n):
-        node_1 = {'node_1_interest_' + str(i): random.uniform(-0.5, 0.5) for i in range(1, n_interests + 1)}
-        node_1['node_1_att_1'] = random.randint(1, 3)
-        node_2 = {'node_2_interest_' + str(i): random.uniform(-0.5, 0.5) for i in range(1, n_interests + 1)}
-        node_2['node_2_att_1'] = random.randint(1, 3)
-        y_i, p_i = _get_label_function_1(node_1, node_2)
-        ys.append(y_i)
-        ps.append(p_i)
-
-    plt.hist(ys)
-    plt.title("Target")
-    plt.show()
-
-    plt.hist(ps)
-    plt.title("Probability")
-    plt.show()
-
-
-def easy_problem(n=10_000, alpha=5, beta=5):
-    ys = []
-    ps = []
-    xs = []
-    x_1 = []
-    x_2 = []
-    dataset = []
-    alpha = alpha
-    for i in range(n):
-        node_1_interest_1 = random.uniform(-1, 1)
-        node_2_interest_1 = random.uniform(-1, 1)
-        interest_1 = node_1_interest_1 * node_2_interest_1
-        node_1_interest_2 = random.uniform(-1, 1)
-        node_2_interest_2 = random.uniform(-1, 1)
-        interest_2 = node_1_interest_2 * node_2_interest_2
-        p = 1 / (1 + math.exp(-(alpha * interest_1 + beta * interest_2)))
-        xs.append(interest_1)
-        x_1.append(node_1_interest_1)
-        x_2.append(node_2_interest_1)
-        ps.append(p)
-        y = 0
-        if random.random() <= p:
-            y = 1
-        ys.append(y)
-        dataset.append({"node_1_interest_1": node_1_interest_1, "node_2_interest_1": node_2_interest_1, #"interest_1": interest_1,
-                        "node_1_interest_2": node_1_interest_2, "node_2_interest_2": node_2_interest_2, #"interest_2": interest_2
-                        })
-
-    plt.scatter(xs, ys)
-    plt.title(f"Label, alpha: {alpha}")
-    plt.xlabel("Feature 1 * Feature 2")
-    plt.show()
-
-    plt.scatter(xs, ps)
-    plt.title(f"Probability, alpha: {alpha}")
-    plt.xlabel("Feature 1 * Feature 2")
-    plt.show()
-
-    plt.scatter(x_1, ys)
-    plt.title(f"Label, alpha: {alpha}")
-    plt.xlabel("Feature 1")
-    plt.show()
-
-    plt.scatter(x_1, ps)
-    plt.title(f"Probability, alpha: {alpha}")
-    plt.xlabel("Feature 1")
-    plt.show()
-
-    return dataset, ys
-
-
-
-if __name__ == "__main__":
-
-    #print_probas(n=10_000)
-
-    N_SAMPLES = 50_000
-
-    #dataset, ys = easy_problem(n=N_SAMPLES, alpha=30, beta=25)
-    #stream = zip(dataset, ys)
-    #feature_names = list(dataset[0].keys())
-
-    DRIFT_POSITION = int(N_SAMPLES * 0.5)
-    DRIFT_WIDTH = int(N_SAMPLES * 0.2)
-
-    x_i, _ = get_sample(classification_function=1)
-    feature_names = list(x_i.keys())
-
-    model = AdaptiveRandomForestClassifier(n_models=10, binary_split=True, max_depth=10)
-    #model = HoeffdingTreeClassifier(max_depth=10)
-    #model = LogisticRegression()
-
-
-    model_function = RiverToPredictionFunction(
-        model,
-        classification=True,
-        feature_names=feature_names
-    ).predict
-
-    loss_function = get_loss_function(task=BINARY_CLF)
-
-    imputer, storage = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution='marginal joint',
-        reservoir_kind='geometric',
-        reservoir_length=1000,
-        cat_feature_names=[],
-        num_feature_names=feature_names
-    )
-
-    incremental_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=0.001,
-        n_inner_samples=5
-    )
-
-    incremental_pfi = IncrementalPFI(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=0.001,
-        n_samples=5
-    )
-
-    metric = Accuracy()
-    metric = Rolling(metric, window_size=1000)
-
-    plotter = FeatureImportancePlotter(
-        feature_names=feature_names
-    )
-
-    model_loss_tracker = ExponentialSmoothingTracker(alpha=0.001)
-    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=0.001)
-    model_loss = []
-    marginal_loss = []
-
-    performance = []
-
-    stream_1 = iter([get_sample(classification_function=1) for i in range(N_SAMPLES*2)])
-    stream_1 = BatchStream(stream=stream_1, n_features=len(feature_names), task=BINARY_CLF)
-    stream_2 = iter([get_sample(classification_function=2) for i in range(N_SAMPLES*2)])
-    stream_2 = BatchStream(stream=stream_2, n_features=len(feature_names), task=BINARY_CLF)
-    stream = river.datasets.synth.concept_drift_stream.ConceptDriftStream(
-        stream=stream_1, drift_stream=stream_2, position=DRIFT_POSITION, width=DRIFT_WIDTH)
-
-    y_all = []
-
-    for n, (x_i, y_i) in enumerate(stream):
-        y_all.append(y_i)
-
-        y_pred = model.predict_one(x_i)
-        model.learn_one(x_i, y_i)
-
-
-        if n > 100:
-            y_i_pred = model_function(x_i)
-            loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-            model_loss_tracker.update(loss_i)
-            model_loss.append({"loss": model_loss_tracker.get()})
-            loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-            marginal_loss_tracker.update(loss_marginal_i)
-            marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-            fi_values = incremental_sage.explain_one(x_i, y_i)
-            plotter.update(fi_values, facet_name='sage')
-
-            fi_values = incremental_pfi.explain_one(x_i, y_i)
-            plotter.update(fi_values, facet_name='pfi')
-
-        metric.update(y_i, y_pred)
-        performance.append({'performance': metric.get()})
-
-        if n % 1000 == 0:
-            print(f"{n} performance {metric.get()}")
-            print(f"{n} iSAGE       {incremental_sage.importance_values}")
-            print(f"{n} iPFI        {incremental_pfi.importance_values}")
-
-        if n >= N_SAMPLES:
-            break
-
-    plt.hist(y_all)
-    plt.show()
-
-    performance_kw_sage = {
-        "y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
-        "line_names": ["loss"], "names_to_highlight": ['loss'],
-        "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-        "markevery": {"model_loss": 100, "marginal_loss": 100},
-        "secondary_legends": [
-            {"legend_props": {"loc": "center right", "ncol": 1, "fontsize": "small"},
-             "legend_items": [("marginal loss", "dashed", "red"), ("model loss", "solid", "red")]}],
-        "fill_between_props": [{'facet_1': 'model_loss', 'facet_2': 'marginal_loss', 'line_name_1': 'loss',
-                                'line_name_2': 'loss', 'hatch': '///', 'color': 'red', 'alpha': 0.1}]
-    }
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
-        performance_kw=performance_kw_sage,
-        title=f'Online Link Prediction based on common Interests',
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['node_1_interest_1', 'node_2_interest_1', 'node_1_interest_2',
-                            'node_2_interest_2', 'node_1_interest_3', 'node_2_interest_3',
-                            'graph_connectivity'
-                            ],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'sage': '-', 'pfi': ''},
-        legend_style={"fontsize": "small", "ncol": 4, "loc": "upper center"},
-        markevery={'sage': 100, 'pfi': 100},
-        y_ticks=[0, 0.02, 0.04, 0.06, 0.08, 0.1, 0.12],
-        y_min=-0.02,
-        y_max=0.14,
-        color_list=['#ef27a6', '#4ea5d9', '#7d53de', '#44001A', '#ef27a6', '#4ea5d9', '#7d53de', '#44cfcb', '#44001A'],
-        save_name="sage_probabilistic_edge_gradual_long.png"
-    )
-
-    # PFI --------------------------------------------------------------------------------------------------------------
-
-    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "accuracy", "color_list": ["red"],
-                      "line_names": ["performance"], "names_to_highlight": ['performance']}
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"accuracy": performance},
-        performance_kw=performance_kw,
-        title=f'Online Link Prediction based on common Interests',
-        y_label='PFI values',
-        x_label='Samples',
-        names_to_highlight=['node_1_interest_1', 'node_2_interest_1', 'node_1_interest_2',
-                            'node_2_interest_2', 'node_1_interest_3', 'node_2_interest_3',
-                            'graph_connectivity'
-                            ],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'sage': '', 'pfi': '-'},
-        legend_style={"fontsize": "small", "ncol": 4, "loc": "upper center"},
-        markevery={'sage': 100, 'pfi': 100},
-        y_ticks=[0, 0.1, 0.2],
-        y_min=-0.04,
-        y_max=0.24,
-        color_list=['#ef27a6', '#4ea5d9', '#7d53de', '#44001A', '#ef27a6', '#4ea5d9', '#7d53de', '#44cfcb', '#44001A'],
-        save_name="pfi_probabilistic_edge_gradual_long.png"
-    )
-
-
-
diff --git a/experiments/pfi/agrawal.py b/experiments/pfi/agrawal.py
deleted file mode 100644
index f2fe401..0000000
--- a/experiments/pfi/agrawal.py
+++ /dev/null
@@ -1,130 +0,0 @@
-import numpy as np
-
-from river.ensemble import AdaptiveRandomForestClassifier
-from river import metrics
-from river.utils import Rolling
-
-from data.stream.synth import Agrawal
-
-from increment_explain.imputer import MarginalImputer
-from increment_explain.storage import UniformReservoirStorage
-from increment_explain.utils.converters import RiverToPredictionFunction
-from increment_explain.explainer.pfi import IncrementalPFI
-from increment_explain.visualization import FeatureImportancePlotter
-
-from sage.utils import CrossEntropyLoss
-
-
-def cross_entropy_loss(y_prediction, y_true):
-    y_prediction = np.asarray([y_prediction])
-    y_prediction = y_prediction.reshape((len(y_prediction), 1))
-    y_true = np.asarray([y_true])
-    y_true = y_true.reshape((len(y_true))).astype(int)
-    return CROSS_ENTROPY(pred=y_prediction, target=y_true)
-
-
-if __name__ == "__main__":
-    CROSS_ENTROPY = CrossEntropyLoss(reduction='mean')
-    RANDOM_SEED = 1
-    CLASSIFICATION_FUNCTIONS = (1, 2)
-
-    STREAM_LENGTH = 10000
-    STREAM_POSITION = int(STREAM_LENGTH * 0.5)
-    N_STREAM_1 = STREAM_POSITION
-    N_STREAM_2 = STREAM_LENGTH - N_STREAM_1
-
-    EXPLAINER_NAME = "inc-PFI"
-
-    SMOOTHING_ALPHA = 0.001
-
-    for i in range(1):
-
-        # Setup Data ---------------------------------------------------------------------------------------------------
-        dataset_1 = Agrawal(classification_function=CLASSIFICATION_FUNCTIONS[0], random_seed=RANDOM_SEED, n_samples=N_STREAM_1)
-        dataset_2 = Agrawal(classification_function=CLASSIFICATION_FUNCTIONS[1], random_seed=RANDOM_SEED, n_samples=N_STREAM_1)
-        stream_1 = dataset_1.stream
-        stream_2 = dataset_2.stream
-
-        feature_names = list(dataset_1.feature_names)
-
-        # Model and training setup
-        model = AdaptiveRandomForestClassifier(seed=RANDOM_SEED)
-        model_function = RiverToPredictionFunction(model, feature_names).predict
-        metric = metrics.Accuracy()
-        metric = Rolling(metric, window_size=500)
-
-        loss_function = cross_entropy_loss
-
-        plotter = FeatureImportancePlotter(
-            feature_names=feature_names
-        )
-
-        model_performance = []
-
-        # Setup explainers ---------------------------------------------------------------------------------------------
-
-        # Instantiating objects
-        storage = UniformReservoirStorage(store_targets=False, size=100)
-        # storage = SequenceStorage(store_targets=True)
-        imputer = MarginalImputer(model_function, 'product', storage)
-        # imputer = DefaultImputer(model, values={'N_1': 5, 'N_2': 3, 'N_10': 2})
-        incremental_explainer = IncrementalPFI(
-            model_function=model_function,
-            feature_names=feature_names,
-            storage=storage,
-            imputer=imputer,
-            n_samples=2,
-            loss_function=loss_function,
-            smoothing_alpha=SMOOTHING_ALPHA,
-            dynamic_setting=True
-        )
-
-        # warm-up because of errors ------------------------------------------------------------------------------------
-        for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
-            model.learn_one(x_i, y_i)
-            if n > 10:
-                break
-
-        # Concept 1 ----------------------------------------------------------------------------------------------------
-
-        for (n, (x_i, y_i)) in enumerate(stream_1, start=1):
-            y_i_pred = model.predict_one(x_i)
-            metric.update(y_true=y_i, y_pred=y_i_pred)
-            model_performance.append({'accuracy': metric.get()})
-            inc_fi_values = incremental_explainer.explain_one(x_i, y_i)
-            plotter.update(importance_values=inc_fi_values, facet_name='inc')
-            model.learn_one(x_i, y_i)
-            if n % 1000 == 0:
-                print(f"{n}: performance {metric.get()}\n"
-                      f"{n}: {EXPLAINER_NAME} {incremental_explainer.importance_values}")
-            if n >= N_STREAM_1:
-                break
-
-        # Concept 2 ----------------------------------------------------------------------------------------------------
-
-        for (n, (x_i, y_i)) in enumerate(stream_2, start=1):
-            y_i_pred = model.predict_one(x_i)
-            metric.update(y_true=y_i, y_pred=y_i_pred)
-            model_performance.append({'accuracy': metric.get()})
-            inc_fi_values = incremental_explainer.explain_one(x_i, y_i)
-            plotter.update(importance_values=inc_fi_values, facet_name='inc')
-            model.learn_one(x_i, y_i)
-            if n % 1000 == 0:
-                print(f"{n}: performance {metric.get()}\n"
-                      f"{n}: {EXPLAINER_NAME} {incremental_explainer.importance_values}")
-            if n >= N_STREAM_2:
-                break
-
-        # Plotting -----------------------------------------------------------------------------------------------------
-        plotter.plot(
-            figsize=(5, 10),
-            model_performance={'accuracy': model_performance},
-            title=f'Agrawal Stream (fn. {CLASSIFICATION_FUNCTIONS[0]} to fn. {CLASSIFICATION_FUNCTIONS[1]})',
-            y_label='SAGE values',
-            x_label='Samples',
-            names_to_highlight=['salary', 'commission', 'age', 'elevel'],
-            v_lines=[{'x': N_STREAM_1, 'ls': '--', 'c': 'black', 'linewidth': 1}],
-            h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-            line_styles={'inc': '-', 'int': '--'},
-            legend_style={}
-        )
diff --git a/experiments/setup/__init__.py b/experiments/setup/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/experiments/setup/data.py b/experiments/setup/data.py
deleted file mode 100644
index f73f793..0000000
--- a/experiments/setup/data.py
+++ /dev/null
@@ -1,97 +0,0 @@
-from typing import Optional, Union
-
-import data.batch
-import data.stream
-import data.stream.synth
-from data.stream._base import get_concept_drift_stream
-from data.stream._base import StreamDataset
-
-DATASET_NAMES = ['bank', 'adult', 'bike', 'elec2', 'agrawal', 'stagger', 'planes', 'insects', 'credit_card']
-BATCH_DATASETS = ['bank', 'adult', 'bike']
-SYNTH_DATASETS = ['agrawal', 'stagger', 'planes']
-STREAM_DATASETS = ['elec2', 'insects', 'credit_card', "stream_bike"]
-
-__all__ = [
-    "get_dataset",
-    "get_concept_drift_dataset"
-]
-
-
-def _get_batch_dataset(dataset_name, shuffle_dataset=False, random_seed=None, n_samples=None):
-    if dataset_name == 'bank':
-        dataset = data.batch.BankMarketing(random_seed=random_seed, shuffle_dataset=shuffle_dataset, n_samples=n_samples)
-    elif dataset_name == 'adult':
-        dataset = data.batch.Adult(random_seed=random_seed, shuffle_dataset=shuffle_dataset, n_samples=n_samples)
-    elif dataset_name == 'bike':
-        dataset = data.batch.BikeSharing(random_seed=random_seed, shuffle_dataset=shuffle_dataset, n_samples=n_samples)
-    else:
-        raise NotImplementedError(f"The batch dataset with name {dataset_name} is not implemented.")
-    return dataset
-
-
-def get_dataset(dataset_name, shuffle=False, random_seed=None, n_samples=None):
-    dataset_parameter = dataset_name.split(' ')[1] if len(dataset_name.split(' ')) > 1 else None
-    dataset_name = dataset_name.split(' ')[0]
-    if dataset_name in BATCH_DATASETS:
-        dataset = _get_batch_dataset(dataset_name=dataset_name, shuffle_dataset=shuffle, random_seed=random_seed, n_samples=n_samples)
-    elif dataset_name == 'elec2':
-        dataset = data.stream.Elec2(n_samples=n_samples)
-    elif dataset_name == 'stream_bike':
-        dataset = data.stream.Bike(n_samples=n_samples)
-    elif dataset_name == 'agrawal':
-        if dataset_parameter is None:
-            dataset_parameter = 1
-        dataset = data.stream.synth.Agrawal(classification_function=int(dataset_parameter), random_seed=random_seed, n_samples=n_samples)
-    elif dataset_name == 'stagger':
-        if dataset_parameter is None:
-            dataset_parameter = 1
-        dataset = data.stream.synth.Stagger(classification_function=int(dataset_parameter), random_seed=random_seed, n_samples=n_samples)
-    elif dataset_name == 'planes':
-        dataset = data.stream.synth.Planes2D(random_seed=random_seed, n_samples=n_samples)
-    elif dataset_name == 'insects':
-        if dataset_parameter is None:
-            dataset_parameter = "abrupt_balanced"
-        dataset = data.stream.Insects(variant=dataset_parameter, n_samples=n_samples)
-    elif dataset_name == 'credit_card':
-        dataset = data.stream.CreditCard()
-    else:
-        raise NotImplementedError(f"The dataset {dataset_name} is not implemented. Implemented Datasets are {DATASET_NAMES}.")
-    return dataset
-
-
-def get_concept_drift_dataset(dataset_1_name: str, dataset_2_name: str,
-                              dataset_1, dataset_2,
-                              position: Union[float, int] = 0.5,
-                              width: Union[float, int] = 0.05,
-                              features_to_switch: Optional[Union[dict, str]] = None):
-    dataset_name = dataset_1_name.split(' ')[0]
-    stream_1 = dataset_1.stream
-    if dataset_name in BATCH_DATASETS or dataset_name in STREAM_DATASETS:
-        stream_2 = None
-        n_samples = dataset_1.n_samples
-    else:
-        stream_2 = dataset_2.stream
-        n_samples = dataset_1.n_samples + dataset_2.n_samples
-    if position < 1:
-        position = int(position * n_samples)
-    if width < 1:
-        width = int(width * n_samples)
-
-    # create the concept drift stream
-    concept_drift_stream = get_concept_drift_stream(
-        stream=stream_1, drift_stream=stream_2,
-        position=position, width=width, feature_remapping=features_to_switch)
-
-    # create the new dataset that contains the concept drift stream
-    dataset = StreamDataset(
-        stream=concept_drift_stream,
-        n_samples=n_samples,
-        feature_names=dataset_1.feature_names,
-        cat_feature_names=dataset_1.cat_feature_names,
-        num_feature_names=dataset_1.num_feature_names,
-        task=dataset_1.task,
-        n_features=dataset_1.n_features,
-        n_outputs=dataset_1.n_outputs
-    )
-
-    return dataset
diff --git a/experiments/setup/explainer.py b/experiments/setup/explainer.py
deleted file mode 100644
index 101a9cd..0000000
--- a/experiments/setup/explainer.py
+++ /dev/null
@@ -1,147 +0,0 @@
-import typing
-
-from increment_explain.storage import GeometricReservoirStorage, UniformReservoirStorage, TreeStorage
-from increment_explain.imputer import MarginalImputer, TreeImputer
-from increment_explain.explainer.sage import IncrementalSageExplainer, BatchSageExplainer, IntervalSageExplainer
-from increment_explain.explainer.pfi import IncrementalPFI
-
-
-def get_imputer_and_storage(
-        model_function,
-        feature_removal_distribution,
-        reservoir_kind,
-        reservoir_length,
-        cat_feature_names=None,
-        num_feature_names=None
-):
-    if feature_removal_distribution in {'marginal joint', 'marginal product'}:
-        if reservoir_kind == 'geometric':
-            storage = GeometricReservoirStorage(
-                size=reservoir_length,
-                store_targets=False
-            )
-        elif reservoir_kind == 'uniform':
-            storage = UniformReservoirStorage(
-                size=reservoir_length,
-                store_targets=False
-            )
-        else:
-            raise NotImplementedError(f"Only 'geometric', or 'uniform' storage implemented, not {reservoir_kind}.")
-
-        imputer = MarginalImputer(
-            model_function=model_function,
-            storage_object=storage,
-            sampling_strategy=feature_removal_distribution.split(" ")[1]
-        )
-    elif feature_removal_distribution == 'conditional':
-        assert cat_feature_names is not None and num_feature_names is not None, \
-            "cat. and num. feature names must be provided"
-
-        storage = TreeStorage(
-            cat_feature_names=cat_feature_names,
-            num_feature_names=num_feature_names
-        )
-
-        imputer = TreeImputer(
-            model_function=model_function,
-            storage_object=storage,
-            use_storage=True,
-            direct_predict_numeric=True
-        )
-    else:
-        raise NotImplementedError(f"Only 'marginal joint', 'marginal product', and 'conditional' feature removal "
-                                  f"distributions are implemented. Not {feature_removal_distribution}")
-
-    return imputer, storage
-
-
-def get_incremental_sage_explainer(
-        model_function,
-        feature_names,
-        loss_function,
-        distribution_kind: str = 'marginal',
-        reservoir_kind: str = 'geometric',
-        reservoir_length: int = 1000,
-        smoothing_alpha: float = 0.001,
-        n_inner_samples: int = 1,
-        cat_feature_names: typing.Optional[typing.List[str]] = None,
-        num_feature_names: typing.Optional[typing.List[str]] = None
-
-) -> IncrementalSageExplainer:
-
-    imputer, storage = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution=distribution_kind,
-        reservoir_kind=reservoir_kind,
-        reservoir_length=reservoir_length,
-        cat_feature_names=cat_feature_names,
-        num_feature_names=num_feature_names
-    )
-    explainer = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=smoothing_alpha,
-        n_inner_samples=n_inner_samples
-    )
-    return explainer
-
-
-def get_incremental_pfi_explainer(
-        model_function,
-        feature_names,
-        loss_function,
-        reservoir_kind: str = 'geometric',
-        sample_strategy: str = 'joint',
-        reservoir_length: int = 1000,
-        smoothing_alpha: float = 0.001,
-        n_inner_samples: int = 1,
-) -> IncrementalPFI:
-
-    imputer, storage = get_imputer_and_storage(model_function, 'marginal', reservoir_kind, reservoir_length, sample_strategy)
-    explainer = IncrementalPFI(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        feature_names=feature_names,
-        smoothing_alpha=smoothing_alpha,
-        n_samples=n_inner_samples,
-        storage=storage
-    )
-    return explainer
-
-
-def get_batch_sage_explainer(
-        model_function,
-        loss_function,
-        feature_names,
-        n_inner_samples: int = 1
-) -> BatchSageExplainer:
-
-    explainer = BatchSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        feature_names=feature_names,
-        n_inner_samples=n_inner_samples
-    )
-    return explainer
-
-
-def get_interval_sage_explainer(
-        model_function,
-        loss_function,
-        feature_names,
-        n_inner_samples: int = 1,
-        interval_length: int = 100
-) -> IntervalSageExplainer:
-
-    explainer = IntervalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        feature_names=feature_names,
-        n_inner_samples=n_inner_samples,
-        interval_length=interval_length
-    )
-    return explainer
diff --git a/experiments/setup/loss.py b/experiments/setup/loss.py
deleted file mode 100644
index 79e555d..0000000
--- a/experiments/setup/loss.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import numpy as np
-from river.datasets.base import BINARY_CLF, REG, MULTI_CLF
-from sage.utils import MSELoss, CrossEntropyLoss
-from river.metrics import Accuracy, MAE, CrossEntropy
-from river.utils import Rolling
-
-__all__ = [
-    "get_loss_function",
-    "cross_entropy_loss",
-    "mse_loss"
-]
-
-CROSS_ENTROPY = CrossEntropyLoss(reduction='mean')
-MSE = MSELoss(reduction='mean')
-
-
-
-
-def mse_loss(y_prediction, y_true):
-    y_prediction = np.asarray([y_prediction])
-    y_prediction = y_prediction.reshape((len(y_prediction), 1))
-    y_true = np.asarray([y_true])
-    y_true = y_true.reshape((len(y_true))).astype(int)
-    return MSE(pred=y_prediction, target=y_true)
-
-
-def cross_entropy_loss(y_prediction, y_true):
-    y_prediction = np.asarray([y_prediction])
-    y_prediction = y_prediction.reshape((len(y_prediction), 1))
-    y_true = np.asarray([y_true])
-    y_true = y_true.reshape((len(y_true))).astype(int)
-    return CROSS_ENTROPY(pred=y_prediction, target=y_true)
-
-
-def cross_entropy_loss_multi(y_prediction, y_true):
-    y_prediction = np.asarray([y_prediction])
-    y_prediction = y_prediction.reshape((len(y_prediction), 1))
-    y_true = np.asarray([y_true])
-    y_true = y_true.reshape((len(y_true))).astype(int)
-    return CROSS_ENTROPY(pred=y_prediction, target=y_true)
-
-
-def get_loss_function(task):
-    if task == BINARY_CLF:  # TODO make task checks with River Global variables
-        loss_function = cross_entropy_loss
-    elif task == REG:
-        loss_function = mse_loss
-    elif task == MULTI_CLF:
-        loss_function = cross_entropy_loss_multi
-    else:
-        raise NotImplementedError(f"No standard loss implemented for task {task}.")
-    return loss_function
-
-
-def get_training_metric(task, rolling_window=1000):
-    if task == BINARY_CLF:
-        training_metric = CrossEntropy()
-    elif task == REG:
-        training_metric = MAE()
-    elif task == MULTI_CLF:
-        training_metric = CrossEntropy()
-    else:
-        raise NotImplementedError(f"No standard loss implemented for task {task}.")
-    if rolling_window > 0:
-        rolling_training_metric = Rolling(training_metric, rolling_window)
-    else:
-        rolling_training_metric = training_metric
-    return rolling_training_metric, training_metric
diff --git a/experiments/setup/model.py b/experiments/setup/model.py
deleted file mode 100644
index f668211..0000000
--- a/experiments/setup/model.py
+++ /dev/null
@@ -1,24 +0,0 @@
-from river.datasets import base
-from river.ensemble import AdaptiveRandomForestClassifier, AdaptiveRandomForestRegressor
-from increment_explain.utils.converters import RiverToPredictionFunction
-
-
-def _get_arf_model(task, **kwargs):
-    if task == base.BINARY_CLF:
-        model = AdaptiveRandomForestClassifier(**kwargs)
-    else:
-        model = AdaptiveRandomForestRegressor(**kwargs)
-    return model
-
-
-def get_model(model_name, task, feature_names, **model_kw):
-    if model_name == 'ARF':
-        model = _get_arf_model(task, **model_kw)
-    else:
-        raise NotImplementedError
-    model_function = RiverToPredictionFunction(
-        model,
-        classification=task == base.BINARY_CLF,
-        feature_names=feature_names
-    ).predict
-    return model, model_function
diff --git a/experiments/storage/check_reservoir_mechanisms.py b/experiments/storage/check_reservoir_mechanisms.py
deleted file mode 100644
index d5c3cae..0000000
--- a/experiments/storage/check_reservoir_mechanisms.py
+++ /dev/null
@@ -1,39 +0,0 @@
-from increment_explain.storage import UniformReservoirStorage, GeometricReservoirStorage
-import matplotlib.pyplot as plt
-
-
-def dummy_stream(t):
-    return [({'t': i}, i) for i in range(t)]
-
-
-if __name__ == "__main__":
-
-    STORE_TARGETS = False
-    SIZE = 100
-    T = 1000
-
-    reservoir_values = {i: 0 for i in range(T)}
-    stream = dummy_stream(t=T)
-    for _ in range(1000):
-        storage = UniformReservoirStorage(store_targets=STORE_TARGETS, size=SIZE)
-        for x_i, y_i in stream:
-            storage.update(x_i, y_i)
-        for feature in storage.get_data()[0]:
-            reservoir_values[feature['t']] += 1
-
-    plt.bar(reservoir_values.keys(), reservoir_values.values())
-    plt.title(f"Uniform Reservoir: size {SIZE}, T {T}")
-    plt.show()
-
-    reservoir_values = {i: 0 for i in range(T)}
-    stream = dummy_stream(t=T)
-    for _ in range(1000):
-        storage = GeometricReservoirStorage(store_targets=STORE_TARGETS, size=SIZE, constant_probability=1)
-        for x_i, y_i in stream:
-            storage.update(x_i, y_i)
-        for feature in storage.get_data()[0]:
-            reservoir_values[feature['t']] += 1
-
-    plt.bar(reservoir_values.keys(), reservoir_values.values())
-    plt.title(f"Geometric Reservoir: size {SIZE}, T {T}")
-    plt.show()
diff --git a/experiments/visualization/__init__.py b/experiments/visualization/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/experiments/visualization/boxplots.py b/experiments/visualization/boxplots.py
deleted file mode 100644
index c9165a3..0000000
--- a/experiments/visualization/boxplots.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import numpy as np
-import pandas as pd
-from matplotlib import pyplot as plt
-
-# from increment_explain.visualization.color import DEFAULT_COLOR_LIST
-
-DEFAULT_COLOR_LIST = ['#1d4289']
-
-
-def grouped_boxplot(data, feature_names, explainer_names, save_path, legend, width=0.5, min_space=0.1, feature_spacing=0.5, y_min=-1., y_max=1., legend_pos=0):
-    n_explainer = len(explainer_names)
-    n_features = len(feature_names)
-
-    feature_names_short = [name[0:3]+'.' if len(name) > 3 else name[0:3] for name in feature_names]
-
-    group_length = width * n_explainer + min_space * (n_explainer-1)
-    x_range = np.array([x * group_length for x in range(n_features)])
-    x_range = np.array([x_range[i] + feature_spacing * i for i in range(n_features)])
-    x_locations = sorted(
-        [feature_loc + explainer * (width + min_space) for explainer in range(n_explainer) for feature_loc in x_range])
-    feature_x_locations = x_range + group_length / 2 - width / 2
-
-    fig = plt.figure(0, figsize=[9, 4.8])
-    ax = fig.add_subplot(111)
-    x_position_index = 0
-    for feature in feature_names:
-        color_index = 0
-        for explainer in explainer_names:
-            if explainer == 'batch_total' or explainer == 'batch_interval':
-                color = 'black'
-            else:
-                color = DEFAULT_COLOR_LIST[color_index]
-                color_index += 1
-            #ax.boxplot(
-            #    data[explainer][feature], positions=[x_locations[x_position_index]],
-            #    widths=width, patch_artist=True,
-            #    medianprops={'color': 'red', 'linestyle': '-'},boxprops=dict(facecolor=color, color=None)
-            #)
-            violin_parts = ax.violinplot(
-                data[explainer][feature], positions=[x_locations[x_position_index]],
-                widths=width,
-                showmeans=True, showmedians=True
-            )
-            for pc in violin_parts['bodies']:
-                #pc.set_facecolor(color)
-                #pc.set_edgecolor(color)
-                pc.set_color(color)
-            for partname in ('cbars', 'cmins', 'cmaxes', 'cmeans'):
-                vp = violin_parts[partname]
-                vp.set_edgecolor(color)
-                vp.set_linewidth(1)
-            vp = violin_parts['cmedians']
-            vp.set_edgecolor('red')
-            vp.set_linewidth(1)
-
-            x_position_index += 1
-    x_position_index = 0
-    color_index = 0
-    for explainer in explainer_names:
-        if explainer == 'batch_total' or explainer == 'batch_interval':
-            color = 'black'
-        else:
-            color = DEFAULT_COLOR_LIST[color_index]
-            color_index += 1
-        legend_name = legend[x_position_index]
-        ax.plot([], c=color, label=legend_name)
-        x_position_index += 1
-
-    ax.legend(edgecolor="0.8", fancybox=False, loc=legend_pos)
-    ax.set_xticks(feature_x_locations, feature_names_short)
-    ax.set_ylim(y_min, y_max)
-    ax.axhline(y=0, color=(0.5, 0.5, 0.5, 0.3), ls='--')
-    ax.set_ylabel('SAGE Values')
-    ax.set_xlabel('Features')
-    plt.tight_layout()
-    # plt.savefig(save_path, dpi=300)
-    plt.show()
-
-if __name__ == "__main__":
-
-    bat = [{'age': 0.028108668948204605, 'capital-gain': 0.05819510844543615, 'capital-loss': -0.009544486429741544, 'hours-per-week': 0.013155252465282052, 'fnlwgt': -0.0029857507201568488, 'workclass': -0.017613726611499247, 'education': 0.00042386958935468827, 'marital-status': 0.05439771678248133, 'occupation': 0.0016695371446097832, 'relationship': 0.08693410403102586, 'race': -0.005477809004474466, 'sex': 0.002669468345976513, 'native-country': -0.016564094138247252, 'education-num': 0.05385642606196482}, {'age': 0.027682255884510155, 'capital-gain': 0.055071822538957, 'capital-loss': -0.009050878850901029, 'hours-per-week': 0.01507412109689512, 'fnlwgt': -0.015753524269126337, 'workclass': -0.011765950551764718, 'education': -0.006686509514523585, 'marital-status': 0.08200240882771447, 'occupation': -0.003363941900762748, 'relationship': 0.06269993831493424, 'race': -0.007287603328323027, 'sex': 0.01845915407224611, 'native-country': -0.022423876746061333, 'education-num': 0.059211911293994245}, {'age': 0.02629933586076304, 'capital-gain': 0.047348888828515276, 'capital-loss': -0.008380769643279714, 'hours-per-week': 0.007628869030043197, 'fnlwgt': -0.009889991206343277, 'workclass': -0.0019752946425159617, 'education': -0.004365397345300091, 'marital-status': 0.08520484051636022, 'occupation': -0.0015253027093378555, 'relationship': 0.08201273651008828, 'race': -0.020244490079056546, 'sex': 0.0015202149566501612, 'native-country': -0.014535516949131752, 'education-num': 0.055141044503774064}, {'age': 0.02086520576527711, 'capital-gain': 0.05584322953650521, 'capital-loss': -0.004451512804291052, 'hours-per-week': 0.005407017217543419, 'fnlwgt': -0.014223539832536403, 'workclass': -0.010672812804902657, 'education': -0.0019274026641863853, 'marital-status': 0.07939688843223419, 'occupation': -0.00531566351257984, 'relationship': 0.08223375779888964, 'race': -0.012255298884018497, 'sex': 0.007240672284945291, 'native-country': -0.01716196642752966, 'education-num': 0.06409383725688973}, {'age': 0.03172187298173967, 'capital-gain': 0.05036333938639466, 'capital-loss': -0.017535865242490987, 'hours-per-week': 0.02263163451454883, 'fnlwgt': -0.010640433634705583, 'workclass': -0.016220548261482247, 'education': -0.006509064981150751, 'marital-status': 0.07753561376098746, 'occupation': -0.003088613293430229, 'relationship': 0.08879378828102573, 'race': -0.008228024623543877, 'sex': -0.001257477176132556, 'native-country': -0.016788444707527907, 'education-num': 0.05506402160315771}, {'age': 0.022885249213733968, 'capital-gain': 0.05188981662209141, 'capital-loss': -0.0027212247719945056, 'hours-per-week': 0.00735226411503803, 'fnlwgt': -0.030371798621983047, 'workclass': 0.0030405837219790945, 'education': -0.011568819869035936, 'marital-status': 0.05701380307681665, 'occupation': -0.006279217802575051, 'relationship': 0.10828757377516984, 'race': -0.014709792824596288, 'sex': 0.008093334216971913, 'native-country': -0.011103819544325651, 'education-num': 0.06916272816360407}, {'age': 0.017188197096873293, 'capital-gain': 0.055104508872217864, 'capital-loss': -0.0043284752591741665, 'hours-per-week': 0.006456527408449193, 'fnlwgt': -0.014094963296313321, 'workclass': -0.014463412048273606, 'education': -0.0016933164783028856, 'marital-status': 0.07471348400793593, 'occupation': -0.00790547489291557, 'relationship': 0.09988813376009567, 'race': -0.01540886370994617, 'sex': 0.00838854098765192, 'native-country': -0.017578997993545512, 'education-num': 0.056055099365500016}, {'age': 0.010354291915876435, 'capital-gain': 0.05330415567352984, 'capital-loss': -0.0018205928845302774, 'hours-per-week': 0.0092418954085724, 'fnlwgt': -0.016693608415608147, 'workclass': -0.012153118246136775, 'education': -0.0045776481276726, 'marital-status': 0.08940861686604011, 'occupation': -0.001520708784934167, 'relationship': 0.07302702595104182, 'race': -0.007689926107507798, 'sex': 0.009310812298269513, 'native-country': -0.020448972951357527, 'education-num': 0.06652569120600935}, {'age': 0.019755915593179133, 'capital-gain': 0.057325516840486486, 'capital-loss': -0.012006296604516168, 'hours-per-week': 0.011072765922383139, 'fnlwgt': -0.016524759309212735, 'workclass': -0.0012707924559163684, 'education': -0.0002908179329278292, 'marital-status': 0.0835673896045199, 'occupation': 0.005234255075447237, 'relationship': 0.0787406870013968, 'race': -0.008605934869527833, 'sex': 0.005992083815394923, 'native-country': -0.021159927171322278, 'education-num': 0.04913790222487249}, {'age': 0.02647903389514357, 'capital-gain': 0.05688205032555184, 'capital-loss': 0.0006749931554564515, 'hours-per-week': 0.012228737037665475, 'fnlwgt': -0.010864641096924343, 'workclass': -0.01321899165924379, 'education': 0.004665774153179211, 'marital-status': 0.05506181592877661, 'occupation': -0.005296510368831303, 'relationship': 0.09392146296134306, 'race': -0.009885204257821112, 'sex': -0.003951912535501598, 'native-country': -0.012028018331600254, 'education-num': 0.054405072119631005}]
-    inc = [{'age': 0.026212565230670844, 'capital-gain': 0.051853815484481695, 'capital-loss': -0.03375947462819334, 'hours-per-week': -0.005952738049372549, 'fnlwgt': 0.010891015572607606, 'workclass': 0.007855645437528758, 'education': 0.020711098122370295, 'marital-status': 0.06818484889581049, 'occupation': 0.008780778089487512, 'relationship': 0.11479715867765075, 'race': -0.0396267917209024, 'sex': 0.01715652361833196, 'native-country': -0.01924195964756475, 'education-num': 0.05166071497440728}, {'age': 0.031953365210154086, 'capital-gain': 0.040233816057044174, 'capital-loss': 0.015308258533997407, 'hours-per-week': 0.024763094442210132, 'fnlwgt': -0.023864240785632842, 'workclass': -0.005790773616792286, 'education': 0.002009900888309228, 'marital-status': 0.09022362975547611, 'occupation': 0.018962049430597573, 'relationship': 0.06255732633063874, 'race': -0.029619480401851726, 'sex': 0.007644171018501113, 'native-country': -0.006982363425793644, 'education-num': 0.04791278779268208}, {'age': 0.03105109977374419, 'capital-gain': 0.050636700735616574, 'capital-loss': 0.0010303195419861496, 'hours-per-week': 0.006107991972910388, 'fnlwgt': -0.007916388612774319, 'workclass': 0.014082826220676083, 'education': -0.010737141118534802, 'marital-status': 0.06908178217647476, 'occupation': 0.031675961832971174, 'relationship': 0.059901137800125445, 'race': -0.007157037614547071, 'sex': -0.0009777636433992965, 'native-country': -0.007626602269783604, 'education-num': 0.058542561430969496}, {'age': 0.0019795192237329323, 'capital-gain': 0.06266937606807628, 'capital-loss': 0.014599549092842078, 'hours-per-week': 0.007318639276298247, 'fnlwgt': -0.010726128120754559, 'workclass': 0.004489314667250441, 'education': 0.0008595487221832177, 'marital-status': 0.0873553837252126, 'occupation': 0.005523453993294373, 'relationship': 0.07941329917704586, 'race': -0.016871830093080787, 'sex': 0.0033822590452917494, 'native-country': -0.024440122334842395, 'education-num': 0.08584025865593141}, {'age': 0.011240568934383448, 'capital-gain': 0.04916588978528996, 'capital-loss': -0.002950679370124982, 'hours-per-week': 0.031098753850121487, 'fnlwgt': -0.01817694217174261, 'workclass': 0.0011091359097070786, 'education': 0.005897980198222674, 'marital-status': 0.08630570758662444, 'occupation': -0.010636354979754884, 'relationship': 0.10562996823761453, 'race': -0.03278296714088403, 'sex': 0.01516986822424592, 'native-country': 0.00030476243689187574, 'education-num': 0.04474164286708921}, {'age': 0.02527621561549269, 'capital-gain': 0.05942634621416649, 'capital-loss': -0.00778843166475, 'hours-per-week': -0.006697969601842125, 'fnlwgt': -0.002620768074243089, 'workclass': -0.011759245580332561, 'education': 0.009436931521865479, 'marital-status': 0.050490512312519775, 'occupation': -0.007253971655294637, 'relationship': 0.1055138636402115, 'race': -0.004324591297986123, 'sex': 0.00974487665515802, 'native-country': -0.016807565593019727, 'education-num': 0.08101708872067219}, {'age': 0.013665420626564943, 'capital-gain': 0.04084367707389386, 'capital-loss': -0.010685161970382525, 'hours-per-week': 0.026638396909574487, 'fnlwgt': -0.006610950459774061, 'workclass': -0.01297298975404451, 'education': 0.014663972299887171, 'marital-status': 0.05502675604932766, 'occupation': -0.01721292825385452, 'relationship': 0.10073932601856056, 'race': 0.01145610023483624, 'sex': -0.006986945163094203, 'native-country': -0.0017781354711467162, 'education-num': 0.06348274995816154}, {'age': -0.009320204798651387, 'capital-gain': 0.06737366771417695, 'capital-loss': 0.026711433068659206, 'hours-per-week': 0.01883960509326243, 'fnlwgt': 0.007472164747561111, 'workclass': -0.047188634961137156, 'education': -0.003399483946126723, 'marital-status': 0.10785094801889761, 'occupation': -0.01781491820830615, 'relationship': 0.06674819763646075, 'race': -0.0032157086034249644, 'sex': 0.00456514963644816, 'native-country': -0.010325849140494201, 'education-num': 0.07132928098600357}, {'age': 0.024523050501542555, 'capital-gain': 0.07033876936681563, 'capital-loss': 0.002557534743437581, 'hours-per-week': 0.02273157756914485, 'fnlwgt': -0.01480068897641932, 'workclass': -0.017999555202484696, 'education': 0.011004783289738721, 'marital-status': 0.09243051384466423, 'occupation': 0.01293094691824848, 'relationship': 0.05214557258523368, 'race': 0.004067037722099184, 'sex': -0.007338460285802911, 'native-country': -0.04306589901901369, 'education-num': 0.07354986167466582}, {'age': 0.027940164264492553, 'capital-gain': 0.03407085118913663, 'capital-loss': 0.0038879367482070283, 'hours-per-week': 0.0037577188617372538, 'fnlwgt': -0.0018272174107804018, 'workclass': 0.008329442815579695, 'education': -0.011475516553836418, 'marital-status': 0.06979644873463138, 'occupation': 0.03565973822196416, 'relationship': 0.0738045663238286, 'race': -0.012000996879253012, 'sex': 0.001722160244286283, 'native-country': -0.017234569261312706, 'education-num': 0.05323684425911136}]
-
-    feature_names = list(inc[0].keys())
-
-    bat = pd.DataFrame(bat)
-    inc = pd.DataFrame(inc)
-
-
-
-    data = {'batch_total': bat, 'incremental': inc}
-
-    explainer_names = list(data.keys())
-
-    grouped_boxplot(
-        data,
-        feature_names,
-        explainer_names,
-        save_path=None,
-        legend=['Batch SAGE (left)', 'iSAGE (right)'], legend_pos=2,
-        width=0.5, min_space=0.1, feature_spacing=0.5,
-        y_min=-0.06, y_max=0.12
-    )
\ No newline at end of file
diff --git a/experiments/visualization/plot_feature_importance.py b/experiments/visualization/plot_feature_importance.py
deleted file mode 100644
index 74076b2..0000000
--- a/experiments/visualization/plot_feature_importance.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import copy
-import os
-from fractions import Fraction
-import matplotlib.pyplot as plt
-import pandas as pd
-
-from increment_explain.visualization.config import SAVE_DPI
-from increment_explain.visualization.line_plots import plot_multi_line_graph
-
-
-def plot_stacked_line_plots(top_data, bot_data, top_styles, bot_styles,
-                            figsize=(10, 5), top_portion=0.5, title=None, show=False,
-                            save_path=None, top_margin=0.88):
-    ratio = Fraction(top_portion).limit_denominator()
-    top_ratio, bot_ratio = (ratio.numerator, ratio.denominator)
-    top_styles = copy.deepcopy(top_styles)
-    bot_styles = copy.deepcopy(bot_styles)
-    fig, (top_axis, bot_axis) = plt.subplots(nrows=2, ncols=1, sharex='all',
-                                             figsize=figsize,
-                                             gridspec_kw={'height_ratios': [top_ratio, bot_ratio]})
-    plt.subplots_adjust(hspace=0.000)
-    bot_axis = plot_multi_line_graph(axis=bot_axis,
-                                     y_data=bot_data['y_data'],
-                                     #x_data=bot_data['x_data'],
-                                     line_names=bot_data['line_names'],
-                                     **bot_styles)
-    top_axis = plot_multi_line_graph(axis=top_axis,
-                                     y_data=top_data['y_data'],
-                                     #x_data=top_data['x_data'],
-                                     line_names=top_data['line_names'],
-                                     **top_styles)
-    plt.tight_layout()
-    if title is not None:
-        fig.suptitle(title)
-        plt.subplots_adjust(hspace=0.000, wspace=0.000, top=top_margin)
-    else:
-        plt.subplots_adjust(hspace=0.000, wspace=0.000)
-    if save_path is not None:
-        plt.savefig(save_path)
-    if show:
-        plt.show()
-        return None
-    else:
-        return fig, (top_axis, bot_axis)
-
-
-def plot_double_stacked_line_plots(top_left_data, top_right_data, bot_left_data, bot_right_data,
-                                   top_left_styles, top_right_styles, bot_left_styles, bot_right_styles,
-                                   figsize=(10, 5), top_portion=1., left_proportion=1., title=None, show=False,
-                                   different_scales=False,
-                                   save_path=None, top_margin=0.88, wspace=0.0):
-    ratio_vertical = Fraction(top_portion).limit_denominator()
-    top_ratio, bot_ratio = (ratio_vertical.numerator, ratio_vertical.denominator)
-    ratio_horizontal = Fraction(left_proportion).limit_denominator()
-    left_ratio, right_ratio = (ratio_horizontal.numerator, ratio_horizontal.denominator)
-    sharey = 'row' if not different_scales else 'none'
-    fig, axes = plt.subplots(nrows=2, ncols=2, sharex='col', sharey=sharey,
-                             dpi=SAVE_DPI,
-                             figsize=figsize,
-                             gridspec_kw={'height_ratios': [top_ratio, bot_ratio],
-                                          'width_ratios': [left_ratio, right_ratio]})
-
-    (top_left_axis, top_right_axis), (bot_left_axis, bot_right_axis) = axes
-    top_left_axis = plot_multi_line_graph(axis=top_left_axis,
-                                          y_data=top_left_data['y_data'],
-                                          #x_data=top_left_data['x_data'],
-                                          line_names=top_left_data['line_names'],
-                                          **top_left_styles)
-    top_right_axis = plot_multi_line_graph(axis=top_right_axis,
-                                           y_data=top_right_data['y_data'],
-                                           #x_data=top_right_data['x_data'],
-                                           line_names=top_right_data['line_names'],
-                                           **top_right_styles)
-    bot_left_axis = plot_multi_line_graph(axis=bot_left_axis,
-                                          y_data=bot_left_data['y_data'],
-                                          #x_data=bot_left_data['x_data'],
-                                          line_names=bot_left_data['line_names'],
-                                          **bot_left_styles)
-    bot_right_axis = plot_multi_line_graph(axis=bot_right_axis,
-                                           y_data=bot_right_data['y_data'],
-                                           #x_data=bot_right_data['x_data'],
-                                           line_names=bot_right_data['line_names'],
-                                           **bot_right_styles)
-    plt.tight_layout()
-
-    if title is not None:
-        fig.suptitle(title)
-        plt.subplots_adjust(hspace=0., wspace=wspace, top=top_margin)
-    else:
-        plt.subplots_adjust(hspace=0., wspace=wspace)
-    if save_path is not None:
-        plt.savefig(save_path)
-    if show:
-        plt.show()
-        return None
-    else:
-        return fig, (top_left_axis, top_right_axis, bot_left_axis, bot_right_axis)
-
-
-if __name__ == "__main__":
-    pass
diff --git a/increment_explain/visualization/plotting.py b/increment_explain/visualization/plotting.py
index 31405be..0548e34 100644
--- a/increment_explain/visualization/plotting.py
+++ b/increment_explain/visualization/plotting.py
@@ -68,13 +68,13 @@ def plot(
             model_performances: Optional[dict[str, Sequence]] = None,
             performance_kw: Optional[dict] = None,
             save_name: Optional[str] = None,
-            **line_kw,
+            **plot_kw,
     ) -> None:
 
-        if 'names_to_highlight' not in line_kw:
-            line_kw['names_to_highlight'] = self.feature_names
-        if 'line_names' not in line_kw:
-            line_kw['line_names'] = self.feature_names
+        if 'names_to_highlight' not in plot_kw:
+            plot_kw['names_to_highlight'] = self.feature_names
+        if 'line_names' not in plot_kw:
+            plot_kw['line_names'] = self.feature_names
 
         if model_performances is not None:
             fig, (performance_axis, fi_axis) = plt.subplots(
@@ -83,12 +83,12 @@ def plot(
             )
             plt.subplots_adjust(hspace=0.000, wspace=0.000)
             title = None
-            if 'title' in line_kw:
-                title = copy.copy(line_kw['title'])
-                del line_kw['title']
+            if 'title' in plot_kw:
+                title = copy.copy(plot_kw['title'])
+                del plot_kw['title']
             v_lines = None
-            if 'v_lines' in line_kw:
-                v_lines = line_kw['v_lines']
+            if 'v_lines' in plot_kw:
+                v_lines = plot_kw['v_lines']
             performance_kw = {} if performance_kw is None else performance_kw
             performance_axis = plot_multi_line_graph(
                 axis=performance_axis,
@@ -108,7 +108,7 @@ def plot(
             axis=fi_axis,
             y_data=self.y_data,
             #x_data=self.x_data, TODO fix bug it's not plotting
-            **line_kw
+            **plot_kw
         )
         if save_name is not None:
             plt.savefig(save_name, dpi=200)


Test patch:
diff --git a/experiments/iSAGE_paper/test.py b/experiments/iSAGE_paper/test.py
deleted file mode 100644
index bb96e54..0000000
--- a/experiments/iSAGE_paper/test.py
+++ /dev/null
@@ -1,193 +0,0 @@
-import time
-
-import pandas as pd
-
-from experiments.setup.data import get_dataset
-from experiments.setup.explainer import get_imputer_and_storage
-from experiments.setup.loss import get_loss_function, get_training_metric
-from experiments.setup.model import get_model
-from increment_explain.explainer.sage import IncrementalSageExplainer
-from increment_explain.utils.tracker.exponential_smoothing import ExponentialSmoothingTracker
-from increment_explain.visualization import FeatureImportancePlotter
-
-DEBUG = True
-
-DATASET_1_NAME = 'stream_bike'
-
-DATASET_RANDOM_SEED = 1
-SHUFFLE_DATA = False
-N_SAMPLES_1 = None
-
-MODEL_NAME = 'ARF'
-
-N_INNER_SAMPLES: int = 1
-SMOOTHING_ALPHA: float = 0.001
-FEATURE_REMOVAL_DISTRIBUTION: str = 'marginal joint'  # ['marginal joint', 'marginal product', 'conditional']
-RESERVOIR_LENGTH: int = 100
-RESERVOIR_KIND: str = 'geometric'  # ['geometric', 'uniform']
-
-TRAINING_METRIC_WINDOW = 500
-
-MODEL_PARAMS = {
-    'ARF': {
-        'n_models': 25,
-    }
-}
-
-
-if __name__ == "__main__":
-
-    # Get Data ---------------------------------------------------------------------------------------------------------
-    dataset = get_dataset(
-        dataset_name=DATASET_1_NAME, random_seed=DATASET_RANDOM_SEED, shuffle=SHUFFLE_DATA, n_samples=N_SAMPLES_1)
-    feature_names = dataset.feature_names
-    feature_names.remove('moment')
-    cat_feature_names = dataset.cat_feature_names
-    num_feature_names = dataset.num_feature_names
-    n_samples = dataset.n_samples
-    stream = dataset.stream
-
-    # Get Model / Loss and Training Procedure --------------------------------------------------------------------------
-    task = dataset.task
-    loss_function = get_loss_function(task=task)
-    rolling_training_metric, training_metric = get_training_metric(task=task, rolling_window=TRAINING_METRIC_WINDOW)
-    model, model_function = get_model(
-        model_name=MODEL_NAME, task=task, feature_names=feature_names, **MODEL_PARAMS[MODEL_NAME])
-
-    # Get imputer and explainers ---------------------------------------------------------------------------------------
-    imputer, storage = get_imputer_and_storage(
-        model_function=model_function,
-        feature_removal_distribution=FEATURE_REMOVAL_DISTRIBUTION,
-        reservoir_kind=RESERVOIR_KIND,
-        reservoir_length=RESERVOIR_LENGTH,
-        cat_feature_names=cat_feature_names,
-        num_feature_names=num_feature_names
-    )
-
-    incremental_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        imputer=imputer,
-        storage=storage,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    incremental_welford_sage = IncrementalSageExplainer(
-        model_function=model_function,
-        loss_function=loss_function,
-        dynamic_setting=False,
-        feature_names=feature_names,
-        smoothing_alpha=SMOOTHING_ALPHA,
-        n_inner_samples=N_INNER_SAMPLES
-    )
-
-    # warm-up because of errors ----------------------------------------------------------------------------------------
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        del x_i['moment']
-        print(x_i)
-        model.learn_one(x_i, y_i)
-        if n > 10:
-            break
-
-    plotter = FeatureImportancePlotter(
-        feature_names=feature_names
-    )
-
-    time_sage = 0.
-    time_interval = 0.
-    time_learning = 0.
-
-    sage_fi_values = []
-    sage_welford_fi_values = []
-
-    model_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    marginal_loss_tracker = ExponentialSmoothingTracker(alpha=SMOOTHING_ALPHA)
-    model_loss = []
-    marginal_loss = []
-
-    for (n, (x_i, y_i)) in enumerate(stream, start=1):
-        del x_i['moment']
-        # predicting
-        y_i_pred = model_function(x_i)
-        loss_i = loss_function(y_true=y_i, y_prediction=y_i_pred)
-        model_loss_tracker.update(loss_i)
-        model_loss.append({"loss": model_loss_tracker.get()})
-        loss_marginal_i = loss_function(y_true=y_i, y_prediction=incremental_sage._marginal_prediction_tracker.get())
-        marginal_loss_tracker.update(loss_marginal_i)
-        marginal_loss.append({"loss": marginal_loss_tracker.get()})
-
-        # sage inc
-        #start_time = time.time()
-        #fi_values = incremental_welford_sage.explain_one(x_i, y_i)
-        #time_sage += time.time() - start_time
-        #sage_welford_fi_values.append(fi_values)
-        #plotter.update(fi_values, facet_name='inc-w')
-
-        # sage inc
-        #start_time = time.time()
-        #fi_values = incremental_sage.explain_one(x_i, y_i)
-        #time_sage += time.time() - start_time
-        #sage_fi_values.append(fi_values)
-        #plotter.update(fi_values, facet_name='inc')
-
-        # learning
-        start_time = time.time()
-        model.learn_one(x_i, y_i)
-        time_learning += time.time() - start_time
-
-        if DEBUG and n % 1000 == 0:
-            print(f"{n}: x_i                 {x_i}\n"
-                  f"{n}: x_i                 {model_loss_tracker.get()}\n"
-                  f"{n}: inc-sage            {incremental_sage.importance_values}\n"
-                  f"{n}: inc-sage-welford    {incremental_welford_sage.importance_values}\n"
-                  )
-
-        if n >= n_samples:
-            break
-
-    # Store the results in a database ----------------------------------------------------------------------------------
-
-    facet = "left.csv"
-    data_folder = "static_sage_in_dynamic_setting_insects"
-
-    name = "inc_fi_values"
-    df = pd.DataFrame(sage_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "inc_welford_fi_values"
-    df = pd.DataFrame(sage_welford_fi_values)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "model_loss"
-    df = pd.DataFrame(model_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    name = "marginal_loss"
-    df = pd.DataFrame(marginal_loss)
-    df.to_csv(f"plots/{data_folder}/{'_'.join((name, facet))}", index=False)
-
-    performance_kw = {"y_min": 0, "y_max": 1, "y_label": "loss", "color_list": ["red", "black"],
-                      "line_names": ["loss"], "names_to_highlight": ['loss'],
-                      "line_styles": {"model_loss": "solid", "marginal_loss": "dashed"},
-                      "markevery": {"model_loss": 100, "marginal_loss": 100},
-                      "legend": {"legend_props": {"loc": 7, "ncol": 1, "fontsize": "small"},
-                                 "legend_items": [("marginal loss", "dashed", "red"),
-                                                  ("model loss", "solid", "red")]}}
-
-    plotter.plot(
-        figsize=(5, 10),
-        model_performances={"model_loss": model_loss, "marginal_loss": marginal_loss},
-        performance_kw=performance_kw,
-        title=f'Agrawal Stream with Concept Drift',
-        y_label='SAGE values',
-        x_label='Samples',
-        names_to_highlight=['salary', 'commission', 'age', 'elevel'],
-        h_lines=[{'y': 0., 'ls': '--', 'c': 'grey', 'linewidth': 1}],
-        line_styles={'inc': '-', 'inc-w': '--'},
-        legend_style={"fontsize": "small"},
-        markevery={'inc': 10, 'inc-w': 10},
-        y_min=-0.05,
-        y_max=0.42,
-    )