Repository: ITMO-NSS-team/GAMLET (language: python)

Issue:
Implement abstract meta-algorithm (damaged commit)
Introduces flexible modular structure for conducting meta-learning experiments.

Main components are:

### DatasetsLoader

Responsible for loading and caching of datasets.
Implemented transformations:

- `DATASET_NAMES -> DATASETS_CACHE`
- `DATASET_CACHE-> DATASET`

### MetaFeaturesExtractor

Responsible for meta-features extraction and caching. Uses `DatasetsLoader` on demand.
Implemented transformations: 

- `DATASETS_CACHE-> META_FEATURES_DATAFRAME, DATASET_NAMES`
- `DATASET_NAMES -> META_FEATURES_DATAFRAME, DATASET_NAMES`


### ModelSelector

Responsible for loading models from outer sources based on any given criteria.  Uses `DatasetsLoader` on demand. 
Expected transformation:

- `... -> DATASET_NAMES, MODELS`

Implemented various transformations based on FEDOT launch results. 

### DatasetsSimilarityAssessor

Responsible for selecting datasets based on meta-features. 
Implemented transformations:

- `META_FEATURES_DATAFRAME, DATASET_NAMES -> DATASET_NAMES`

### ModelAdvisor

Responsible for advising models for datasets based on previously learned data.

Expected transformation:

- `fit(DATASET_NAMES, MODELS)` (current implementation utilizes pre-trained `DatasetsSimilarityAssessor` instead of `DATASET_NAMES`)

- `preditct(DATASET_NAMES) -> DATASET_NAMES, MODELS`.


Discussion / hints:

Fix patch:
diff --git a/.pep8speaks.yml b/.pep8speaks.yml
new file mode 100644
index 00000000..03fa3391
--- /dev/null
+++ b/.pep8speaks.yml
@@ -0,0 +1,9 @@
+scanner:
+  diff_only: False  # The entire file touched by the PR will be scanned for errors
+  linter: flake8
+
+flake8:
+  max-line-length: 120  # Default is 79 in PEP 8
+  ignore: []
+
+no_blank_comment: False  # Every PR will have its own pep8speaks stats
diff --git a/examples/0_loading_data/load_list_of_datasests.py b/examples/0_loading_data/load_list_of_datasests.py
new file mode 100644
index 00000000..c2ee1cbb
--- /dev/null
+++ b/examples/0_loading_data/load_list_of_datasests.py
@@ -0,0 +1,16 @@
+from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
+
+
+def get_datasets():
+    dataset_names = [
+        'nomao', 'sylvine', 'kc1', 'jungle_chess_2pcs_raw_endgame_complete', 'credit-g', 'delta_ailerons', 'pol'
+    ]
+    datasets_loader = OpenMLDatasetsLoader()
+    datasets = datasets_loader.load(dataset_names)
+    print(f'Datasets "{", ".join(dataset_names)}" are available at the paths:')
+    print('\n'.join(str(d) for d in datasets))
+    return datasets
+
+
+if __name__ == '__main__':
+    result = get_datasets()
diff --git a/examples/1_loading_models/load_best_models_from_fedot_results.py b/examples/1_loading_models/load_best_models_from_fedot_results.py
new file mode 100644
index 00000000..c18883f1
--- /dev/null
+++ b/examples/1_loading_models/load_best_models_from_fedot_results.py
@@ -0,0 +1,10 @@
+from meta_automl.data_preparation.models_loaders import FEDOTPipelinesLoader
+
+
+def main():
+    models_loader = FEDOTPipelinesLoader(datasets_to_load=['cnae-9'], launch_dir=r'\\10.9.14.114\calc\Nikitin\aggregated_4h\agg')
+    return models_loader.load()
+
+
+if __name__ == '__main__':
+    result = main()
diff --git a/examples/2_extracting_datasets_meta_features/extract_with_load_on_demand.py b/examples/2_extracting_datasets_meta_features/extract_with_load_on_demand.py
new file mode 100644
index 00000000..9519e6ca
--- /dev/null
+++ b/examples/2_extracting_datasets_meta_features/extract_with_load_on_demand.py
@@ -0,0 +1,15 @@
+from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
+from meta_automl.data_preparation.meta_features_extractors import PymfeExtractor
+
+
+def main():
+    dataset_names = [
+        'nomao', 'sylvine'
+    ]
+    extractor = PymfeExtractor(extractor_params={'groups': 'general'}, datasets_loader=OpenMLDatasetsLoader())
+    meta_features = extractor.extract(dataset_names)
+    return meta_features
+
+
+if __name__ == '__main__':
+    result = main()
diff --git a/examples/2_extracting_datasets_meta_features/load_and_extract_features_sequentially.py b/examples/2_extracting_datasets_meta_features/load_and_extract_features_sequentially.py
new file mode 100644
index 00000000..f1d21cf4
--- /dev/null
+++ b/examples/2_extracting_datasets_meta_features/load_and_extract_features_sequentially.py
@@ -0,0 +1,18 @@
+from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
+from meta_automl.data_preparation.meta_features_extractors import PymfeExtractor
+
+
+def main():
+    dataset_names = [
+        'nomao', 'sylvine', 'kc1', 'jungle_chess_2pcs_raw_endgame_complete', 'credit-g', 'delta_ailerons', 'pol'
+    ]
+    loader = OpenMLDatasetsLoader()
+    extractor = PymfeExtractor(extractor_params={'groups': 'general'})
+
+    cached_datasets = loader.load(dataset_names)
+    meta_features = extractor.extract(cached_datasets)
+    return meta_features
+
+
+if __name__ == '__main__':
+    result = main()
diff --git a/examples/3_selecting_similar_datasets/select_similar_datasets_by_knn.py b/examples/3_selecting_similar_datasets/select_similar_datasets_by_knn.py
new file mode 100644
index 00000000..dc1c190c
--- /dev/null
+++ b/examples/3_selecting_similar_datasets/select_similar_datasets_by_knn.py
@@ -0,0 +1,26 @@
+from sklearn.model_selection import train_test_split
+
+from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
+from meta_automl.data_preparation.meta_features_extractors import PymfeExtractor
+from meta_automl.meta_algorithm.datasets_similarity_assessors import KNNSimilarityAssessor
+
+
+def main():
+    # Define datasets.
+    dataset_names = ['monks-problems-1', 'apsfailure', 'australian', 'bank-marketing']
+    # Extract meta-features and load on demand.
+    extractor = PymfeExtractor(extractor_params={'groups': 'general'}, datasets_loader=OpenMLDatasetsLoader())
+    meta_features = extractor.extract(dataset_names)
+    # Preprocess meta-features, as KNN does not support NaNs.
+    meta_features = meta_features.dropna(axis=1, how='any')
+    # Split datasets to train (preprocessing) and test (actual meta-algorithm objects).
+    x_train, x_test = train_test_split(meta_features, train_size=0.75, random_state=42)
+    y_train = x_train.index
+    assessor = KNNSimilarityAssessor({'n_neighbors': 1}, n_best=2)
+    assessor.fit(x_train, y_train)
+    # Get models for the best fitting datasets from train.
+    return x_test.index, assessor.predict(x_test)
+
+
+if __name__ == '__main__':
+    result = main()
diff --git a/examples/4_advising_models/advise_models_from_similar_datasets.py b/examples/4_advising_models/advise_models_from_similar_datasets.py
new file mode 100644
index 00000000..85d62b48
--- /dev/null
+++ b/examples/4_advising_models/advise_models_from_similar_datasets.py
@@ -0,0 +1,41 @@
+from fedot.core.optimisers.fitness import SingleObjFitness
+from fedot.core.pipelines.pipeline_builder import PipelineBuilder
+from sklearn.model_selection import train_test_split
+
+from meta_automl.data_preparation.dataset import DatasetCache
+from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
+from meta_automl.data_preparation.meta_features_extractors import PymfeExtractor
+from meta_automl.data_preparation.model import Model
+from meta_automl.meta_algorithm.datasets_similarity_assessors import KNNSimilarityAssessor
+from meta_automl.meta_algorithm.model_advisors import DiverseFEDOTPipelineAdvisor
+
+
+def main():
+    # Define datasets.
+    dataset_names = ['monks-problems-1', 'apsfailure', 'australian', 'bank-marketing']
+    # Extract meta-features and load on demand.
+    extractor = PymfeExtractor(extractor_params={'groups': 'general'}, datasets_loader=OpenMLDatasetsLoader())
+    meta_features = extractor.extract(dataset_names)
+    # Preprocess meta-features, as KNN does not support NaNs.
+    meta_features = meta_features.dropna(axis=1, how='any')
+    # Split datasets to train (preprocessing) and test (actual meta-algorithm objects).
+    x_train, x_test = train_test_split(meta_features, train_size=0.75, random_state=42)
+    y_train = x_train.index
+    assessor = KNNSimilarityAssessor({'n_neighbors': 2}, n_best=2)
+    assessor.fit(x_train, y_train)
+    # Define best models for datasets.
+    best_pipelines = [
+        PipelineBuilder().add_node('scaling').add_node('rf').to_pipeline(),
+        PipelineBuilder().add_node('normalization').add_node('logit').to_pipeline(),
+        PipelineBuilder().add_node('rf').add_node('logit').to_pipeline()
+    ]
+    best_models = [[Model(pipeline, SingleObjFitness(1), DatasetCache(dataset_name))]
+                   for dataset_name, pipeline in zip(y_train, best_pipelines)]
+
+    dataset_names_to_best_pipelines = dict(zip(y_train, best_models))
+    advisor = DiverseFEDOTPipelineAdvisor(assessor, minimal_distance=2).fit(dataset_names_to_best_pipelines)
+    return advisor.predict(x_test)
+
+
+if __name__ == '__main__':
+    result = main()
diff --git a/meta_automl/__init__.py b/meta_automl/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/meta_automl/data_preparation/__init__.py b/meta_automl/data_preparation/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/meta_automl/data_preparation/data_manager.py b/meta_automl/data_preparation/data_manager.py
new file mode 100644
index 00000000..0a743e28
--- /dev/null
+++ b/meta_automl/data_preparation/data_manager.py
@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import pickle
+from os import PathLike
+from pathlib import Path
+from typing import Dict, Any, Union
+
+PathType = Union[PathLike, str]
+DEFAULT_CACHE_EXTENSION = '.pkl'
+
+
+class DataManager:
+
+    @classmethod
+    def get_dataset_cache_path(cls, dataset_name: str) -> Path:
+        return cls.get_datasets_dir().joinpath(dataset_name).with_suffix(DEFAULT_CACHE_EXTENSION)
+
+    @classmethod
+    def get_datasets_dir(cls) -> Path:
+        datasets_dir = cls.get_data_dir().joinpath('datasets')
+        return cls.ensure_dir_exists(datasets_dir)
+
+    @classmethod
+    def get_data_dir(cls) -> Path:
+        data_dir = cls.get_project_root().joinpath('data')
+        return cls.ensure_dir_exists(data_dir)
+
+    @classmethod
+    def ensure_dir_exists(cls, dir_: Path) -> Path:
+        if not dir_.exists():
+            dir_.mkdir()
+        return dir_
+
+    @classmethod
+    def get_project_root(cls) -> Path:
+        """Returns project root folder."""
+        return Path(__file__).parents[2]
+
+    @classmethod
+    def get_meta_features_cache_path(cls, dataset_name: str, source_name: str):
+        meta_features_dir = cls.ensure_dir_exists(cls.get_data_dir().joinpath(source_name))
+        return meta_features_dir.joinpath(dataset_name).with_suffix('.pkl')
+
+    @classmethod
+    def get_meta_features_dict(cls, dataset_name: str, source_name: str) -> Dict[str, Any]:
+        meta_features_file = cls.get_meta_features_cache_path(dataset_name, source_name)
+        if not meta_features_file.exists():
+            return {}
+        with open(meta_features_file, 'rb') as f:
+            meta_features = pickle.load(f)
+        return meta_features
+
+    @classmethod
+    def update_meta_features_dict(cls, dataset_name: str, source_name: str, meta_features: Dict[str, Any]):
+        meta_features_file = cls.get_meta_features_cache_path(dataset_name, source_name)
+        meta_features_old = cls.get_meta_features_dict(dataset_name, source_name)
+        with open(meta_features_file, 'wb') as f:
+            meta_features_old.update(meta_features)
+            pickle.dump(meta_features, f)
diff --git a/meta_automl/data_preparation/dataset.py b/meta_automl/data_preparation/dataset.py
new file mode 100644
index 00000000..23dda83c
--- /dev/null
+++ b/meta_automl/data_preparation/dataset.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import pickle
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Union, Optional, List
+
+import numpy as np
+import pandas as pd
+import scipy as sp
+
+from meta_automl.data_preparation.data_manager import DataManager
+
+
+class NoCacheError(FileNotFoundError):
+    pass
+
+
+@dataclass
+class DatasetCache:
+    name: str
+    _cache_path: Optional[Path] = None
+    _id: Optional[int] = None
+
+    @property
+    def id(self):
+        return self._id or self.name
+
+    @property
+    def cache_path(self):
+        return self._cache_path or DataManager.get_dataset_cache_path(self.name)
+
+    @cache_path.setter
+    def cache_path(self, val):
+        self._cache_path = val
+
+    def from_cache(self) -> Dataset:
+        if not self.cache_path.exists():
+            raise NoCacheError(f'Dataset {self.name} not found!')
+        with open(self.cache_path, 'rb') as f:
+            dataset = pickle.load(f)
+        dataset.cache_path = self.cache_path
+        return dataset
+
+
+@dataclass
+class Dataset:
+    name: str
+    x: Union[np.ndarray, pd.DataFrame, sp.sparse.csr_matrix]
+    y: Optional[Union[np.ndarray, pd.DataFrame]] = None
+    categorical_indicator: Optional[List[bool]] = None
+    attribute_names: Optional[List[str]] = None
+    cache_path: Optional[Path] = None
+    _id: Optional[int] = None
+
+    def dump_to_cache(self, cache_path: Optional[Path] = None) -> DatasetCache:
+        cache_path = cache_path or self.cache_path
+        with open(cache_path, 'wb') as f:
+            pickle.dump(self, f)
+        return DatasetCache(self.name, cache_path, self.id)
+
+    @property
+    def id(self):
+        return self._id or self.name
diff --git a/meta_automl/data_preparation/datasets_loaders/__init__.py b/meta_automl/data_preparation/datasets_loaders/__init__.py
new file mode 100644
index 00000000..3908c8e0
--- /dev/null
+++ b/meta_automl/data_preparation/datasets_loaders/__init__.py
@@ -0,0 +1,2 @@
+from .datasets_loader import DatasetsLoader
+from .openml_datasets_loader import OpenMLDatasetsLoader, OpenMLDatasetID
diff --git a/meta_automl/data_preparation/datasets_loaders/datasets_loader.py b/meta_automl/data_preparation/datasets_loaders/datasets_loader.py
new file mode 100644
index 00000000..8faba6d0
--- /dev/null
+++ b/meta_automl/data_preparation/datasets_loaders/datasets_loader.py
@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import List, Type
+
+from meta_automl.data_preparation.data_manager import DataManager
+from meta_automl.data_preparation.dataset import Dataset, DatasetCache, NoCacheError
+
+
+class DatasetsLoader:
+    data_manager: Type[DataManager] = DataManager
+
+    @abstractmethod
+    def load(self, *args, **kwargs) -> List[DatasetCache]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def load_single(self, *args, **kwargs) -> DatasetCache:
+        raise NotImplementedError()
+
+    def cache_to_memory(self, dataset: DatasetCache) -> Dataset:
+        try:
+            return dataset.from_cache()
+        except NoCacheError:
+            return self.load_single(dataset.id).from_cache()
diff --git a/meta_automl/data_preparation/datasets_loaders/openml_datasets_loader.py b/meta_automl/data_preparation/datasets_loaders/openml_datasets_loader.py
new file mode 100644
index 00000000..f22bb0c6
--- /dev/null
+++ b/meta_automl/data_preparation/datasets_loaders/openml_datasets_loader.py
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import List, Union
+
+import openml
+
+from meta_automl.data_preparation.dataset import DatasetCache, Dataset
+from meta_automl.data_preparation.datasets_loaders import DatasetsLoader
+
+OpenMLDatasetID = Union[str, int]
+
+
+class OpenMLDatasetsLoader(DatasetsLoader):
+
+    def __init__(self):
+        self.dataset_sources = []
+
+    def load(self, dataset_sources: List[OpenMLDatasetID]) -> List[DatasetCache]:
+        self.dataset_sources = dataset_sources
+
+        datasets = []
+        # TODO: Optimize like this
+        #  https://github.com/openml/automlbenchmark/commit/a09dc8aee96178dd14837d9e1cd519d1ec63f804
+        for source in self.dataset_sources:
+            dataset = self.load_single(source)
+            datasets.append(dataset)
+        return datasets
+
+    def load_single(self, source: OpenMLDatasetID):
+        return self.get_openml_dataset(source)
+
+    def get_openml_dataset(self, dataset_id: OpenMLDatasetID, force_download: bool = False) -> DatasetCache:
+        openml_dataset = openml.datasets.get_dataset(dataset_id, download_data=False)
+        name = openml_dataset.name.lower()
+        dataset_cache_path = self.data_manager.get_dataset_cache_path(name)
+        if dataset_cache_path.exists() and not force_download:
+            dataset_cache = DatasetCache(name, dataset_cache_path)
+        else:
+            dataset_id = openml_dataset.id
+            X, y, categorical_indicator, attribute_names = openml_dataset.get_data(
+                target=openml_dataset.default_target_attribute,
+                dataset_format='array'
+            )
+            dataset = Dataset(name, X, y, categorical_indicator, attribute_names, _id=dataset_id)
+            dataset_cache = dataset.dump_to_cache(dataset_cache_path)
+        return dataset_cache
diff --git a/meta_automl/data_preparation/meta_features_extractors/__init__.py b/meta_automl/data_preparation/meta_features_extractors/__init__.py
new file mode 100644
index 00000000..04909175
--- /dev/null
+++ b/meta_automl/data_preparation/meta_features_extractors/__init__.py
@@ -0,0 +1,2 @@
+from .meta_features_extractor import MetaFeaturesExtractor
+from .pymfe_extractor import PymfeExtractor
diff --git a/meta_automl/data_preparation/meta_features_extractors/meta_features_extractor.py b/meta_automl/data_preparation/meta_features_extractors/meta_features_extractor.py
new file mode 100644
index 00000000..dc7ccf5a
--- /dev/null
+++ b/meta_automl/data_preparation/meta_features_extractors/meta_features_extractor.py
@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Optional, Iterable, Dict, Any, Type
+
+import pandas as pd
+
+from meta_automl.data_preparation.data_manager import DataManager
+
+
+class MetaFeaturesExtractor:
+    DEFAULT_PARAMS: Optional[Dict[str, Any]] = None
+    SOURCE: Optional[str] = None
+    data_manager: Type[DataManager] = DataManager
+
+    @abstractmethod
+    def extract(self, datasets) -> pd.DataFrame:
+        raise NotImplementedError()
+
+    def _get_meta_features_cache(self, dataset_name: str, meta_feature_names: Iterable[str]):
+        cache = self.data_manager.get_meta_features_dict(dataset_name, self.SOURCE)
+        if set(meta_feature_names) ^ cache.keys():
+            return None
+        else:
+            return {mf_name: cache[mf_name] for mf_name in meta_feature_names}
+
+    def _update_meta_features_cache(self, dataset_name: str, meta_features_dict: Dict[str, Any]):
+        self.data_manager.update_meta_features_dict(dataset_name, self.SOURCE, meta_features_dict)
diff --git a/meta_automl/data_preparation/meta_features_extractors/pymfe_extractor.py b/meta_automl/data_preparation/meta_features_extractors/pymfe_extractor.py
new file mode 100644
index 00000000..2848998e
--- /dev/null
+++ b/meta_automl/data_preparation/meta_features_extractors/pymfe_extractor.py
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import List, Union, Dict, Any
+
+import pandas as pd
+from pymfe.mfe import MFE
+
+from meta_automl.data_preparation.dataset import DatasetCache
+from meta_automl.data_preparation.datasets_loaders import DatasetsLoader, OpenMLDatasetsLoader
+from meta_automl.data_preparation.meta_features_extractors import MetaFeaturesExtractor
+
+
+class PymfeExtractor(MetaFeaturesExtractor):
+    DEFAULT_PARAMS = {'groups': 'default'}
+    SOURCE = 'pymfe'
+
+    def __init__(self, extractor_params: Dict[str, Any] = None, datasets_loader: DatasetsLoader = None):
+        self.extractor_params = extractor_params if extractor_params is not None else self.DEFAULT_PARAMS
+        self._datasets_loader = datasets_loader or OpenMLDatasetsLoader()
+        self._extractor = MFE(**self.extractor_params)
+
+    @property
+    def datasets_loader(self) -> DatasetsLoader:
+        if not self._datasets_loader:
+            raise ValueError("Datasets loader not provided!")
+        return self._datasets_loader
+
+    def extract(self, datasets: List[Union[DatasetCache, str]]) -> pd.DataFrame:
+        meta_features = {}
+        meta_feature_names = self._extractor.extract_metafeature_names()
+        load_dataset = self.datasets_loader.cache_to_memory
+        for dataset in datasets:
+            if isinstance(dataset, str):
+                dataset = DatasetCache(dataset)
+            if mfs := self._get_meta_features_cache(dataset.name, meta_feature_names):
+                meta_features[dataset.name] = mfs
+            else:
+                loaded_dataset = load_dataset(dataset)
+                cat_cols = [i for i, val in enumerate(loaded_dataset.categorical_indicator) if val]
+                mfe = self._extractor.fit(loaded_dataset.x, loaded_dataset.y, cat_cols=cat_cols)
+                feature_names, dataset_features = mfe.extract(out_type=tuple)
+                mfs = dict(zip(feature_names, dataset_features))
+                self._update_meta_features_cache(dataset.name, mfs)
+                meta_features[dataset.name] = mfs
+        meta_features = pd.DataFrame.from_dict(meta_features, orient='index')
+        return meta_features
diff --git a/meta_automl/data_preparation/model.py b/meta_automl/data_preparation/model.py
new file mode 100644
index 00000000..44543dfe
--- /dev/null
+++ b/meta_automl/data_preparation/model.py
@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+from typing import Any
+
+from fedot.core.optimisers.fitness import Fitness
+
+from meta_automl.data_preparation.dataset import DatasetCache
+
+
+@dataclass
+class Model:
+    predictor: Any
+    fitness: Fitness
+    data: DatasetCache
diff --git a/meta_automl/data_preparation/models_loaders/__init__.py b/meta_automl/data_preparation/models_loaders/__init__.py
new file mode 100644
index 00000000..448b52e7
--- /dev/null
+++ b/meta_automl/data_preparation/models_loaders/__init__.py
@@ -0,0 +1,2 @@
+from .models_loader import ModelsLoader
+from .fedot_pipelines_loader import FEDOTPipelinesLoader
diff --git a/meta_automl/data_preparation/models_loaders/fedot_pipelines_loader.py b/meta_automl/data_preparation/models_loaders/fedot_pipelines_loader.py
new file mode 100644
index 00000000..b5fe2abf
--- /dev/null
+++ b/meta_automl/data_preparation/models_loaders/fedot_pipelines_loader.py
@@ -0,0 +1,130 @@
+from functools import partial
+from pathlib import Path
+from typing import List, Union, Optional, Literal
+
+import numpy as np
+from fedot.core.data.data import InputData
+from fedot.core.log import default_log
+from fedot.core.optimisers.objective import PipelineObjectiveEvaluate
+from fedot.core.optimisers.objective.metrics_objective import MetricsObjective
+from fedot.core.pipelines.pipeline import Pipeline
+from fedot.core.repository.dataset_types import DataTypesEnum
+from fedot.core.repository.quality_metrics_repository import ClassificationMetricsEnum
+from fedot.core.repository.tasks import Task, TaskTypesEnum
+from fedot.core.validation.split import tabular_cv_generator
+from tqdm import tqdm
+
+from meta_automl.data_preparation.data_manager import PathType
+from meta_automl.data_preparation.dataset import DatasetCache, NoCacheError
+from meta_automl.data_preparation.datasets_loaders import DatasetsLoader, OpenMLDatasetsLoader
+from meta_automl.data_preparation.model import Model
+from meta_automl.data_preparation.models_loaders import ModelsLoader
+
+
+def evaluate_classification_fedot_pipeline(pipeline, input_data):
+    cv_folds = partial(tabular_cv_generator, input_data, folds=5)
+    objective_eval = PipelineObjectiveEvaluate(MetricsObjective(ClassificationMetricsEnum.ROCAUC), cv_folds,
+                                               eval_n_jobs=-1)
+    fitness = objective_eval(pipeline)
+    return fitness
+
+
+def get_n_best_fedot_performers(dataset_cache: DatasetCache, pipelines: List[Pipeline], datasets_loader: DatasetsLoader,
+                                n_best: int = 1) -> List[Model]:
+    loaded_dataset = datasets_loader.cache_to_memory(dataset_cache)
+    X, y_test = loaded_dataset.x, loaded_dataset.y
+    input_data = InputData(idx=np.arange(0, len(X)), features=X, target=y_test, data_type=DataTypesEnum.table,
+                           task=Task(TaskTypesEnum.classification))
+    fitnesses = []
+    models = []
+    for pipeline in tqdm(pipelines, desc='Evaluating pipelines'):
+        fitness = evaluate_classification_fedot_pipeline(pipeline, input_data)
+        fitnesses.append(fitness)
+        models.append(Model(pipeline, fitness, dataset_cache))
+
+    best_models = [models.pop(np.argmax(fitnesses)) for _ in range(min(n_best, len(pipelines)))]
+    return best_models
+
+
+class FEDOTPipelinesLoader(ModelsLoader):
+    def __init__(self, datasets_to_load: Union[List[Union[DatasetCache, str]], Literal['auto']] = 'auto',
+                 candidate_pipelines: Optional[List[List[Pipeline]]] = None,
+                 candidate_pipeline_paths: Optional[List[List[PathType]]] = None,
+                 launch_dir: Optional[PathType] = None,
+                 datasets_loader: Optional[DatasetsLoader] = None):
+
+        self.log = default_log(self)
+
+        self.datasets_loader = datasets_loader or OpenMLDatasetsLoader()
+
+        self.launch_dir: Path = Path(launch_dir) if isinstance(launch_dir, str) else launch_dir
+
+        self._datasets: List[DatasetCache] = (self._define_datasets() if datasets_to_load == 'auto'
+                                              else self._dataset_names_to_cache(datasets_to_load))
+
+        self.candidate_pipelines = candidate_pipelines
+
+        if self.candidate_pipelines is None:
+            candidate_pipeline_paths = candidate_pipeline_paths or self._define_pipeline_paths()
+            self._import_pipelines(candidate_pipeline_paths)
+
+    def load(self, datasets: Union[List[str], Literal['auto']] = 'auto', n_best: int = 1) -> List[List[Model]]:
+        if datasets != 'auto':
+            datasets = self._dataset_names_to_cache(datasets)
+            difference = set(d.name for d in datasets) - set(self.dataset_names)
+            if difference:
+                raise ValueError(f'Results for these datasets are not available: {difference}.')
+        else:
+            datasets = self._datasets
+
+        models = []
+        for dataset, candidate_pipelines in tqdm(list(zip(datasets, self.candidate_pipelines)),
+                                                 desc='Selecting best models', unit='dataset'):
+            best_performers = get_n_best_fedot_performers(dataset, candidate_pipelines, self.datasets_loader, n_best)
+            models.append(best_performers)
+        return models
+
+    def _define_pipeline_paths(self) -> List[List[Path]]:
+        if not self.launch_dir:
+            raise ValueError('Launch dir or model paths must be provided!')
+
+        dataset_names = self.dataset_names
+        datasets_models_paths = dict(zip(dataset_names, [[]] * len(dataset_names)))
+
+        for dataset_name in tqdm(dataset_names, desc='Defining model paths', unit='dataset'):
+            for model_path in self.launch_dir.joinpath(dataset_name).glob(r'FEDOT*\*\*\launch_*.json'):
+                datasets_models_paths[dataset_name].append(model_path)
+
+        return [datasets_models_paths[dataset.name] for dataset in self._datasets]
+
+    def _import_pipelines(self, candidate_pipeline_paths: List[List[PathType]]):
+        candidate_pipelines = []
+        for dataset, paths in tqdm(list(zip(self._datasets, candidate_pipeline_paths)),
+                                   desc='Importing pipelines', unit='dataset'):
+            candidates_for_dataset = [Pipeline.from_serialized(str(p)) for p in paths]
+            if not candidates_for_dataset:
+                self.log.warning(f'No pipelines found for the dataset "{dataset.name}".')
+            candidate_pipelines.append(candidates_for_dataset)
+        self.candidate_pipelines = candidate_pipelines
+
+    def _define_datasets(self) -> List[DatasetCache]:
+        if not self.launch_dir:
+            raise ValueError('Launch dir or datasets must be provided!')
+
+        datasets = list({p.parents[2].name for p in self.launch_dir.glob(r'*\FEDOT*\*\launch_0')})
+        datasets.sort()
+        datasets = self._dataset_names_to_cache(datasets)
+        return datasets
+
+    @property
+    def dataset_names(self):
+        return [d.name if isinstance(d, DatasetCache) else d for d in self._datasets]
+
+    @staticmethod
+    def _dataset_names_to_cache(datasets: List[Union[str, DatasetCache]]) -> List[DatasetCache]:
+        new_list = []
+        for dataset in datasets:
+            if isinstance(dataset, str):
+                dataset = DatasetCache(dataset)
+            new_list.append(dataset)
+        return new_list
diff --git a/meta_automl/data_preparation/models_loaders/models_loader.py b/meta_automl/data_preparation/models_loaders/models_loader.py
new file mode 100644
index 00000000..d6f4d5c6
--- /dev/null
+++ b/meta_automl/data_preparation/models_loaders/models_loader.py
@@ -0,0 +1,12 @@
+from abc import abstractmethod
+
+from typing import List
+
+from meta_automl.data_preparation.model import Model
+
+
+class ModelsLoader:
+
+    @abstractmethod
+    def load(self, *args, **kwargs) -> List[List[Model]]:
+        raise NotImplementedError()
diff --git a/meta_automl/meta_algorithm/__init__.py b/meta_automl/meta_algorithm/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/meta_automl/meta_algorithm/datasets_similarity_assessors/__init__.py b/meta_automl/meta_algorithm/datasets_similarity_assessors/__init__.py
new file mode 100644
index 00000000..621a68e0
--- /dev/null
+++ b/meta_automl/meta_algorithm/datasets_similarity_assessors/__init__.py
@@ -0,0 +1,2 @@
+from .datasets_similarity_assessor import DatasetsSimilarityAssessor
+from .predict_proba_similarity_assessors import KNNSimilarityAssessor, PredictProbaSimilarityAssessor
diff --git a/meta_automl/meta_algorithm/datasets_similarity_assessors/datasets_similarity_assessor.py b/meta_automl/meta_algorithm/datasets_similarity_assessors/datasets_similarity_assessor.py
new file mode 100644
index 00000000..b595d0cb
--- /dev/null
+++ b/meta_automl/meta_algorithm/datasets_similarity_assessors/datasets_similarity_assessor.py
@@ -0,0 +1,18 @@
+from abc import abstractmethod
+
+from typing import List
+
+
+class DatasetsSimilarityAssessor:
+    @abstractmethod
+    def fit(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def predict(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    @property
+    @abstractmethod
+    def datasets(self) -> List[str]:
+        raise NotImplementedError()
diff --git a/meta_automl/meta_algorithm/datasets_similarity_assessors/predict_proba_similarity_assessors.py b/meta_automl/meta_algorithm/datasets_similarity_assessors/predict_proba_similarity_assessors.py
new file mode 100644
index 00000000..8254c745
--- /dev/null
+++ b/meta_automl/meta_algorithm/datasets_similarity_assessors/predict_proba_similarity_assessors.py
@@ -0,0 +1,59 @@
+from typing import Optional, Dict, Any, List, Iterable
+
+import numpy as np
+import pandas as pd
+from sklearn.neighbors import KNeighborsClassifier
+
+from meta_automl.meta_algorithm.datasets_similarity_assessors.datasets_similarity_assessor import \
+    DatasetsSimilarityAssessor
+
+
+class PredictProbaSimilarityAssessor(DatasetsSimilarityAssessor):
+    def __init__(self, model, n_best: int = 1):
+        self._inner_model = model
+        self.n_best = n_best
+
+    @property
+    def datasets(self) -> List[str]:
+        return self._inner_model.classes_
+
+    @property
+    def feature_names(self) -> List[str]:
+        return self._inner_model.feature_names_in_
+
+    @staticmethod
+    def preprocess_meta_features(meta_features: pd.DataFrame) -> pd.DataFrame:
+        return meta_features.dropna(axis=1, how='any')
+
+    def _preprocess_predict_features(self, meta_features: pd.DataFrame) -> pd.DataFrame:
+        return meta_features[self.feature_names]
+
+    def fit(self, meta_features: pd.DataFrame, datasets: Iterable[str]):
+        meta_features = self.preprocess_meta_features(meta_features)
+        self._inner_model.fit(meta_features, datasets)
+
+    def predict_proba(self, meta_features: pd.DataFrame) -> List[List[float]]:
+        return self._inner_model.predict_proba(meta_features)
+
+    def predict(self, meta_features: pd.DataFrame) -> List[List[str]]:
+        meta_features = self._preprocess_predict_features(meta_features)
+        predict_probs = self.predict_proba(meta_features)
+        final_prediction = []
+        for probabilities in predict_probs:
+            probabilities = list(probabilities)
+            predictions = []
+            for _ in range(self.n_best):
+                predicted_class_idx = np.argmax(probabilities)
+                predicted_class = self.datasets[predicted_class_idx]
+                predictions.append(predicted_class)
+                probabilities.pop(predicted_class_idx)
+            final_prediction.append(predictions)
+
+        return final_prediction
+
+
+class KNNSimilarityAssessor(PredictProbaSimilarityAssessor):
+    def __init__(self, model_params: Optional[Dict[str, Any]] = None, n_best: int = 1):
+        model_params = model_params or dict()
+        model = KNeighborsClassifier(**model_params)
+        super().__init__(model, n_best)
diff --git a/meta_automl/meta_algorithm/model_advisors/__init__.py b/meta_automl/meta_algorithm/model_advisors/__init__.py
new file mode 100644
index 00000000..5ce829a2
--- /dev/null
+++ b/meta_automl/meta_algorithm/model_advisors/__init__.py
@@ -0,0 +1,5 @@
+from meta_automl.meta_algorithm.model_advisors.model_advisor import (
+    ModelAdvisor,
+    SimpleSimilarityModelAdvisor
+)
+from meta_automl.meta_algorithm.model_advisors.diverse_fedot_pipeline_advisor import DiverseFEDOTPipelineAdvisor
diff --git a/meta_automl/meta_algorithm/model_advisors/diverse_fedot_pipeline_advisor.py b/meta_automl/meta_algorithm/model_advisors/diverse_fedot_pipeline_advisor.py
new file mode 100644
index 00000000..aa91e0db
--- /dev/null
+++ b/meta_automl/meta_algorithm/model_advisors/diverse_fedot_pipeline_advisor.py
@@ -0,0 +1,27 @@
+from typing import Callable, List, Iterable
+
+from fedot.core.dag.linked_graph import get_distance_between
+from fedot.core.pipelines.pipeline import Pipeline
+
+from meta_automl.data_preparation.model import Model
+from meta_automl.meta_algorithm.datasets_similarity_assessors import DatasetsSimilarityAssessor
+from meta_automl.meta_algorithm.model_advisors import SimpleSimilarityModelAdvisor
+
+
+class DiverseFEDOTPipelineAdvisor(SimpleSimilarityModelAdvisor):
+    def __init__(self,
+                 fitted_similarity_assessor: DatasetsSimilarityAssessor,
+                 minimal_distance: int = 1,
+                 distance_func: Callable[[Pipeline, Pipeline], int] = get_distance_between):
+        super().__init__(fitted_similarity_assessor)
+        self.minimal_distance = minimal_distance
+        self.distance_func = distance_func
+
+    def _predict_single(self, similar_dataset_names: Iterable[str]) -> List[Model]:
+        dataset_advice = super()._predict_single(similar_dataset_names)
+        first_model = dataset_advice[0]
+        diverse_dataset_advice = [first_model]
+        for model in dataset_advice[1:]:
+            if self.distance_func(first_model.predictor, model.predictor) > self.minimal_distance:
+                diverse_dataset_advice.append(model)
+        return diverse_dataset_advice
diff --git a/meta_automl/meta_algorithm/model_advisors/model_advisor.py b/meta_automl/meta_algorithm/model_advisors/model_advisor.py
new file mode 100644
index 00000000..b585bf27
--- /dev/null
+++ b/meta_automl/meta_algorithm/model_advisors/model_advisor.py
@@ -0,0 +1,41 @@
+from abc import abstractmethod
+from typing import List, Dict, Iterable
+
+import pandas as pd
+
+from meta_automl.data_preparation.model import Model
+from meta_automl.meta_algorithm.datasets_similarity_assessors import DatasetsSimilarityAssessor
+
+
+class ModelAdvisor:
+
+    @abstractmethod
+    def predict(self, *args, **kwargs) -> List[List[Model]]:
+        raise NotImplementedError()
+
+
+class SimpleSimilarityModelAdvisor(ModelAdvisor):
+    def __init__(self, fitted_similarity_assessor: DatasetsSimilarityAssessor):
+        self.similarity_assessor = fitted_similarity_assessor
+        self.best_models: Dict[str, List[Model]] = {}
+
+    @property
+    def datasets(self):
+        return self.similarity_assessor.datasets
+
+    def fit(self, dataset_names_to_best_pipelines: Dict[str, List[Model]]):
+        self.best_models.update(dataset_names_to_best_pipelines)
+        return self
+
+    def predict(self, meta_features: pd.DataFrame) -> List[List[Model]]:
+        assessor_predictions = self.similarity_assessor.predict(meta_features)
+        advised_pipelines = []
+        for similar_datasets in assessor_predictions:
+            advised_pipelines.append(self._predict_single(similar_datasets))
+        return advised_pipelines
+
+    def _predict_single(self, similar_dataset_names: Iterable[str]) -> List[Model]:
+        dataset_pipelines = []
+        for dataset_name in similar_dataset_names:
+            dataset_pipelines += self.best_models.get(dataset_name)
+        return dataset_pipelines


Test patch:
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/test/constants.py b/test/constants.py
new file mode 100644
index 00000000..2eea547c
--- /dev/null
+++ b/test/constants.py
@@ -0,0 +1,2 @@
+CACHED_DATASETS = ['australian', 'monks-problems-1']
+DATASETS_WITH_CACHED_META_FEATURES = ['australian', 'monks-problems-2']
diff --git a/test/data/datasets/australian.pkl b/test/data/datasets/australian.pkl
new file mode 100644
index 00000000..be8a9ae3
Binary files /dev/null and b/test/data/datasets/australian.pkl differ
diff --git a/test/data/datasets/monks-problems-1.pkl b/test/data/datasets/monks-problems-1.pkl
new file mode 100644
index 00000000..58061c48
Binary files /dev/null and b/test/data/datasets/monks-problems-1.pkl differ
diff --git a/test/data/pymfe/australian.pkl b/test/data/pymfe/australian.pkl
new file mode 100644
index 00000000..be778531
Binary files /dev/null and b/test/data/pymfe/australian.pkl differ
diff --git a/test/data/pymfe/monks-problems-2.pkl b/test/data/pymfe/monks-problems-2.pkl
new file mode 100644
index 00000000..eb3e4736
Binary files /dev/null and b/test/data/pymfe/monks-problems-2.pkl differ
diff --git a/test/data_manager.py b/test/data_manager.py
new file mode 100644
index 00000000..1bb78bc6
--- /dev/null
+++ b/test/data_manager.py
@@ -0,0 +1,9 @@
+from pathlib import Path
+
+from meta_automl.data_preparation.data_manager import DataManager
+
+
+class TestDataManager(DataManager):
+    @classmethod
+    def get_data_dir(cls) -> Path:
+        return cls.get_project_root().joinpath('test/data')
diff --git a/test/general_checks.py b/test/general_checks.py
new file mode 100644
index 00000000..a1d8610d
--- /dev/null
+++ b/test/general_checks.py
@@ -0,0 +1,25 @@
+from pathlib import Path
+from typing import Union
+
+from meta_automl.data_preparation.dataset import Dataset, DatasetCache
+from test.constants import CACHED_DATASETS
+from test.data_manager import TestDataManager
+
+
+def assert_file_unmodified_during_test(path: Path, test_start_timestamp: float):
+    assert path.stat().st_mtime < test_start_timestamp, f'The file should not be modified during the test: ' \
+                                                        f'"{path.relative_to(TestDataManager.get_project_root())}".'
+
+
+def assert_cache_file_exists(path: Path):
+    assert path.exists(), 'Cache not found at the path: ' \
+                          f'"{path.relative_to(TestDataManager.get_project_root())}".'
+
+
+def check_dataset_and_cache(dataset_or_cache: Union[Dataset, DatasetCache], desired_name: str, desired_path: Path,
+                            test_start_time: float):
+    assert dataset_or_cache.name == desired_name
+    assert dataset_or_cache.cache_path == desired_path
+    assert_cache_file_exists(desired_path)
+    if desired_name in CACHED_DATASETS:
+        assert_file_unmodified_during_test(desired_path, test_start_time)
diff --git a/test/unit/__init__.py b/test/unit/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/test/unit/test_dataset.py b/test/unit/test_dataset.py
new file mode 100644
index 00000000..3ac46d6d
--- /dev/null
+++ b/test/unit/test_dataset.py
@@ -0,0 +1,40 @@
+import numpy as np
+import pytest
+
+from meta_automl.data_preparation.dataset import DatasetCache, NoCacheError
+from test.constants import CACHED_DATASETS
+from test.data_manager import TestDataManager
+
+
+@pytest.fixture
+def dumped_cache_path():
+    path = TestDataManager.get_dataset_cache_path('data_dumped')
+    yield path
+    path.unlink()
+
+
+def test_dataset_caching(dumped_cache_path):
+    dataset_name = CACHED_DATASETS[0]
+
+    cache_path = TestDataManager.get_dataset_cache_path(dataset_name)
+
+    dataset_cache = DatasetCache(dataset_name, cache_path)
+    dataset = dataset_cache.from_cache()
+    dumped_cache = dataset.dump_to_cache(dumped_cache_path)
+    reloaded_dataset = dumped_cache.from_cache()
+    # Check data integrity.
+    assert dataset.name == dataset_name
+    assert reloaded_dataset.name == dataset_name
+    assert dataset.id == reloaded_dataset.id
+    assert np.all(np.equal(dataset.x, reloaded_dataset.x))
+    assert np.all(np.equal(dataset.y, reloaded_dataset.y))
+    # Check caching integrity.
+    assert dataset_cache.cache_path == cache_path
+    assert dataset.cache_path == cache_path
+    assert dumped_cache.cache_path == dumped_cache_path
+    assert reloaded_dataset.cache_path == dumped_cache_path
+
+
+def test_error_on_missing_dataset_cache():
+    with pytest.raises(NoCacheError):
+        DatasetCache('random_missing_cache').from_cache()
diff --git a/test/unit/test_datasets_loaders.py b/test/unit/test_datasets_loaders.py
new file mode 100644
index 00000000..1596e312
--- /dev/null
+++ b/test/unit/test_datasets_loaders.py
@@ -0,0 +1,50 @@
+import time
+
+import pytest
+
+from meta_automl.data_preparation.dataset import DatasetCache
+from meta_automl.data_preparation.datasets_loaders import OpenMLDatasetsLoader
+from test.general_checks import check_dataset_and_cache
+from test.constants import CACHED_DATASETS
+from test.data_manager import TestDataManager
+
+
+@pytest.fixture
+def dataset_names():
+    dataset_names = ['australian', 'blood-transfusion-service-center']
+    yield dataset_names
+    for dataset_name in dataset_names:
+        if dataset_name not in CACHED_DATASETS:
+            TestDataManager.get_dataset_cache_path(dataset_name).unlink(missing_ok=True)
+
+
+def test_group_load_new_datasets(dataset_names):
+    test_start_time = time.time()
+    loader = OpenMLDatasetsLoader()
+    loader.data_manager = TestDataManager
+
+    datasets = loader.load(dataset_names)
+
+    assert loader.dataset_sources == dataset_names
+
+    for dataset_name, dataset_cache in zip(dataset_names, datasets):
+        check_dataset_and_cache(dataset_cache, dataset_name, dataset_cache.cache_path, test_start_time)
+
+
+def test_load_single(dataset_names):
+    test_start_time = time.time()
+    loader = OpenMLDatasetsLoader()
+    loader.data_manager = TestDataManager
+    for dataset_name in dataset_names:
+        dataset_cache = loader.load_single(dataset_name)
+        check_dataset_and_cache(dataset_cache, dataset_name, dataset_cache.cache_path, test_start_time)
+
+
+def test_load_new_datasets_on_demand(dataset_names):
+    test_start_time = time.time()
+    loader = OpenMLDatasetsLoader()
+    loader.data_manager = TestDataManager
+    for dataset_name in dataset_names:
+        cache_path = TestDataManager.get_dataset_cache_path(dataset_name)
+        dataset = loader.cache_to_memory(DatasetCache(dataset_name, cache_path))
+        check_dataset_and_cache(dataset, dataset_name, cache_path, test_start_time)
diff --git a/test/unit/test_meta_features_extractors.py b/test/unit/test_meta_features_extractors.py
new file mode 100644
index 00000000..c5625f53
--- /dev/null
+++ b/test/unit/test_meta_features_extractors.py
@@ -0,0 +1,38 @@
+import time
+
+import pytest
+
+from meta_automl.data_preparation.meta_features_extractors import PymfeExtractor
+from test.general_checks import assert_file_unmodified_during_test, assert_cache_file_exists
+from test.data_manager import TestDataManager
+from test.constants import CACHED_DATASETS, DATASETS_WITH_CACHED_META_FEATURES
+
+
+@pytest.fixture
+def dataset_names():
+    dataset_names = ['australian', 'monks-problems-1', 'monks-problems-2', 'blood-transfusion-service-center']
+    yield dataset_names
+    for dataset_name in dataset_names:
+        if dataset_name not in CACHED_DATASETS + DATASETS_WITH_CACHED_META_FEATURES:
+            TestDataManager.get_dataset_cache_path(dataset_name).unlink(missing_ok=True)
+        if dataset_name not in DATASETS_WITH_CACHED_META_FEATURES:
+            TestDataManager.get_meta_features_cache_path(dataset_name, PymfeExtractor.SOURCE).unlink(missing_ok=True)
+
+
+def test_meta_features_extraction(dataset_names):
+    test_start_time = time.time()
+    extractor = PymfeExtractor(extractor_params={'groups': 'general'})
+    extractor.data_manager = TestDataManager
+    extractor.datasets_loader.data_manager = TestDataManager
+    meta_features = extractor.extract(dataset_names)
+    assert list(meta_features.index) == dataset_names
+    for dataset_name in dataset_names:
+        meta_features_cache_path = TestDataManager.get_meta_features_cache_path(
+            dataset_name, extractor.SOURCE)
+        assert_cache_file_exists(meta_features_cache_path)
+
+        if dataset_name in DATASETS_WITH_CACHED_META_FEATURES:
+            assert_file_unmodified_during_test(meta_features_cache_path, test_start_time)
+        else:
+            cache_path = TestDataManager.get_dataset_cache_path(dataset_name)
+            assert_cache_file_exists(cache_path)