Repository: mindsdb/mindsdb

Issue:
Validity check and migrations for Config
**Is your feature request related to a problem? Please describe.**
After updating from 1.99.11 to 2.0.0 is not possible to run Mindsb without editing/removing existing config

**Describe the solution you'd like**
Check config version on start and apply some migrations.



Discussion:

Resolving patch:
diff --git a/mindsdb/__main__.py b/mindsdb/__main__.py
--- a/mindsdb/__main__.py
+++ b/mindsdb/__main__.py
@@ -5,7 +5,8 @@
 import os
 
 import torch.multiprocessing as mp
-from torch.multiprocessing import Process
+
+from mindsdb_native.config import CONFIG
 
 from mindsdb.utilities.config import Config
 from mindsdb.interfaces.native.mindsdb import MindsdbNative
@@ -14,6 +15,7 @@
 from mindsdb.utilities.fs import get_or_create_dir_struct
 from mindsdb.interfaces.database.database import DatabaseWrapper
 
+
 def close_api_gracefully(p_arr):
     for p in p_arr:
         sys.stdout.flush()
@@ -21,6 +23,7 @@ def close_api_gracefully(p_arr):
         p.join()
         sys.stdout.flush()
 
+
 if __name__ == '__main__':
     mp.freeze_support()
 
@@ -33,11 +36,13 @@ def close_api_gracefully(p_arr):
     config_path = args.config
     if config_path is None:
         config_dir, _, _ = get_or_create_dir_struct()
-        config_path = os.path.join(config_dir,'config.json')
+        config_path = os.path.join(config_dir, 'config.json')
 
     print(f'Using configuration file: {config_path}')
     config = Config(config_path)
 
+    CONFIG.MINDSDB_STORAGE_PATH = config['interface']['mindsdb_native']['storage_dir']
+
     if args.api is None:
         api_arr = [api for api in config['api']]
     else:
@@ -68,7 +73,7 @@ def close_api_gracefully(p_arr):
     for api in api_arr:
         print(f'Starting Mindsdb {api} API !')
         try:
-            p = ctx.Process(target=start_functions[api], args=(config_path,True,))
+            p = ctx.Process(target=start_functions[api], args=(config_path, True,))
             p.start()
             p_arr.append(p)
             print(f'Started Mindsdb {api} API !')
diff --git a/mindsdb/utilities/config.py b/mindsdb/utilities/config.py
--- a/mindsdb/utilities/config.py
+++ b/mindsdb/utilities/config.py
@@ -2,7 +2,9 @@
 import json
 import hashlib
 
+
 class Config(object):
+    current_version = '1.1'
     _config = {}
 
     def __init__(self, config_path):
@@ -16,13 +18,58 @@ def __init__(self, config_path):
         else:
             raise TypeError('Argument must be string representing a file path <Later on to be switched to file path and/or database connection info>')
 
+    def _migrate(self):
+        def m1_0(config):
+            if 'default_clickhouse' in config['integrations'] and 'type' not in config['integrations']['default_clickhouse']:
+                config['integrations']['default_clickhouse']['type'] = 'clickhouse'
+            if 'default_mariadb' in config['integrations'] and 'type' not in config['integrations']['default_mariadb']:
+                config['integrations']['default_mariadb']['type'] = 'mariadb'
+            if 'datasources' in config['api']['mysql']:
+                del config['api']['mysql']['datasources']
+            config['config_version'] = '1.1'
+            return config
+
+        migrations = {
+            '1.0': m1_0
+        }
+
+        current_version = self._parse_version(self._config['config_version'])
+        target_version = self._parse_version(self.current_version)
+        while current_version < target_version:
+            str_version = '.'.join([str(x) for x in current_version])
+            self._config = migrations[str_version](self._config)
+            current_version = self._parse_version(self._config['config_version'])
+
+    def _validate(self):
+        integrations = self._config.get('integrations', {})
+        for key in integrations:
+            if 'type' not in integrations[key]:
+                raise KeyError(f"Config error: for integration '{key}' key 'type' must be specified")
+
+    def _parse_version(self, version):
+        if isinstance(version, str):
+            version = [int(x) for x in version.split('.')]
+        elif isinstance(version, int):
+            version = [version]
+        if len(version) == 1:
+            version.append(0)
+        return version
+
     def _read(self):
         if isinstance(self.config_path, str) and os.path.isfile(self.config_path):
             with open(self.config_path, 'r') as fp:
                 self._config = json.load(fp)
+                if self._parse_version(self._config['config_version']) < self._parse_version(self.current_version):
+                    self._migrate()
+                    self._save()
+                self._validate()
         else:
             raise TypeError('`self.config_path` must be a string representing a local file path to a json config')
 
+    def _save(self):
+        with open(self.config_path, 'w') as fp:
+            json.dump(self._config, fp, indent=4, sort_keys=True)
+
     def _gen_hash(self):
         with open(self.config_path, 'rb') as fp:
             return hashlib.md5(fp.read()).hexdigest()
@@ -52,9 +99,9 @@ def set(self, key_chain, value, delete=False):
 
         c = self._config
         for i, k in enumerate(key_chain):
-            if k in c and i+1 < len(key_chain):
+            if k in c and i + 1 < len(key_chain):
                 c = c[k]
-            elif k not in c and i+1 < len(key_chain):
+            elif k not in c and i + 1 < len(key_chain):
                 c[k] = {}
                 c = c[k]
             else:
@@ -62,9 +109,7 @@ def set(self, key_chain, value, delete=False):
                     del c[k]
                 else:
                     c[k] = value
-
-        with open(self.config_path, 'w') as fp:
-            json.dump(self._config, fp, indent=4, sort_keys=True)
+        self._save()
 
     # Higher level interface
     def add_db_integration(self, name, dict):
diff --git a/mindsdb/utilities/wizards.py b/mindsdb/utilities/wizards.py
--- a/mindsdb/utilities/wizards.py
+++ b/mindsdb/utilities/wizards.py
@@ -21,7 +21,7 @@ def _in(ask, default, use_default):
 def auto_config(python_path,pip_path,predictor_dir,datasource_dir):
     config = {
         "debug": False
-        ,"config_version": 1
+        ,"config_version": "1.1"
         ,"python_interpreter": python_path
         ,"pip_path": pip_path
         ,"api": {
@@ -124,7 +124,6 @@ def cli_config(python_path,pip_path,predictor_dir,datasource_dir,config_dir,use_
                 "file_level": "INFO",
                 "console_level": "INFO"
             }
-            ,"datasources": []
         }
         config['api']['mysql']['host'] = _in('MYSQL interface host','127.0.0.1',use_default)
         config['api']['mysql']['port'] = _in('MYSQL interface port','47335',use_default)


Test patch:
diff --git a/tests/integration_tests/api/test_http.py b/tests/integration_tests/api/test_http.py
--- a/tests/integration_tests/api/test_http.py
+++ b/tests/integration_tests/api/test_http.py
@@ -48,14 +48,14 @@ def test_1_config(self):
         integration_names = res.json()
         assert set(integration_names['integrations']) == set(['default_mariadb', 'default_clickhouse'])
 
-        test_integration_data = {'enabled': False, 'host':'test'}
-        res = requests.put(f'{root}/config/integrations/test_integration', json={'params':test_integration_data})
+        test_integration_data = {'enabled': False, 'host': 'test', 'type': 'clickhouse'}
+        res = requests.put(f'{root}/config/integrations/test_integration', json={'params': test_integration_data})
         assert res.status_code == 200
 
         res = requests.get(f'{root}/config/integrations/test_integration')
         assert res.status_code == 200
         test_integration = res.json()
-        assert len(test_integration) == 2
+        assert len(test_integration) == 3
 
         res = requests.delete(f'{root}/config/integrations/test_integration')
         assert res.status_code == 200
diff --git a/tests/integration_tests/flows/common.py b/tests/integration_tests/flows/common.py
--- a/tests/integration_tests/flows/common.py
+++ b/tests/integration_tests/flows/common.py
@@ -78,7 +78,7 @@ def prepare_config(config, db):
 
     temp_config_path = str(TEMP_DIR.joinpath('config.json').resolve())
     with open(temp_config_path, 'wt') as f:
-        f.write(json.dumps(config._config))
+        json.dump(config._config, f, indent=4, sort_keys=True)
 
     return temp_config_path
 
diff --git a/tests/integration_tests/flows/config/config.json b/tests/integration_tests/flows/config/config.json
--- a/tests/integration_tests/flows/config/config.json
+++ b/tests/integration_tests/flows/config/config.json
@@ -6,7 +6,6 @@
         },
         "mysql": {
             "certificate_path": "tests/integration_tests/flows/config/cert.pem",
-            "datasources": [],
             "host": "127.0.0.1",
             "log": {
                 "console_level": "INFO",
@@ -20,7 +19,7 @@
             "user": "mindsdb"
         }
     },
-    "config_version": 1,
+    "config_version": "1.1",
     "debug": true,
     "integrations": {
         "default_clickhouse": {