Repository: tomplus/kubernetes_asyncio
Pull request #69 (2019-04-07 00:57:46)

feat: merging kubeconfig files
Add support for multiple config files.

ref: https://github.com/kubernetes-client/python-base/pull/94

codecov-io: # [Codecov](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69?src=pr&el=h1) Report
> Merging [#69](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69?src=pr&el=desc) into [master](https://codecov.io/gh/tomplus/kubernetes_asyncio/commit/959ca9fa4c25fa93c2ee2fd663440b66bad0d468?src=pr&el=desc) will **decrease** coverage by `0.44%`.
> The diff coverage is `85.41%`.

[![Impacted file tree graph](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69/graphs/tree.svg?width=650&token=0Dvm4fqopq&height=150&src=pr)](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master      #69      +/-   ##
==========================================
- Coverage   94.46%   94.02%   -0.45%     
==========================================
  Files          20       20              
  Lines        1374     1456      +82     
==========================================
+ Hits         1298     1369      +71     
- Misses         76       87      +11
```


| [Impacted Files](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [kubernetes\_asyncio/config/kube\_config\_test.py](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69/diff?src=pr&el=tree#diff-a3ViZXJuZXRlc19hc3luY2lvL2NvbmZpZy9rdWJlX2NvbmZpZ190ZXN0LnB5) | `92% <70.96%> (-1.77%)` | :arrow_down: |
| [kubernetes\_asyncio/config/kube\_config.py](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69/diff?src=pr&el=tree#diff-a3ViZXJuZXRlc19hc3luY2lvL2NvbmZpZy9rdWJlX2NvbmZpZy5weQ==) | `92.89% <92.3%> (+0.56%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69?src=pr&el=footer). Last update [959ca9f...e7c3bda](https://codecov.io/gh/tomplus/kubernetes_asyncio/pull/69?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/kubernetes_asyncio/config/kube_config.py b/kubernetes_asyncio/config/kube_config.py
index b5bc48c8..37d8207e 100644
--- a/kubernetes_asyncio/config/kube_config.py
+++ b/kubernetes_asyncio/config/kube_config.py
@@ -15,10 +15,12 @@
 import asyncio
 import atexit
 import base64
+import copy
 import datetime
 import json
 import logging
 import os
+import platform
 import tempfile
 
 import yaml
@@ -33,6 +35,7 @@ from .openid import OpenIDRequestor
 
 EXPIRY_SKEW_PREVENTION_DELAY = datetime.timedelta(minutes=5)
 KUBE_CONFIG_DEFAULT_LOCATION = os.environ.get('KUBECONFIG', '~/.kube/config')
+ENV_KUBECONFIG_PATH_SEPARATOR = ';' if platform.system() == 'Windows' else ':'
 PROVIDER_TYPE_OIDC = 'oidc'
 _temp_files = {}
 
@@ -128,7 +131,12 @@ class KubeConfigLoader(object):
                  get_google_credentials=None,
                  config_base_path="",
                  config_persister=None):
-        self._config = ConfigNode('kube-config', config_dict)
+
+        if isinstance(config_dict, ConfigNode):
+            self._config = config_dict
+        else:
+            self._config = ConfigNode('kube-config', config_dict)
+
         self._current_context = None
         self._user = None
         self._cluster = None
@@ -308,9 +316,10 @@ class KubeConfigLoader(object):
             logging.error(str(e))
 
     def _load_user_token(self):
+        base_path = self._get_base_path(self._user.path)
         token = FileOrData(
             self._user, 'tokenFile', 'token',
-            file_base_path=self._config_base_path,
+            file_base_path=base_path,
             base64_file_content=False).as_data()
         if token:
             self.token = "Bearer %s" % token
@@ -323,19 +332,27 @@ class KubeConfigLoader(object):
                 basic_auth.encode()).decode('utf-8')
             return True
 
+    def  _get_base_path(self, config_path):
+        if self._config_base_path is not None:
+            return self._config_base_path
+        if config_path is not None:
+            return os.path.abspath(os.path.dirname(config_path))
+        return ""
+
     def _load_cluster_info(self):
         if 'server' in self._cluster:
             self.host = self._cluster['server'].rstrip('/')
             if self.host.startswith("https"):
+                base_path = self._get_base_path(self._cluster.path)
                 self.ssl_ca_cert = FileOrData(
                     self._cluster, 'certificate-authority',
-                    file_base_path=self._config_base_path).as_file()
+                    file_base_path=base_path).as_file()
                 self.cert_file = FileOrData(
                     self._user, 'client-certificate',
-                    file_base_path=self._config_base_path).as_file()
+                    file_base_path=base_path).as_file()
                 self.key_file = FileOrData(
                     self._user, 'client-key',
-                    file_base_path=self._config_base_path).as_file()
+                    file_base_path=base_path).as_file()
         if 'insecure-skip-tls-verify' in self._cluster:
             self.verify_ssl = not self._cluster['insecure-skip-tls-verify']
 
@@ -368,9 +385,10 @@ class ConfigNode(object):
     message in case of missing keys. The assumption is all access keys are
     present in a well-formed kube-config."""
 
-    def __init__(self, name, value):
+    def __init__(self, name, value, path=None):
         self.name = name
         self.value = value
+        self.path = path
 
     def __contains__(self, key):
         return key in self.value
@@ -390,7 +408,7 @@ class ConfigNode(object):
                 'Invalid kube-config file. Expected key %s in %s'
                 % (key, self.name))
         if isinstance(v, dict) or isinstance(v, list):
-            return ConfigNode('%s/%s' % (self.name, key), v)
+            return ConfigNode('%s/%s' % (self.name, key), v, self.path)
         else:
             return v
 
@@ -415,7 +433,12 @@ class ConfigNode(object):
                         'Expected only one object with name %s in %s list'
                         % (name, self.name))
         if result is not None:
-            return ConfigNode('%s[name=%s]' % (self.name, name), result)
+            if isinstance(result, ConfigNode):
+                return result
+            else:
+                return ConfigNode(
+                    '%s[name=%s]' %
+                    (self.name, name), result, self.path)
         if safe:
             return None
         raise ConfigException(
@@ -423,12 +446,79 @@ class ConfigNode(object):
             'Expected object with name %s in %s list' % (name, self.name))
 
 
-def _get_kube_config_loader_for_yaml_file(filename, **kwargs):
-    with open(filename) as f:
-        return KubeConfigLoader(
-            config_dict=yaml.safe_load(f),
-            config_base_path=os.path.abspath(os.path.dirname(filename)),
-            **kwargs)
+class KubeConfigMerger:
+
+    """Reads and merges configuration from one or more kube-config's.
+    The propery `config` can be passed to the KubeConfigLoader as config_dict.
+    It uses a path attribute from ConfigNode to store the path to kubeconfig.
+    This path is required to load certs from relative paths.
+    A method `save_changes` updates changed kubeconfig's (it compares current
+    state of dicts with).
+    """
+
+    def __init__(self, paths):
+        self.paths = []
+        self.config_files = {}
+        self.config_merged = None
+
+        for path in paths.split(ENV_KUBECONFIG_PATH_SEPARATOR):
+            if path:
+                path = os.path.expanduser(path)
+                if os.path.exists(path):
+                    self.paths.append(path)
+                    self.load_config(path)
+        self.config_saved = copy.deepcopy(self.config_files)
+
+    @property
+    def config(self):
+        return self.config_merged
+
+    def load_config(self, path):
+        with open(path) as f:
+            config = yaml.safe_load(f)
+
+        if self.config_merged is None:
+            config_merged = copy.deepcopy(config)
+            for item in ('clusters', 'contexts', 'users'):
+                config_merged[item] = []
+            self.config_merged = ConfigNode(path, config_merged, path)
+
+        for item in ('clusters', 'contexts', 'users'):
+            self._merge(item, config[item], path)
+        self.config_files[path] = config
+
+    def _merge(self, item, add_cfg, path):
+        for new_item in add_cfg:
+            for exists in self.config_merged.value[item]:
+                if exists['name'] == new_item['name']:
+                    break
+            else:
+                self.config_merged.value[item].append(ConfigNode(
+                    '{}/{}'.format(path, new_item), new_item, path))
+
+    def save_changes(self):
+        for path in self.paths:
+            if self.config_saved[path] != self.config_files[path]:
+                self.save_config(path)
+        self.config_saved = copy.deepcopy(self.config_files)
+
+    def save_config(self, path):
+        with open(path, 'w') as f:
+            yaml.safe_dump(self.config_files[path], f,
+                           default_flow_style=False)
+
+
+def _get_kube_config_loader_for_yaml_file(
+        filename, persist_config=False, **kwargs):
+
+    kcfg = KubeConfigMerger(filename)
+    if persist_config and 'config_persister' not in kwargs:
+        kwargs['config_persister'] = kcfg.save_changes()
+
+    return KubeConfigLoader(
+        config_dict=kcfg.config,
+        config_base_path=None,
+        **kwargs)
 
 
 def list_kube_config_contexts(config_file=None):
@@ -456,18 +546,11 @@ async def load_kube_config(config_file=None, context=None,
     """
 
     if config_file is None:
-        config_file = os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION)
-
-    config_persister = None
-    if persist_config:
-        def _save_kube_config(config_map):
-            with open(config_file, 'w') as f:
-                yaml.safe_dump(config_map, f, default_flow_style=False)
-        config_persister = _save_kube_config
+        config_file = KUBE_CONFIG_DEFAULT_LOCATION
 
     loader = _get_kube_config_loader_for_yaml_file(
         config_file, active_context=context,
-        config_persister=config_persister)
+        persist_config=persist_config)
     if client_configuration is None:
         config = type.__call__(Configuration)
         await loader.load_and_set(config)


Test patch:
diff --git a/kubernetes_asyncio/config/kube_config_test.py b/kubernetes_asyncio/config/kube_config_test.py
index 36552657..163ecbd2 100644
--- a/kubernetes_asyncio/config/kube_config_test.py
+++ b/kubernetes_asyncio/config/kube_config_test.py
@@ -25,9 +25,11 @@ from six import PY3
 
 from .config_exception import ConfigException
 from .kube_config import (
-    ConfigNode, FileOrData, KubeConfigLoader, _cleanup_temp_files,
-    _create_temp_file_with_content, list_kube_config_contexts,
-    load_kube_config, new_client_from_config, refresh_token,
+    ENV_KUBECONFIG_PATH_SEPARATOR, ConfigNode,
+    FileOrData, KubeConfigLoader, KubeConfigMerger,
+    _cleanup_temp_files, _create_temp_file_with_content,
+    list_kube_config_contexts, load_kube_config,
+    new_client_from_config, refresh_token,
 )
 
 BEARER_TOKEN_FORMAT = "Bearer %s"
@@ -918,5 +920,167 @@ class TestKubeConfigLoader(BaseTestCase):
                          loader.token)
 
 
+class TestKubeConfigMerger(BaseTestCase):
+    TEST_KUBE_CONFIG_PART1 = {
+        "current-context": "no_user",
+        "contexts": [
+            {
+                "name": "no_user",
+                "context": {
+                    "cluster": "default"
+                }
+            },
+        ],
+        "clusters": [
+            {
+                "name": "default",
+                "cluster": {
+                    "server": TEST_HOST
+                }
+            },
+        ],
+        "users": []
+    }
+
+    TEST_KUBE_CONFIG_PART2 = {
+        "current-context": "",
+        "contexts": [
+            {
+                "name": "ssl",
+                "context": {
+                    "cluster": "ssl",
+                    "user": "ssl"
+                }
+            },
+            {
+                "name": "simple_token",
+                "context": {
+                    "cluster": "default",
+                    "user": "simple_token"
+                }
+            },
+        ],
+        "clusters": [
+            {
+                "name": "ssl",
+                "cluster": {
+                    "server": TEST_SSL_HOST,
+                    "certificate-authority-data":
+                        TEST_CERTIFICATE_AUTH_BASE64,
+                }
+            },
+        ],
+        "users": [
+            {
+                "name": "ssl",
+                "user": {
+                    "token": TEST_DATA_BASE64,
+                    "client-certificate-data": TEST_CLIENT_CERT_BASE64,
+                    "client-key-data": TEST_CLIENT_KEY_BASE64,
+                }
+            },
+        ]
+    }
+
+    TEST_KUBE_CONFIG_PART3 = {
+        "current-context": "no_user",
+        "contexts": [
+            {
+                "name": "expired_oidc",
+                "context": {
+                    "cluster": "default",
+                    "user": "expired_oidc"
+                }
+            },
+            {
+                "name": "ssl",
+                "context": {
+                    "cluster": "skipped-part2-defined-this-context",
+                    "user": "skipped"
+                }
+            },
+        ],
+        "clusters": [
+        ],
+        "users": [
+            {
+                "name": "expired_oidc",
+                "user": {
+                    "auth-provider": {
+                        "name": "oidc",
+                        "config": {
+                            "client-id": "tectonic-kubectl",
+                            "client-secret": "FAKE_SECRET",
+                            "id-token": TEST_OIDC_EXPIRED_LOGIN,
+                            "idp-certificate-authority-data": TEST_OIDC_CA,
+                            "idp-issuer-url": "https://example.org/identity",
+                            "refresh-token":
+                                "lucWJjEhlxZW01cXI3YmVlcYnpxNGhzk"
+                        }
+                    }
+                }
+            },
+            {
+                "name": "simple_token",
+                "user": {
+                    "token": TEST_DATA_BASE64,
+                    "username": TEST_USERNAME,  # should be ignored
+                    "password": TEST_PASSWORD,  # should be ignored
+                }
+            },
+        ]
+    }
+
+    def _create_multi_config(self):
+        files = []
+        for part in (
+                self.TEST_KUBE_CONFIG_PART1,
+                self.TEST_KUBE_CONFIG_PART2,
+                self.TEST_KUBE_CONFIG_PART3):
+            files.append(self._create_temp_file(yaml.safe_dump(part)))
+        return ENV_KUBECONFIG_PATH_SEPARATOR.join(files)
+
+    def test_list_kube_config_contexts(self):
+        kubeconfigs = self._create_multi_config()
+        expected_contexts = [
+            {'context': {'cluster': 'default'}, 'name': 'no_user'},
+            {'context': {'cluster': 'ssl', 'user': 'ssl'}, 'name': 'ssl'},
+            {'context': {'cluster': 'default', 'user': 'simple_token'},
+             'name': 'simple_token'},
+            {'context': {'cluster': 'default', 'user': 'expired_oidc'}, 'name': 'expired_oidc'}]
+
+        contexts, active_context = list_kube_config_contexts(
+            config_file=kubeconfigs)
+
+        self.assertEqual(contexts, expected_contexts)
+        self.assertEqual(active_context, expected_contexts[0])
+
+    async def test_new_client_from_config(self):
+        kubeconfigs = self._create_multi_config()
+        client = await new_client_from_config(
+            config_file=kubeconfigs, context="simple_token")
+        self.assertEqual(TEST_HOST, client.configuration.host)
+        self.assertEqual(BEARER_TOKEN_FORMAT % TEST_DATA_BASE64,
+                         client.configuration.api_key['authorization'])
+
+    def test_save_changes(self):
+        kubeconfigs = self._create_multi_config()
+
+        # load configuration, update token, save config
+        kconf = KubeConfigMerger(kubeconfigs)
+        user = kconf.config['users'].get_with_name('expired_oidc')['user']
+        provider = user['auth-provider']['config']
+        provider.value['id-token'] = "token-changed"
+        kconf.save_changes()
+
+        # re-read configuration
+        kconf = KubeConfigMerger(kubeconfigs)
+        user = kconf.config['users'].get_with_name('expired_oidc')['user']
+        provider = user['auth-provider']['config']
+
+        # new token
+        self.assertEqual(provider.value['id-token'], "token-changed")
+
+
 if __name__ == '__main__':
     main()