Repository: godaddy/kubernetes-external-secrets
Pull request #488 (2020-09-19 14:49:14)

Cache Vault clients/tokens on a per-role&mountpoint basis.
Fixes https://github.com/godaddy/kubernetes-external-secrets/issues/487

When you have a single k8s cluster with multiple namespaces and each `ExternalSecret` is defined with a very limited scoped `vaultRole` (limiting the vault data that role can read) the existing token caching causes issues.

This change caches each `vaultClient` against the `vaultRole` it's authenticated against and can therefore KES can read all the secrets it is meant to regardless of how many `vaultRoles` are defined across your ExternalSecrets

pault28: Lovely. This will solve the issue I am currently facing. Guessing other folks would want this as it follows vault policy per role.
Flydiverny: Some conflicts after #472 merge, mind resolving?
megakid: I can't explain why the E2E tests are failing here whilst the unit tests are passing.  
Flydiverny: Yeah don't think E2E tests are related to this PR, seems to be failing everywhere but I cant really tell why <_<
megakid: Should I just tidy up the commits (rebase) in preparation for merge or do the e2e tests need sorting first?
Flydiverny: I dont have the rights to merge without the E2E tests passing anyway so :) and all merges are squash as well. But I'll try pinging someone on slack.
E2E test ran fine locally so 🤷 
Flydiverny: E2E tests have been restored and should be working after a rebase :) 
megakid: All good now 👍 

Patch:
diff --git a/charts/kubernetes-external-secrets/README.md b/charts/kubernetes-external-secrets/README.md
index c790c45..372b0c8 100644
--- a/charts/kubernetes-external-secrets/README.md
+++ b/charts/kubernetes-external-secrets/README.md
@@ -74,7 +74,10 @@ The following table lists the configurable parameters of the `kubernetes-externa
 | `crds.create`                             | For Helm V2 installations of the chart to install the CRD, for V3 installations use `--skip-crds` appropriately                   | `false`                               |
 | `customResourceManagerDisabled`           | Disables the custom resource manager, requiring the CRD be installed via the chart or other means                                 | `false`                               |
 | `env.AWS_REGION`                          | Set AWS_REGION in Deployment Pod                                                                                                  | `us-west-2`                           |
+| `env.AWS_INTERMEDIATE_ROLE_ARN`           | Specifies a role to be assumed before assuming role arn specified in external secrets                                             |                                       |
 | `env.LOG_LEVEL`                           | Set the application log level                                                                                                     | `info`                                |
+| `env.LOG_MESSAGE_KEY`                     | Set the key for log messages log text, for example when running on GCP it might be nice to set to `message`                       | `msg`                                 |
+| `env.USE_HUMAN_READABLE_LOG_LEVELS`       | Sets log levels as string instead of ints eg `info` instead of `30`, setting this to any value will switch                        | `nil`                                 |
 | `env.METRICS_PORT`                        | Specify the port for the prometheus metrics server                                                                                | `3001`                                |
 | `env.ROLE_PERMITTED_ANNOTATION`           | Specify the annotation key where to lookup the role arn permission boundaries                                                     | `iam.amazonaws.com/permitted`         |
 | `env.POLLER_INTERVAL_MILLISECONDS`        | Set POLLER_INTERVAL_MILLISECONDS in Deployment Pod                                                                                | `10000`                               |
diff --git a/charts/kubernetes-external-secrets/values.yaml b/charts/kubernetes-external-secrets/values.yaml
index aa30728..8f8e981 100644
--- a/charts/kubernetes-external-secrets/values.yaml
+++ b/charts/kubernetes-external-secrets/values.yaml
@@ -21,7 +21,9 @@ env:
   # USE_HUMAN_READABLE_LOG_LEVELS: true
   METRICS_PORT: 3001
   VAULT_ADDR: http://127.0.0.1:8200
-#  GOOGLE_APPLICATION_CREDENTIALS: /app/gcp-creds/gcp-creds.json
+  # Set a role to be used when assuming roles specified in external secret (AWS only)
+  # AWS_INTERMEDIATE_ROLE_ARN:
+  # GOOGLE_APPLICATION_CREDENTIALS: /app/gcp-creds/gcp-creds.json
 
 # Create environment variables from existing k8s secrets
 # envVarsFromSecret:
diff --git a/config/aws-config.js b/config/aws-config.js
index df2a1b6..a3a75a3 100644
--- a/config/aws-config.js
+++ b/config/aws-config.js
@@ -7,6 +7,8 @@ const merge = require('lodash.merge')
 
 const localstack = process.env.LOCALSTACK || 0
 
+const intermediateRole = process.env.AWS_INTERMEDIATE_ROLE_ARN || 0
+
 let secretsManagerConfig = {}
 let systemManagerConfig = {}
 let stsConfig = {
@@ -29,6 +31,13 @@ if (localstack) {
   }
 }
 
+const intermediateCredentials = intermediateRole ? new AWS.ChainableTemporaryCredentials({
+  params: {
+    RoleArn: intermediateRole
+  },
+  stsConfig
+}) : null
+
 module.exports = {
   secretsManagerFactory: (opts = {}) => {
     if (localstack) {
@@ -43,15 +52,10 @@ module.exports = {
     return new AWS.SSM(opts)
   },
   assumeRole: (assumeRoleOpts) => {
-    const sts = new AWS.STS(stsConfig)
-
-    return new Promise((resolve, reject) => {
-      sts.assumeRole(assumeRoleOpts, (err, res) => {
-        if (err) {
-          return reject(err)
-        }
-        resolve(res)
-      })
+    return new AWS.ChainableTemporaryCredentials({
+      params: assumeRoleOpts,
+      masterCredentials: intermediateCredentials,
+      stsConfig
     })
   }
 }
diff --git a/config/index.js b/config/index.js
index 951e8a2..c9f5d7b 100644
--- a/config/index.js
+++ b/config/index.js
@@ -76,14 +76,15 @@ if (envConfig.vaultNamespace) {
     'X-VAULT-NAMESPACE': envConfig.vaultNamespace
   }
 }
-const vaultClient = vault(vaultOptions)
+const vaultFactory = () => vault(vaultOptions)
+
 // The Vault token is renewed only during polling, not asynchronously. The default tokenRenewThreshold
 // is three times larger than the pollerInterval so that the token is renewed before it
 // expires and with at least one remaining poll opportunty to retry renewal if it fails.
 const vaultTokenRenewThreshold = envConfig.vaultTokenRenewThreshold
   ? Number(envConfig.vaultTokenRenewThreshold) : 3 * envConfig.pollerIntervalMilliseconds / 1000
 const vaultBackend = new VaultBackend({
-  client: vaultClient,
+  vaultFactory: vaultFactory,
   tokenRenewThreshold: vaultTokenRenewThreshold,
   logger: logger,
   defaultVaultMountPoint: envConfig.defaultVaultMountPoint,
diff --git a/lib/backends/secrets-manager-backend.js b/lib/backends/secrets-manager-backend.js
index f7df15a..61f75a3 100644
--- a/lib/backends/secrets-manager-backend.js
+++ b/lib/backends/secrets-manager-backend.js
@@ -31,14 +31,12 @@ class SecretsManagerBackend extends KVBackend {
 
     let client = this._client
     if (roleArn) {
-      const res = await this._assumeRole({
+      const credentials = this._assumeRole({
         RoleArn: roleArn,
         RoleSessionName: 'k8s-external-secrets'
       })
       client = this._clientFactory({
-        accessKeyId: res.Credentials.AccessKeyId,
-        secretAccessKey: res.Credentials.SecretAccessKey,
-        sessionToken: res.Credentials.SessionToken
+        credentials
       })
     }
 
diff --git a/lib/backends/system-manager-backend.js b/lib/backends/system-manager-backend.js
index 79300b4..778cb58 100644
--- a/lib/backends/system-manager-backend.js
+++ b/lib/backends/system-manager-backend.js
@@ -29,14 +29,12 @@ class SystemManagerBackend extends KVBackend {
 
     let client = this._client
     if (roleArn) {
-      const res = await this._assumeRole({
+      const credentials = this._assumeRole({
         RoleArn: roleArn,
         RoleSessionName: 'k8s-external-secrets'
       })
       client = this._clientFactory({
-        accessKeyId: res.Credentials.AccessKeyId,
-        secretAccessKey: res.Credentials.SecretAccessKey,
-        sessionToken: res.Credentials.SessionToken
+        credentials
       })
     }
     try {
diff --git a/lib/backends/vault-backend.js b/lib/backends/vault-backend.js
index 3ebedf3..4674bc3 100644
--- a/lib/backends/vault-backend.js
+++ b/lib/backends/vault-backend.js
@@ -6,16 +6,17 @@ const KVBackend = require('./kv-backend')
 class VaultBackend extends KVBackend {
   /**
    * Create Vault backend.
-   * @param {Object} client - Client for interacting with Vault.
+   * @param {Object} vaultFactory - arrow function to create a vault client.
    * @param {Number} tokenRenewThreshold - tokens are renewed when ttl reaches this threshold
    * @param {Object} logger - Logger for logging stuff.
    */
-  constructor ({ client, tokenRenewThreshold, logger, defaultVaultMountPoint, defaultVaultRole }) {
+  constructor ({ vaultFactory, tokenRenewThreshold, logger, defaultVaultMountPoint, defaultVaultRole }) {
     super({ logger })
-    this._client = client
+    this._vaultFactory = vaultFactory
+    this._clients = new Map()
     this._tokenRenewThreshold = tokenRenewThreshold
-    this.defaultVaultMountPoint = defaultVaultMountPoint
-    this.defaultVaultRole = defaultVaultRole
+    this._defaultVaultMountPoint = defaultVaultMountPoint
+    this._defaultVaultRole = defaultVaultRole
   }
 
   /**
@@ -41,27 +42,38 @@ class VaultBackend extends KVBackend {
    * @returns {Promise} Promise object representing secret property values.
    */
   async _get ({ key, specOptions: { vaultMountPoint = null, vaultRole = null, kvVersion = 2 } }) {
-    if (!this._client.token) {
+    const vaultMountPointGet = vaultMountPoint || this._defaultVaultMountPoint
+    const vaultRoleGet = vaultRole || this._defaultVaultRole
+    // Create cache key for auth specific client
+    const clientCacheKey = `|m${vaultMountPointGet}|r${vaultRoleGet}|`
+    // Lookup existing or create new vault client
+    let client = this._clients.get(clientCacheKey)
+    if (!client) {
+      client = this._vaultFactory()
+      this._clients.set(clientCacheKey, client)
+    }
+
+    if (!client.token) {
       const jwt = this._fetchServiceAccountToken()
-      this._logger.debug('fetching new token from vault')
-      await this._client.kubernetesLogin({
-        mount_point: vaultMountPoint || this.defaultVaultMountPoint,
-        role: vaultRole || this.defaultVaultRole,
+      this._logger.debug(`fetching new token from vault for role ${vaultRoleGet} on ${vaultMountPointGet}`)
+      await client.kubernetesLogin({
+        mount_point: vaultMountPointGet,
+        role: vaultRoleGet,
         jwt: jwt
       })
     } else {
-      this._logger.debug('checking vault token expiry')
-      const tokenStatus = await this._client.tokenLookupSelf()
-      this._logger.debug(`vault token valid for ${tokenStatus.data.ttl} seconds, renews at ${this._tokenRenewThreshold}`)
+      this._logger.debug(`checking vault token expiry for role ${vaultRoleGet} on ${vaultMountPointGet}`)
+      const tokenStatus = await client.tokenLookupSelf()
+      this._logger.debug(`vault token (role ${vaultRoleGet} on ${vaultMountPointGet}) valid for ${tokenStatus.data.ttl} seconds, renews at ${this._tokenRenewThreshold}`)
 
       if (Number(tokenStatus.data.ttl) <= this._tokenRenewThreshold) {
-        this._logger.debug('renewing vault token')
-        await this._client.tokenRenewSelf()
+        this._logger.debug(`renewing role ${vaultRoleGet} on ${vaultMountPointGet} vault token`)
+        await client.tokenRenewSelf()
       }
     }
 
     this._logger.debug(`reading secret key ${key} from vault`)
-    const secretResponse = await this._client.read(key)
+    const secretResponse = await client.read(key)
 
     if (kvVersion === 1) {
       return JSON.stringify(secretResponse.data)
diff --git a/package-lock.json b/package-lock.json
index c470c6e..de03b0b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -5137,9 +5137,9 @@
       }
     },
     "just-extend": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.0.tgz",
-      "integrity": "sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz",
+      "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==",
       "dev": true
     },
     "jwa": {
@@ -7216,9 +7216,9 @@
       "dev": true
     },
     "sinon": {
-      "version": "9.0.3",
-      "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz",
-      "integrity": "sha512-IKo9MIM111+smz9JGwLmw5U1075n1YXeAq8YeSFlndCLhAL5KGn6bLgu7b/4AYHTV/LcEMcRm2wU2YiL55/6Pg==",
+      "version": "9.1.0",
+      "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.1.0.tgz",
+      "integrity": "sha512-9zQShgaeylYH6qtsnNXlTvv0FGTTckuDfHBi+qhgj5PvW2r2WslHZpgc3uy3e/ZAoPkqaOASPi+juU6EdYRYxA==",
       "dev": true,
       "requires": {
         "@sinonjs/commons": "^1.7.2",
@@ -7230,12 +7230,6 @@
         "supports-color": "^7.1.0"
       },
       "dependencies": {
-        "diff": {
-          "version": "4.0.2",
-          "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
-          "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
-          "dev": true
-        },
         "has-flag": {
           "version": "4.0.0",
           "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -7243,9 +7237,9 @@
           "dev": true
         },
         "supports-color": {
-          "version": "7.1.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz",
-          "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==",
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
           "dev": true,
           "requires": {
             "has-flag": "^4.0.0"
diff --git a/package.json b/package.json
index 39e0846..040f88b 100644
--- a/package.json
+++ b/package.json
@@ -60,7 +60,7 @@
     "mocha": "8.1.3",
     "nodemon": "2.0.4",
     "nyc": "15.1.0",
-    "sinon": "9.0.3",
+    "sinon": "9.1.0",
     "standard-version": "9.0.0",
     "supertest": "5.0.0"
   },


Test patch:
diff --git a/lib/backends/secrets-manager-backend.test.js b/lib/backends/secrets-manager-backend.test.js
index feae9f5..2bd0279 100644
--- a/lib/backends/secrets-manager-backend.test.js
+++ b/lib/backends/secrets-manager-backend.test.js
@@ -16,11 +16,7 @@ describe('SecretsManagerBackend', () => {
   const keyOptions = {}
 
   const assumeRoleCredentials = {
-    Credentials: {
-      AccessKeyId: '1234',
-      SecretAccessKey: '3123123',
-      SessionToken: 'asdasdasdad'
-    }
+    fakeObject: 'Fake mock object'
   }
 
   beforeEach(() => {
@@ -28,7 +24,7 @@ describe('SecretsManagerBackend', () => {
     loggerMock = sinon.mock()
     loggerMock.info = sinon.stub()
     clientFactoryMock = sinon.fake.returns(clientMock)
-    assumeRoleMock = sinon.fake.returns(Promise.resolve(assumeRoleCredentials))
+    assumeRoleMock = sinon.fake.returns(assumeRoleCredentials)
     secretsManagerBackend = new SecretsManagerBackend({
       clientFactory: clientFactoryMock,
       assumeRole: assumeRoleMock,
@@ -97,9 +93,7 @@ describe('SecretsManagerBackend', () => {
       })
 
       expect(clientFactoryMock.lastArg).deep.equals({
-        accessKeyId: assumeRoleCredentials.Credentials.AccessKeyId,
-        secretAccessKey: assumeRoleCredentials.Credentials.SecretAccessKey,
-        sessionToken: assumeRoleCredentials.Credentials.SessionToken
+        credentials: assumeRoleCredentials
       })
       expect(clientMock.getSecretValue.calledWith({
         SecretId: 'fakeSecretKey',
@@ -107,9 +101,7 @@ describe('SecretsManagerBackend', () => {
       })).to.equal(true)
       expect(clientFactoryMock.getCall(0).args).deep.equals([])
       expect(clientFactoryMock.getCall(1).args).deep.equals([{
-        accessKeyId: assumeRoleCredentials.Credentials.AccessKeyId,
-        secretAccessKey: assumeRoleCredentials.Credentials.SecretAccessKey,
-        sessionToken: assumeRoleCredentials.Credentials.SessionToken
+        credentials: assumeRoleCredentials
       }])
       expect(assumeRoleMock.callCount).equals(1)
       expect(secretPropertyValue).equals('fakeAssumeRoleSecretValue')
diff --git a/lib/backends/system-manager-backend.test.js b/lib/backends/system-manager-backend.test.js
index 15f9da3..aa86b84 100644
--- a/lib/backends/system-manager-backend.test.js
+++ b/lib/backends/system-manager-backend.test.js
@@ -15,11 +15,7 @@ describe('SystemManagerBackend', () => {
   const specOptions = {}
 
   const assumeRoleCredentials = {
-    Credentials: {
-      AccessKeyId: '1234',
-      SecretAccessKey: '3123123',
-      SessionToken: 'asdasdasdad'
-    }
+    fakeObject: 'Fake mock object'
   }
 
   beforeEach(() => {
@@ -27,7 +23,7 @@ describe('SystemManagerBackend', () => {
     loggerMock = sinon.mock()
     loggerMock.info = sinon.stub()
     clientFactoryMock = sinon.fake.returns(clientMock)
-    assumeRoleMock = sinon.fake.returns(Promise.resolve(assumeRoleCredentials))
+    assumeRoleMock = sinon.fake.returns(assumeRoleCredentials)
 
     systemManagerBackend = new SystemManagerBackend({
       client: clientMock,
@@ -79,9 +75,7 @@ describe('SystemManagerBackend', () => {
         specOptions: { roleArn: 'my-role' }
       })
       expect(clientFactoryMock.lastArg).deep.equals({
-        accessKeyId: assumeRoleCredentials.Credentials.AccessKeyId,
-        secretAccessKey: assumeRoleCredentials.Credentials.SecretAccessKey,
-        sessionToken: assumeRoleCredentials.Credentials.SessionToken
+        credentials: assumeRoleCredentials
       })
       expect(clientMock.getParameter.calledWith({
         Name: 'fakeSecretKey',
@@ -89,9 +83,7 @@ describe('SystemManagerBackend', () => {
       })).to.equal(true)
       expect(clientFactoryMock.getCall(0).args).deep.equals([])
       expect(clientFactoryMock.getCall(1).args).deep.equals([{
-        accessKeyId: assumeRoleCredentials.Credentials.AccessKeyId,
-        secretAccessKey: assumeRoleCredentials.Credentials.SecretAccessKey,
-        sessionToken: assumeRoleCredentials.Credentials.SessionToken
+        credentials: assumeRoleCredentials
       }])
       expect(secretPropertyValue).equals('fakeAssumeRoleSecretValue')
     })
diff --git a/lib/backends/vault-backend.test.js b/lib/backends/vault-backend.test.js
index 52fe1c9..319fa7b 100644
--- a/lib/backends/vault-backend.test.js
+++ b/lib/backends/vault-backend.test.js
@@ -14,11 +14,14 @@ const logger = pino({
 
 describe('VaultBackend', () => {
   let clientMock
+  let clientMock2
   let vaultBackend
   const defaultFakeMountPoint = 'defaultFakeMountPoint'
   const defaultFakeRole = 'defaultFakeRole'
   const mountPoint = 'fakeMountPoint'
+  const mountPoint2 = 'fakeMountPoint2'
   const role = 'fakeRole'
+  const role2 = 'fakeRole2'
   const secretKey = 'fakeSecretKey'
   const secretValue = 'open, sesame'
   const secretData = { [secretKey]: secretValue }
@@ -50,9 +53,10 @@ describe('VaultBackend', () => {
 
   beforeEach(() => {
     clientMock = sinon.mock()
-
+    clientMock2 = sinon.mock()
+    let mock = 0
     vaultBackend = new VaultBackend({
-      client: clientMock,
+      vaultFactory: () => mock++ === 0 ? clientMock : clientMock2,
       tokenRenewThreshold: vaultTokenRenewThreshold,
       logger: logger,
       defaultVaultMountPoint: defaultFakeMountPoint,
@@ -63,6 +67,7 @@ describe('VaultBackend', () => {
   describe('_get', () => {
     beforeEach(() => {
       clientMock.read = sinon.stub().returns(kv2Secret)
+      clientMock.token = undefined
       clientMock.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultMustRenew)
       clientMock.tokenRenewSelf = sinon.stub().returns(true)
       clientMock.kubernetesLogin = sinon.stub().returns({
@@ -71,9 +76,17 @@ describe('VaultBackend', () => {
         }
       })
 
-      vaultBackend._fetchServiceAccountToken = sinon.stub().returns(jwt)
+      clientMock2.read = sinon.stub().returns(kv2Secret)
+      clientMock2.token = undefined
+      clientMock2.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultMustRenew)
+      clientMock2.tokenRenewSelf = sinon.stub().returns(true)
+      clientMock2.kubernetesLogin = sinon.stub().returns({
+        auth: {
+          client_token: '5678'
+        }
+      })
 
-      clientMock.token = undefined
+      vaultBackend._fetchServiceAccountToken = sinon.stub().returns(jwt)
     })
 
     it('logs in and returns secret property value - default', async () => {
@@ -227,6 +240,168 @@ describe('VaultBackend', () => {
       // ... and expect to get its proper value
       expect(secretPropertyValue).equals(quotedSecretValue)
     })
+
+    it('returns secret property value using client associated with role given in _get', async () => {
+      clientMock.read = sinon.stub().returns(kv2Secret)
+      clientMock2.read = sinon.stub().returns(kv2Secret)
+
+      const secretPropertyValue = await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role
+        },
+        key: secretKey
+      })
+
+      // First, we log into Vault with role 1...
+      sinon.assert.calledWith(clientMock.kubernetesLogin, {
+        mount_point: mountPoint,
+        role: role,
+        jwt: jwt
+      })
+
+      // ... then we fetch the secret ...
+      sinon.assert.calledWith(clientMock.read, secretKey)
+
+      // ... and expect to get its proper value
+      expect(secretPropertyValue).equals(quotedSecretValue)
+
+      const secretPropertyValue2 = await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role2
+        },
+        key: secretKey
+      })
+
+      // Now ensure we log into Vault with role 2...
+      sinon.assert.calledWith(clientMock2.kubernetesLogin, {
+        mount_point: mountPoint,
+        role: role2,
+        jwt: jwt
+      })
+
+      // ... then we fetch the secret ...
+      sinon.assert.calledWith(clientMock2.read, secretKey)
+
+      // ... and expect to get its proper value
+      expect(secretPropertyValue2).equals(quotedSecretValue)
+    })
+
+    it('returns secret property value using client associated with mountpoint given in _get', async () => {
+      clientMock.read = sinon.stub().returns(kv2Secret)
+      clientMock2.read = sinon.stub().returns(kv2Secret)
+
+      const secretPropertyValue = await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role
+        },
+        key: secretKey
+      })
+
+      // First, we log into Vault with mountpoint 1...
+      sinon.assert.calledWith(clientMock.kubernetesLogin, {
+        mount_point: mountPoint,
+        role: role,
+        jwt: jwt
+      })
+
+      // ... then we fetch the secret ...
+      sinon.assert.calledWith(clientMock.read, secretKey)
+
+      // ... and expect to get its proper value
+      expect(secretPropertyValue).equals(quotedSecretValue)
+
+      const secretPropertyValue2 = await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint2,
+          vaultRole: role
+        },
+        key: secretKey
+      })
+
+      // Now ensure we log into Vault with mountpoint 2...
+      sinon.assert.calledWith(clientMock2.kubernetesLogin, {
+        mount_point: mountPoint2,
+        role: role,
+        jwt: jwt
+      })
+
+      // ... then we fetch the secret ...
+      sinon.assert.calledWith(clientMock2.read, secretKey)
+
+      // ... and expect to get its proper value
+      expect(secretPropertyValue2).equals(quotedSecretValue)
+    })
+
+    it('ensure we cache the clients for each role', async () => {
+      clientMock.read = sinon.stub().returns(kv2Secret)
+      clientMock2.read = sinon.stub().returns(kv2Secret)
+
+      await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role
+        },
+        key: secretKey
+      })
+
+      // First, we log into Vault with role 1...
+      sinon.assert.calledWith(clientMock.kubernetesLogin, {
+        mount_point: mountPoint,
+        role: role,
+        jwt: jwt
+      })
+      sinon.assert.calledOnce(clientMock.read)
+
+      clientMock.token = 'an-existing-token'
+      clientMock.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultNoRenew)
+
+      // now we have active role 1 client, we now activate role 2 client
+
+      await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role2
+        },
+        key: secretKey
+      })
+
+      // we log into Vault with role 2...
+      sinon.assert.calledWith(clientMock2.kubernetesLogin, {
+        mount_point: mountPoint,
+        role: role2,
+        jwt: jwt
+      })
+      sinon.assert.calledOnce(clientMock2.read)
+
+      clientMock2.token = 'an-existing-token'
+      clientMock2.tokenLookupSelf = sinon.stub().returns(mockTokenLookupResultNoRenew)
+
+      // Back to role1
+      await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role
+        },
+        key: secretKey
+      })
+
+      sinon.assert.calledOnce(clientMock.kubernetesLogin)
+      sinon.assert.calledTwice(clientMock.read)
+
+      await vaultBackend._get({
+        specOptions: {
+          vaultMountPoint: mountPoint,
+          vaultRole: role2
+        },
+        key: secretKey
+      })
+
+      sinon.assert.calledOnce(clientMock2.kubernetesLogin)
+      sinon.assert.calledTwice(clientMock2.read)
+    })
   })
 
   describe('getSecretManifestData', () => {