Repository: aws/aws-sdk-js
Pull request #4049 (2022-03-09 01:35:22)

support static stability in instance metadata credential provider
Implement static stability strategy to support cases when IMDS is unable to refresh credentials.

##### Checklist
<!-- Remove items that do not apply. For completed items, change [ ] to [x]. -->

- [x] `npm run test` passes
- [x] `.d.ts` file is updated
- [x] changelog is added, `npm run add-change`
- [x] run `bundle exec rake docs:api` and inspect `doc/latest/index.html` if documentation is changed
- [x] run `npm run integration` if integration test is changed



Patch:
diff --git a/.changes/next-release/feature-EC2MetadataCredentials-f20878a4.json b/.changes/next-release/feature-EC2MetadataCredentials-f20878a4.json
new file mode 100644
index 000000000..40a0223b4
--- /dev/null
+++ b/.changes/next-release/feature-EC2MetadataCredentials-f20878a4.json
@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "EC2MetadataCredentials",
+  "description": "Allow EC2MetadataCredentials to extend the existing expiration when EC2 Metadata Service returns expired credentials or failure response."
+}
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 331ac6fe3..099da6c30 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -96,6 +96,18 @@ specific service, for example:
 AWS_REGION=[set region] node node_modules/cucumber/bin/cucumber.js --tag @s3
 ```
 
+Hardcoded region strings are not allowed in the SDK. To check the code for
+the presence of hardcoded region strings, run
+
+```
+node ./scripts/region-checker/index.js
+```
+
+from the project root directory. If this command shows any output, the PR will
+not be merged. Any lines that appear here must have the region string removed
+or be added to `scripts/region-checker/allowlist.js`. Lines should only be
+allowlisted if absolutely necessary.
+
 ### Testing the documentation changes:
 
 To test the documentation changes, please ensure you have the following requirements installed (Please make sure you have the latest version of Ruby):
diff --git a/lib/config-base.d.ts b/lib/config-base.d.ts
index 06e07e3c9..4f3afe562 100644
--- a/lib/config-base.d.ts
+++ b/lib/config-base.d.ts
@@ -83,6 +83,7 @@ export interface HTTPOptions {
 export interface Logger {
     write?: (chunk: any, encoding?: string, callback?: () => void) => void
     log?: (...messages: any[]) => void;
+    warn?: (...message: any[]) => void;
 }
 export interface ParamValidation {
     /**
diff --git a/lib/credentials/ec2_metadata_credentials.d.ts b/lib/credentials/ec2_metadata_credentials.d.ts
index 2925a386e..d94ebb684 100644
--- a/lib/credentials/ec2_metadata_credentials.d.ts
+++ b/lib/credentials/ec2_metadata_credentials.d.ts
@@ -1,21 +1,31 @@
 import {Credentials} from '../credentials';
+import {Logger} from '../config-base';
+
 export class EC2MetadataCredentials extends Credentials {
-		/**
-		 * Creates credentials from the metadata service on an EC2 instance.
-		 * @param {object} options - Override the default (1s) timeout period.
-		 */
-		constructor(options?: EC2MetadataCredentialsOptions);
-	}
-    interface EC2MetadataCredentialsOptions {
-        httpOptions?: {
-            /**
-             * Timeout in milliseconds.
-             */
-            timeout?: number
-            /**
-            * Connection timeout in milliseconds.
-            */
-           connectTimeout?: number
-        }
-        maxRetries?: number
+    /**
+     * Creates credentials from the metadata service on an EC2 instance.
+     * @param {object} options - Override the default (1s) timeout period.
+     */
+    constructor(options?: EC2MetadataCredentialsOptions);
+
+    /**
+     * The original expiration of the current credential. In case of AWS
+     * outage, the EC2 metadata will extend expiration of the existing
+     * credential.
+     */
+    originalExpiration?: Date;
+}
+interface EC2MetadataCredentialsOptions {
+    httpOptions?: {
+        /**
+         * Timeout in milliseconds.
+         */
+        timeout?: number
+        /**
+        * Connection timeout in milliseconds.
+        */
+        connectTimeout?: number
     }
+    maxRetries?: number
+    logger?: Logger
+}
diff --git a/lib/credentials/ec2_metadata_credentials.js b/lib/credentials/ec2_metadata_credentials.js
index 1f7f2e52f..2aa2f5e28 100644
--- a/lib/credentials/ec2_metadata_credentials.js
+++ b/lib/credentials/ec2_metadata_credentials.js
@@ -18,15 +18,27 @@ require('../metadata_service');
  * AWS.config.credentials = new AWS.EC2MetadataCredentials({
  *   httpOptions: { timeout: 5000 }, // 5 second timeout
  *   maxRetries: 10, // retry 10 times
- *   retryDelayOptions: { base: 200 } // see AWS.Config for information
+ *   retryDelayOptions: { base: 200 }, // see AWS.Config for information
+ *   logger: console // see AWS.Config for information
  * });
+ * ```
  *
  * If your requests are timing out in connecting to the metadata service, such
  * as when testing on a development machine, you can use the connectTimeout
  * option, specified in milliseconds, which also defaults to 1 second.
- * ```
+ *
+ * If the requests failed or returns expired credentials, it will
+ * extend the expiration of current credential, with a warning message. For more
+ * information, please go to:
+ * https://docs.aws.amazon.com/sdkref/latest/guide/feature-static-credentials.html
+ *
+ * @!attribute originalExpiration
+ *   @return [Date] The optional original expiration of the current credential.
+ *   In case of AWS outage, the EC2 metadata will extend expiration of the
+ *   existing credential.
  *
  * @see AWS.Config.retryDelayOptions
+ * @see AWS.Config.logger
  *
  * @!macro nobrowser
  */
@@ -44,7 +56,7 @@ AWS.EC2MetadataCredentials = AWS.util.inherit(AWS.Credentials, {
        options.httpOptions);
 
     this.metadataService = new AWS.MetadataService(options);
-    this.metadata = {};
+    this.logger = options.logger || AWS.config && AWS.config.logger;
   },
 
   /**
@@ -62,6 +74,13 @@ AWS.EC2MetadataCredentials = AWS.util.inherit(AWS.Credentials, {
    */
   defaultMaxRetries: 3,
 
+  /**
+   * The original expiration of the current credential. In case of AWS
+   * outage, the EC2 metadata will extend expiration of the existing
+   * credential.
+   */
+  originalExpiration: undefined,
+
   /**
    * Loads the credentials from the instance metadata service
    *
@@ -84,24 +103,62 @@ AWS.EC2MetadataCredentials = AWS.util.inherit(AWS.Credentials, {
   load: function load(callback) {
     var self = this;
     self.metadataService.loadCredentials(function(err, creds) {
-      if (!err) {
-        var currentTime = AWS.util.date.getDate();
-        var expireTime = new Date(creds.Expiration);
-        if (expireTime < currentTime) {
-          err = AWS.util.error(
-            new Error('EC2 Instance Metadata Serivce provided expired credentials'),
-            { code: 'EC2MetadataCredentialsProviderFailure' }
-          );
+      if (err) {
+        if (self.hasLoadedCredentials()) {
+          self.extendExpirationIfExpired();
+          callback();
         } else {
-          self.expired = false;
-          self.metadata = creds;
-          self.accessKeyId = creds.AccessKeyId;
-          self.secretAccessKey = creds.SecretAccessKey;
-          self.sessionToken = creds.Token;
-          self.expireTime = expireTime;
+          callback(err);
         }
+      } else {
+        self.setCredentials(creds);
+        self.extendExpirationIfExpired();
+        callback();
       }
-      callback(err);
     });
+  },
+
+  /**
+   * Whether this credential has been loaded.
+   * @api private
+   */
+  hasLoadedCredentials: function hasLoadedCredentials() {
+    return this.AccessKeyId && this.secretAccessKey;
+  },
+
+  /**
+   * if expired, extend the expiration by 15 minutes base plus a jitter of 5
+   * minutes range.
+   * @api private
+   */
+  extendExpirationIfExpired: function extendExpirationIfExpired() {
+    if (this.needsRefresh()) {
+      this.originalExpiration = this.originalExpiration || this.expireTime;
+      this.expired = false;
+      var nextTimeout = 15 * 60 + Math.floor(Math.random() * 5 * 60);
+      var currentTime = AWS.util.date.getDate().getTime();
+      this.expireTime = new Date(currentTime + nextTimeout * 1000);
+      // TODO: add doc link;
+      this.logger.warn('Attempting credential expiration extension due to a '
+          + 'credential service availability issue. A refresh of these '
+          + 'credentials will be attempted again at ' + this.expireTime
+          + '\nFor more information, please visit: https://docs.aws.amazon.com/sdkref/latest/guide/feature-static-credentials.html');
+    }
+  },
+
+  /**
+   * Update the credential with new credential responded from EC2 metadata
+   * service.
+   * @api private
+   */
+  setCredentials: function setCredentials(creds) {
+    var currentTime = AWS.util.date.getDate().getTime();
+    var expireTime = new Date(creds.Expiration);
+    this.expired = currentTime >= expireTime ? true : false;
+    this.metadata = creds;
+    this.accessKeyId = creds.AccessKeyId;
+    this.secretAccessKey = creds.SecretAccessKey;
+    this.sessionToken = creds.Token;
+    this.expireTime = expireTime;
   }
 });


Test patch:
diff --git a/test/.eslintrc b/test/.eslintrc
index 4063e3bde..5c1c35644 100644
--- a/test/.eslintrc
+++ b/test/.eslintrc
@@ -1,6 +1,6 @@
 {
     "extends": "../.eslintrc",
     "parserOptions": {
-        "ecmaVersion": 8
+        "ecmaVersion": 9
     }
 }
diff --git a/test/credentials.spec.js b/test/credentials.spec.js
index a3b9861a2..4cdb2ca68 100644
--- a/test/credentials.spec.js
+++ b/test/credentials.spec.js
@@ -1,3 +1,5 @@
+const exp = require('constants');
+
 // Generated by CoffeeScript 1.12.3
 (function() {
   var AWS, STS, helpers, validateCredentials;
@@ -1120,39 +1122,66 @@
         });
       });
     });
+
     describe('AWS.EC2MetadataCredentials', function() {
-      var creds, mockMetadataService;
-      creds = null;
+      let creds = null;
+      let warnSpy = null;
+
       beforeEach(function() {
-        return creds = new AWS.EC2MetadataCredentials({
-          host: 'host'
+        creds = new AWS.EC2MetadataCredentials({
+          host: 'host',
+          logger: console,
         });
+        warnSpy = helpers.spyOn(console, 'warn');
       });
-      mockMetadataService = function(expireTime) {
+
+      afterEach(function() {
+        AWS.config.update({ logger: null });
+      });
+
+      const expiredEc2MetadataResponse = {
+        Code: 'Success',
+        AccessKeyId: 'KEY',
+        SecretAccessKey: 'SECRET',
+        Token: 'TOKEN',
+        Expiration: new Date(0)
+      };
+
+      const ONE_HOUR = 60 * 60 * 1000;
+      const MINUTE = 60 * 1000;
+      const ec2MetadataResponse = {
+        Code: 'Success',
+        AccessKeyId: 'KEY',
+        SecretAccessKey: 'SECRET',
+        Token: 'TOKEN',
+        Expiration: new Date(Date.now() + ONE_HOUR).toISOString()
+      };
+
+      const mockMetadataService = function(paramList) {
+        let invocationCount = 0;
         return helpers.spyOn(creds.metadataService, 'loadCredentials').andCallFake(function(cb) {
-          return cb(null, {
-            Code: 'Success',
-            AccessKeyId: 'KEY',
-            SecretAccessKey: 'SECRET',
-            Token: 'TOKEN',
-            Expiration: expireTime.toISOString()
-          });
+          cb(paramList[invocationCount][0], paramList[invocationCount][1]);
+          invocationCount++;
         });
       };
+
       describe('constructor', function() {
         it('allows passing of AWS.MetadataService options', function() {
           return expect(creds.metadataService.endpoint).to.equal('http://host');
         });
+
         it('does not modify options object', function() {
           var opts;
           opts = {};
           creds = new AWS.EC2MetadataCredentials(opts);
           return expect(opts).to.eql({});
         });
+
         it('checking default timeout', function() {
           creds = new AWS.EC2MetadataCredentials({});
           return expect(creds.metadataService.httpOptions.timeout).to.equal(1000);
         });
+
         it('allows setting timeout', function() {
           var opts;
           opts = {
@@ -1163,11 +1192,13 @@
           creds = new AWS.EC2MetadataCredentials(opts);
           return expect(creds.metadataService.httpOptions.timeout).to.equal(5000);
         });
+
         it('checking default connectTimeout', function() {
           creds = new AWS.EC2MetadataCredentials({});
           return expect(creds.metadataService.httpOptions.connectTimeout).to.equal(1000);
         });
-        return it('allows setting connectTimeout', function() {
+
+        it('allows setting connectTimeout', function() {
           var opts;
           opts = {
             httpOptions: {
@@ -1177,19 +1208,47 @@
           creds = new AWS.EC2MetadataCredentials(opts);
           return expect(creds.metadataService.httpOptions.connectTimeout).to.equal(5000);
         });
+
+        it('allows setting logger', function() {
+          const logger = { warn: function () {} };
+          creds = new AWS.EC2MetadataCredentials({ logger });
+          expect(creds.logger).to.equal(logger);
+        });
+
+        it('defaults to AWS logger settings if logger is not set', function () {
+          const logger = { warn: function () {} };
+          AWS.config.update({ logger });
+          creds = new AWS.EC2MetadataCredentials();
+          expect(creds.logger).to.equal(logger);
+        });
       });
+
       describe('needsRefresh', function() {
-        return it('can be expired based on expire time from EC2 Metadata service', function(done) {
-          mockMetadataService(new Date(0));
+        it('should expire if valid credentials passed expiration', function(done) {
+          mockMetadataService([
+            [null, ec2MetadataResponse],
+          ]);
           creds.refresh(function () {
+            helpers.spyOn(AWS.util.date, 'getDate').andReturn(new Date(Date.now() + 2 * ONE_HOUR));
             expect(creds.needsRefresh()).to.equal(true);
             done();
           });
         });
+
+        it('should NOT expire when EC2 Metadata service returns expired credentials', function(done) {
+          mockMetadataService([
+            [null, expiredEc2MetadataResponse],
+          ]);
+          creds.refresh(function () {
+            expect(creds.needsRefresh()).to.equal(false);
+            done();
+          });
+        });
       });
-      describe('refresh', function() {
+
+      describe('fresh', function () {
         it('loads credentials from EC2 Metadata service', function(done) {
-          mockMetadataService(new Date(AWS.util.date.getDate().getTime() + 100000));
+          mockMetadataService([[null, ec2MetadataResponse]]);
           creds.refresh(function () {
             expect(creds.metadata.Code).to.equal('Success');
             expect(creds.accessKeyId).to.equal('KEY');
@@ -1199,33 +1258,100 @@
             done();
           });
         });
+
         it('does try to load creds second time if Metadata service failed', function() {
-          var spy;
-          spy = helpers.spyOn(creds.metadataService, 'loadCredentials').andCallFake(function(cb) {
-            return cb(new Error('INVALID SERVICE'));
+          const spy = helpers.spyOn(creds.metadataService, 'loadCredentials').andCallFake(function(cb) {
+            cb(new Error('INVALID SERVICE'));
           });
           creds.refresh(function(err) {
-            return expect(err.message).to.equal('INVALID SERVICE');
+            expect(err.message).to.equal('INVALID SERVICE');
           });
-          return creds.refresh(function() {
-            return creds.refresh(function() {
-              return creds.refresh(function() {
-                return expect(spy.calls.length).to.equal(4);
+          creds.refresh(function() {
+            creds.refresh(function() {
+              creds.refresh(function() {
+                expect(spy.calls.length).to.equal(4);
               });
             });
           });
         });
-        it('fails if the loaded credentials are expired', function (done) {
-          mockMetadataService(new Date(Date.now() - 1));
-          creds.refresh(function (err) {
-            expect(err).to.be.not.null;
-            expect(err.message).to.equal('EC2 Instance Metadata Serivce provided expired credentials');
-            expect(err.code).to.equal('EC2MetadataCredentialsProviderFailure');
+
+        it('should throw if EC2 Metadata service failed at 1st load', function (done) {
+          mockMetadataService([[new Error('Cannot load initial credentials'), null]]);
+          creds.refresh(function(err) {
+            expect(err.message).to.equal('Cannot load initial credentials');
             done();
           });
         });
+
+        it('should extend expiration if EC2 Metadata service returns expired credentials', function (done) {
+          helpers.spyOn(Math, 'random').andReturn(0.5);
+          helpers.spyOn(AWS.util.date, 'getDate').andCallThrough();
+          mockMetadataService([
+            [null, ec2MetadataResponse],
+            [null, { ...expiredEc2MetadataResponse, AccessKeyId: 'KEY2' }],
+            [null, { ...expiredEc2MetadataResponse, AccessKeyId: 'KEY3' }],
+            [null, { ...expiredEc2MetadataResponse, AccessKeyId: 'KEY4' }]
+          ]);
+          creds.refresh(function() {
+            // Loaded valid initial credentials
+            expect(creds.metadata.Code).to.equal('Success');
+            expect(creds.needsRefresh()).to.equal(false);
+            expect(warnSpy.calls.length).to.equal(0);
+            creds.refresh(function() {
+              // Extend expiration of credentials if expired credentials are loaded.
+              expect(creds.needsRefresh()).to.equal(false);
+              expect(creds.metadata.Code).to.equal('Success');
+              expect(creds.accessKeyId).to.equal('KEY2');
+              const timeTillExpiration = creds.expireTime.getTime() - AWS.util.date.getDate().getTime();
+              expect(timeTillExpiration).to.be.greaterThan(15 * MINUTE);
+              expect(timeTillExpiration).to.be.lessThan(20 * MINUTE);
+              expect(warnSpy.calls.length).to.equal(1);
+              expect(warnSpy.calls[0].arguments[0]).to.contain('Attempting credential expiration extension');
+              creds.refresh(function() {
+                // Refresh the credentials and print warning message if force refresh the credentials
+                expect(creds.needsRefresh()).to.equal(false);
+                expect(creds.accessKeyId).to.equal('KEY3');
+                expect(warnSpy.calls.length).to.equal(2);
+
+                // Print warning message if extended expiration is extended.
+                helpers.spyOn(AWS.util.date, 'getDate').andReturn(new Date(Date.now() + 2 * ONE_HOUR));
+                expect(creds.needsRefresh()).to.equal(true);
+                creds.refresh(function() {
+                  expect(creds.needsRefresh()).to.equal(false);
+                  expect(creds.accessKeyId).to.equal('KEY4');
+                  expect(warnSpy.calls.length).to.equal(3);
+                  done();
+                });
+              });
+            });
+          });
+        });
+
+        it('should extend expiration if EC2 Metadata service failed', function (done) {
+          mockMetadataService([
+            [null, ec2MetadataResponse],
+            [new Error('Metadata Service Error'), null]
+          ]);
+          creds.refresh(function () {
+            expect(creds.metadata.Code).to.equal('Success');
+            expect(creds.accessKeyId).to.equal('KEY');
+            expect(creds.needsRefresh()).to.equal(false);
+
+            helpers.spyOn(AWS.util.date, 'getDate').andReturn(new Date(Date.now() + 2 * ONE_HOUR));
+            expect(creds.needsRefresh()).to.equal(true);
+            creds.refresh(function() {
+              expect(creds.metadata.Code).to.equal('Success');
+              expect(creds.accessKeyId).to.equal('KEY');
+              expect(creds.needsRefresh()).to.equal(false);
+              expect(warnSpy.calls.length).to.equal(1);
+              expect(warnSpy.calls[0].arguments[0]).to.contain('SDK cannot renew the credential');
+            });
+          });
+          done();
+        });
       });
     });
+
     describe('AWS.RemoteCredentials', function() {
       var origEnv, creds, mockEndpoint, responseData, responseDataNew;
       creds = null;