Repository: serverless/serverless (JavaScript)
Date: 2019-11-19 05:42:09+00:00
Task type: Bug Fix

Issue:
Credentials not found error after upgrade 
# Bug Report

After upgrade from 1.44.1 to 1.55 we started seeing intermittent errors "AWS provider credentials not found", which is happening in about 15% of deploys. Nothing else is changed. Problem goes away with sls downgrade back to 1.44.

Apparently there was no validation for creds in 1.44:

https://github.com/serverless/serverless/blob/v1.44.1/lib/plugins/aws/lib/validate.js

which introduced later: 

https://github.com/serverless/serverless/blob/d2e75e19cadc0ffaf31434835cccb872dc476a5f/lib/plugins/aws/lib/validate.js

## Description

Debug output:

```

> abc@1.0.0 deploy /workdir/services/abc
--
  | > serverless deploy --force --stage=dev

Serverless: Load command interactiveCli
--
  | Serverless: Load command config
  | Serverless: Load command config:credentials
  | Serverless: Load command config:tabcompletion
  | Serverless: Load command config:tabcompletion:install
  | Serverless: Load command config:tabcompletion:uninstall
  | Serverless: Load command create
  | Serverless: Load command install
  | Serverless: Load command package
  | Serverless: Load command deploy
  | Serverless: Load command deploy:function
  | Serverless: Load command deploy:list
  | Serverless: Load command deploy:list:functions
  | Serverless: Load command invoke
  | Serverless: Load command invoke:local
  | Serverless: Load command info
  | Serverless: Load command logs
  | Serverless: Load command metrics
  | Serverless: Load command print
  | Serverless: Load command remove
  | Serverless: Load command rollback
  | Serverless: Load command rollback:function
  | Serverless: Load command slstats
  | Serverless: Load command plugin
  | Serverless: Load command plugin
  | Serverless: Load command plugin:install
  | Serverless: Load command plugin
  | Serverless: Load command plugin:uninstall
  | Serverless: Load command plugin
  | Serverless: Load command plugin:list
  | Serverless: Load command plugin
  | Serverless: Load command plugin:search
  | Serverless: Load command config
  | Serverless: Load command config:credentials
  | Serverless: Load command rollback
  | Serverless: Load command rollback:function
  | Serverless: Load command login
  | Serverless: Load command logout
  | Serverless: Load command generate-event
  | Serverless: Load command test
  | Serverless: Load command dashboard
  | Serverless: Invoke deploy
  | Serverless: Invoke package
  | Serverless: Invoke aws:common:validate
  | Serverless: Invoke aws:common:cleanupTempDir
  | Serverless: Packaging service...
  | Serverless: Excluding development dependencies...
  | Serverless: Invoke aws:package:finalize
  | Serverless: Invoke aws:common:moveArtifactsToPackage
  | Serverless: Invoke aws:common:validate
  |  
  | Serverless Error ---------------------------------------
  |  
  | ServerlessError: AWS provider credentials not found. Learn how to set up AWS provider credentials in our docs here: <http://slss.io/aws-creds-setup>.
  | at BbPromise.catch.then.identity (/workdir/node_modules/serverless/lib/plugins/aws/lib/validate.js:56:19)
  | From previous event:
  | at AwsCommon.validate (/workdir/node_modules/serverless/lib/plugins/aws/lib/validate.js:48:10)
  | From previous event:
  | at Object.aws:common:validate:validate [as hook] (/workdir/node_modules/serverless/lib/plugins/aws/common/index.js:49:66)
  | at BbPromise.reduce (/workdir/node_modules/serverless/lib/classes/PluginManager.js:489:55)
  | From previous event:
  | at PluginManager.invoke (/workdir/node_modules/serverless/lib/classes/PluginManager.js:489:22)
  | at PluginManager.spawn (/workdir/node_modules/serverless/lib/classes/PluginManager.js:509:17)
  | at AwsDeploy.BbPromise.bind.then (/workdir/node_modules/serverless/lib/plugins/aws/deploy/index.js:69:53)
  | From previous event:
  | at Object.before:deploy:deploy [as hook] (/workdir/node_modules/serverless/lib/plugins/aws/deploy/index.js:69:12)
  | at BbPromise.reduce (/workdir/node_modules/serverless/lib/classes/PluginManager.js:489:55)
  | From previous event:
  | at PluginManager.invoke (/workdir/node_modules/serverless/lib/classes/PluginManager.js:489:22)
  | at getHooks.reduce.then (/workdir/node_modules/serverless/lib/classes/PluginManager.js:524:24)
  | From previous event:
  | at PluginManager.run (/workdir/node_modules/serverless/lib/classes/PluginManager.js:524:8)
  | at variables.populateService.then (/workdir/node_modules/serverless/lib/Serverless.js:115:33)
  | at runCallback (timers.js:705:18)
  | at tryOnImmediate (timers.js:676:5)
  | at processImmediate (timers.js:658:5)
  | at process.topLevelDomainCallback (domain.js:120:23)
  | From previous event:
  | at Serverless.run (/workdir/node_modules/serverless/lib/Serverless.js:102:74)
  | at serverless.init.then (/workdir/node_modules/serverless/bin/serverless.js:72:30)
  | at /workdir/node_modules/serverless/node_modules/graceful-fs/graceful-fs.js:111:16
  | at /workdir/node_modules/serverless/node_modules/graceful-fs/graceful-fs.js:45:10
  | at FSReqWrap.args [as oncomplete] (fs.js:140:20)
  | From previous event:
  | at initializeErrorReporter.then (/workdir/node_modules/serverless/bin/serverless.js:72:8)
  | at runCallback (timers.js:705:18)
  | at tryOnImmediate (timers.js:676:5)
  | at processImmediate (timers.js:658:5)
  | at process.topLevelDomainCallback (domain.js:120:23)
  | From previous event:
  | at Object.<anonymous> (/workdir/node_modules/serverless/bin/serverless.js:61:4)
  | at Module._compile (internal/modules/cjs/loader.js:701:30)
  | at Object.Module._extensions..js (internal/modules/cjs/loader.js:712:10)
  | at Module.load (internal/modules/cjs/loader.js:600:32)
  | at tryModuleLoad (internal/modules/cjs/loader.js:539:12)
  | at Function.Module._load (internal/modules/cjs/loader.js:531:3)
  | at Function.Module.runMain (internal/modules/cjs/loader.js:754:12)
  | at startup (internal/bootstrap/node.js:283:19)
  | at bootstrapNodeJSCore (internal/bootstrap/node.js:622:3)
  |  
  | Get Support --------------------------------------------
  | Docs:          docs.serverless.com
  | Bugs:          github.com/serverless/serverless/issues
  | Issues:        forum.serverless.com
  |  
  | Your Environment Information ---------------------------
  | Operating System:          linux
  | Node Version:              10.15.3
  | Framework Version:         1.57.0
  | Plugin Version:            3.2.2
  | SDK Version:               2.2.1
  | Components Core Version:   1.1.2
  | Components CLI Version:    1.4.0
  |  

```


Issue discussion:
@runk great thanks for report!

What was added with v1.48.0 was to fast fail when credentials are not available (without that, process tend to hang for a long time).

It'll be good to understand why it fails for you. How do you have an AWS authentication setup?

Maybe it's a result of some race condition, that at time of process intialization AWS creds are not ready, while they're ready shortly afterwards so that why it works for you with earlier versions (?)
Hey @medikoo, I think this issue could be related with the `MetadataService` validation approach, as it is not guaranteed that the AWS API Latency will be always 0.1 seconds or less.

The potential solutions that I can think of are:

1. Increase the timeout, however it is not guaranteed that will eliminate the issue.
2. Add a new input parameter if deploying is inside of an EC2 machine. (so, the validation could fail credentials or the input parameter is not provided.)

What do you think about it?
@medikoo we're deploying from inside ec2 machine
@pauloprestes great thanks for hint.

> Add a new input parameter if deploying is inside of an EC2 machine. 

What do you mean exactly by _new input parameter_? New CLI option? 

> (so, the validation could fail credentials or the input parameter is not provided.)

What do you mean exactly? You mean to bypass validation if mentined above input parameter is provided?

@medikoo i meant a new cli option and to bypass the validation.
I've added a PR to explain better what i mean.

Solution patch:
diff --git a/lib/plugins/aws/lib/validate.js b/lib/plugins/aws/lib/validate.js
index 77dad1abfc6..77e3f758b49 100644
--- a/lib/plugins/aws/lib/validate.js
+++ b/lib/plugins/aws/lib/validate.js
@@ -35,27 +35,25 @@ module.exports = {
       !process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI
     ) {
       // first check if the EC2 Metadata Service has creds before throwing error
-      const metadataService = new this.provider.sdk.MetadataService({
-        httpOptions: { timeout: 100, connectTimeout: 100 }, // .1 second timeout
+      const ec2Credentials = new this.provider.sdk.EC2MetadataCredentials({
+        httpOptions: { timeout: 5000 }, // 5 second timeout
         maxRetries: 0, // retry 0 times
       });
       return new BbPromise((resolve, reject) =>
-        metadataService.request('/', (err, data) => {
-          return err ? reject(err) : resolve(data);
-        })
-      )
-        .catch(() => null)
-        .then(identity => {
-          if (!identity) {
+        ec2Credentials.load(err => {
+          if (err) {
             const message = [
               'AWS provider credentials not found.',
               ' Learn how to set up AWS provider credentials',
               ` in our docs here: <${chalk.green('http://slss.io/aws-creds-setup')}>.`,
             ].join('');
             userStats.track('user_awsCredentialsNotFound');
-            throw new this.serverless.classes.Error(message);
+            reject(new this.serverless.classes.Error(message));
+          } else {
+            resolve({});
           }
-        });
+        })
+      );
     }
     return BbPromise.resolve();
   },


Test patch:
diff --git a/lib/plugins/aws/lib/validate.test.js b/lib/plugins/aws/lib/validate.test.js
index 0a0a5e2aeea..652ee0b4a5d 100644
--- a/lib/plugins/aws/lib/validate.test.js
+++ b/lib/plugins/aws/lib/validate.test.js
@@ -9,9 +9,13 @@ chai.use(require('chai-as-promised'));
 
 const expect = chai.expect;
 
-class MetadataService {
-  request(error) {
-    error('error');
+let loadCredentialsMock = callback => {
+  callback('error');
+};
+
+class EC2MetadataCredentials {
+  load(callback) {
+    loadCredentialsMock(callback);
   }
 }
 
@@ -28,7 +32,7 @@ describe('#validate', () => {
     provider = new AwsProvider(serverless, awsPlugin.options);
     provider.cachedCredentials = { accessKeyId: 'foo', secretAccessKey: 'bar' };
     awsPlugin.provider = provider;
-    awsPlugin.provider.sdk = { MetadataService };
+    awsPlugin.provider.sdk = { EC2MetadataCredentials };
     awsPlugin.serverless = serverless;
     awsPlugin.serverless.setProvider('aws', provider);
 
@@ -97,6 +101,20 @@ describe('#validate', () => {
       });
     });
 
+    it('should check the metadata service and pass if return credentials', () => {
+      awsPlugin.options.region = false;
+      awsPlugin.serverless.service.provider = {
+        region: 'some-region',
+      };
+      provider.cachedCredentials = {};
+
+      loadCredentialsMock = callback => callback(null);
+
+      return expect(awsPlugin.validate()).to.be.fulfilled.then(() => {
+        expect(awsPlugin.options.region).to.equal('some-region');
+      });
+    });
+
     it('should not check the metadata service if not using a command that needs creds', () => {
       awsPlugin.options.region = false;
       awsPlugin.serverless.service.provider = {