Repository: serverless/serverless (JavaScript)
Date: 2018-05-17 09:32:47+00:00
Task type: Bug Fix

Issue:
sls deploy: endless loop on credential error in CI
# This is a Bug Report

## Description

I set up a `sls deploy` in CicleCI. As there are currently no credentials configured the build is supposed to fail.

Instead goes into an endless loop: `Serverless: Recoverable error occured (AWS provider credentials not found. Learn how to set up AWS provider credentials in our docs here: <http://bit.ly/aws-creds-setup>.), sleeping 5 seconds`

## Additional Data

* ***Serverless Framework Version you're using***: 1.27.2
* ***Operating System***: Linux, CircleCI default
* ***Stack Trace***: Nope
* ***Provider Error messages***: Serverless: Recoverable error occured (AWS provider credentials not found. Learn how to set up AWS provider credentials in our docs here: <http://bit.ly/aws-creds-setup>.), sleeping 5 seconds



Issue discussion:

Solution patch:
diff --git a/lib/plugins/aws/provider/awsProvider.js b/lib/plugins/aws/provider/awsProvider.js
index ee21dfd8b35..6d8aa97c962 100644
--- a/lib/plugins/aws/provider/awsProvider.js
+++ b/lib/plugins/aws/provider/awsProvider.js
@@ -199,22 +199,27 @@ class AwsProvider {
     const requestOptions = _.isObject(options) ? options : {};
     const shouldCache = _.get(requestOptions, 'useCache', false);
     const paramsHash = objectHash.sha1(params);
+    const MAX_TRIES = 4;
     const persistentRequest = (f) => new BbPromise((resolve, reject) => {
-      const doCall = () => {
+      const doCall = (numTry) => {
         f()
           // We're resembling if/else logic, therefore single `then` instead of `then`/`catch` pair
           .then(resolve, e => {
-            if (e.providerError && e.providerError.retryable) {
+            if (numTry < MAX_TRIES &&
+              ((e.providerError && e.providerError.retryable) || e.statusCode === 429)) {
               that.serverless.cli.log(
-                `Recoverable error occured (${e.message}), sleeping 5 seconds`
+                _.join([
+                  `Recoverable error occured (${e.message}), sleeping 5 seconds.`,
+                  `Try ${numTry + 1} of ${MAX_TRIES}`,
+                ], ' ')
               );
-              setTimeout(doCall, 5000);
+              setTimeout(doCall, 5000, numTry + 1);
             } else {
               reject(e);
             }
           });
       };
-      return doCall();
+      return doCall(0);
     });
 
     // Emit a warning for misuses of the old signature including stage and region
@@ -255,6 +260,11 @@ class AwsProvider {
           ].join('');
           message = errorMessage;
           userStats.track('user_awsCredentialsNotFound');
+          // We do not want to trigger the retry mechanism for credential errors
+          return BbPromise.reject(Object.assign(
+            new this.serverless.classes.Error(message, err.statusCode),
+            { providerError: _.assign({}, err, { retryable: false }) }
+          ));
         }
         return BbPromise.reject(Object.assign(
           new this.serverless.classes.Error(message, err.statusCode),


Test patch:
diff --git a/lib/plugins/aws/provider/awsProvider.test.js b/lib/plugins/aws/provider/awsProvider.test.js
index 9039455e501..b518cc28dc7 100644
--- a/lib/plugins/aws/provider/awsProvider.test.js
+++ b/lib/plugins/aws/provider/awsProvider.test.js
@@ -1,5 +1,7 @@
 'use strict';
 
+/* eslint-disable no-unused-expressions */
+
 const _ = require('lodash');
 const BbPromise = require('bluebird');
 const chai = require('chai');
@@ -230,28 +232,55 @@ describe('AwsProvider', () => {
     });
 
     it('should retry if error code is 429', (done) => {
-      let first = true;
       const error = {
         statusCode: 429,
         retryable: true,
         message: 'Testing retry',
       };
+      const sendFake = {
+        send: sinon.stub(),
+      };
+      sendFake.send.onFirstCall().yields(error);
+      sendFake.send.yields(undefined, {});
       class FakeS3 {
         constructor(credentials) {
           this.credentials = credentials;
         }
 
         error() {
-          return {
-            send(cb) {
-              if (first) {
-                first = false;
-                cb(error);
-              } else {
-                cb(undefined, {});
-              }
-            },
-          };
+          return sendFake;
+        }
+      }
+      awsProvider.sdk = {
+        S3: FakeS3,
+      };
+      awsProvider.request('S3', 'error', {})
+        .then(data => {
+          expect(data).to.exist;
+          expect(sendFake.send).to.have.been.calledTwice;
+          done();
+        })
+        .catch(done);
+    });
+
+    it('should retry if error code is 429 and retryable is set to false', (done) => {
+      const error = {
+        statusCode: 429,
+        retryable: false,
+        message: 'Testing retry',
+      };
+      const sendFake = {
+        send: sinon.stub(),
+      };
+      sendFake.send.onFirstCall().yields(error);
+      sendFake.send.yields(undefined, {});
+      class FakeS3 {
+        constructor(credentials) {
+          this.credentials = credentials;
+        }
+
+        error() {
+          return sendFake;
         }
       }
       awsProvider.sdk = {
@@ -259,8 +288,8 @@ describe('AwsProvider', () => {
       };
       awsProvider.request('S3', 'error', {})
         .then(data => {
-          expect(data).to.exist; // eslint-disable-line no-unused-expressions
-          expect(first).to.be.false; // eslint-disable-line no-unused-expressions
+          expect(data).to.exist;
+          expect(sendFake.send).to.have.been.calledTwice;
           done();
         })
         .catch(done);
@@ -322,6 +351,36 @@ describe('AwsProvider', () => {
         .catch(done);
     });
 
+    it('should not retry for missing credentials', (done) => {
+      const error = {
+        statusCode: 403,
+        message: 'Missing credentials in config',
+      };
+      const sendFake = {
+        send: sinon.stub().yields(error),
+      };
+      class FakeS3 {
+        constructor(credentials) {
+          this.credentials = credentials;
+        }
+
+        error() {
+          return sendFake;
+        }
+      }
+      awsProvider.sdk = {
+        S3: FakeS3,
+      };
+      awsProvider.request('S3', 'error', {})
+        .then(() => done('Should not succeed'))
+        .catch((err) => {
+          expect(sendFake.send).to.have.been.calledOnce;
+          expect(err.message).to.contain('in our docs here:');
+          done();
+        })
+        .catch(done);
+    });
+
     it('should enable S3 acceleration if CLI option is provided', () => {
       // mocking S3 for testing
       class FakeS3 {