Repository: serverless/serverless (JavaScript)
Date: 2016-11-18 17:03:40+00:00
Task type: Bug Fix

Issue:
Cloud Formation Resource Dependencies: Function, Role and Policy
There is a problem when we deploy function that shall run on AWS VPC (subnets and security groups specified). In most of the cases deployment fails because of missing permission required for creating EC2 network interfaces. However, _ec2:DescribeNetworkInterfaces_, _ec2:CreateNetworkInterface_ and _ec2:DeleteNetworkInterface_ are specified as additional **iamRoleStatements**.

It seems the problem is **concurrency** and **parallel** creation of several resources.

If we investigate the CloudFormation template created, we can see (assuming we have Hello Lambda function inside):

- The function **HelloLambdaFunction** references the role **IamRoleLambdaExecution**
- The policy **IamPolicyLambdaExecution** references the role  **IamRoleLambdaExecution**.

The creation of the **HelloLambdaFunction** and **IamPolicyLambdaExecution** goes in parallel.
If **HelloLambdaFunction** creation goes first, it will use the role **IamRoleLambdaExecution** before the policy is actually attached to that role and the whole stack creation process fails (insufficient permission, can’t create EC2 Network Interface). 

Occasionally it can succeed, but that behavior is totally random. In order to avoid this problem we must specify explicit **DependsOn** statements in the template. The **Function** shall depend on the **Policy** and **Policy** on the **Role**.  Stack  creation with explicit dependencies set on goes smoothly without any problems.




Issue discussion:

Solution patch:
diff --git a/lib/plugins/aws/deploy/compile/functions/index.js b/lib/plugins/aws/deploy/compile/functions/index.js
index 9c869f20570..acc8512aa2b 100644
--- a/lib/plugins/aws/deploy/compile/functions/index.js
+++ b/lib/plugins/aws/deploy/compile/functions/index.js
@@ -19,15 +19,44 @@ class AwsCompileFunctions {
     };
   }
 
-  compileRole(role) {
-    if (typeof role === 'object') {
-      // role is an "Fn::GetAtt" object
-      return role;
-    } else if (role.indexOf(':') === -1) {
-      // role is a Logical Role Name
-      return { 'Fn::GetAtt': [role, 'Arn'] };
+  compileRole(newFunction, role) {
+    const compiledFunction = newFunction;
+    const unnsupportedRoleError = new this.serverless.classes
+      .Error(`Unsupported role provided: "${JSON.stringify(role)}"`);
+
+    switch (typeof role) {
+      case 'object':
+        if ('Fn::GetAtt' in role) {
+          // role is an "Fn::GetAtt" object
+          compiledFunction.Properties.Role = role;
+          compiledFunction.DependsOn = [role['Fn::GetAtt'][0]];
+        } else if ('Fn::ImportValue' in role) {
+          // role is an "Fn::ImportValue" object
+          compiledFunction.Properties.Role = role;
+        } else {
+          throw unnsupportedRoleError;
+        }
+        break;
+      case 'string':
+        if (role.startsWith('arn:aws')) {
+          // role is a statically definied iam arn
+          compiledFunction.Properties.Role = role;
+        } else if (role === 'IamRoleLambdaExecution') {
+          // role is the default role generated by the framework
+          compiledFunction.Properties.Role = { 'Fn::GetAtt': [role, 'Arn'] };
+          compiledFunction.DependsOn = [
+            'IamPolicyLambdaExecution',
+            'IamRoleLambdaExecution',
+          ];
+        } else {
+          // role is a Logical Role Name
+          compiledFunction.Properties.Role = { 'Fn::GetAtt': [role, 'Arn'] };
+          compiledFunction.DependsOn = [role];
+        }
+        break;
+      default:
+        throw unnsupportedRoleError;
     }
-    return role; // indicates that role is a Role ARN
   }
 
   compileFunction(functionName) {
@@ -101,11 +130,11 @@ class AwsCompileFunctions {
     }
 
     if ('role' in functionObject) {
-      newFunction.Properties.Role = this.compileRole(functionObject.role);
+      this.compileRole(newFunction, functionObject.role);
     } else if ('role' in this.serverless.service.provider) {
-      newFunction.Properties.Role = this.compileRole(this.serverless.service.provider.role);
+      this.compileRole(newFunction, this.serverless.service.provider.role);
     } else {
-      newFunction.Properties.Role = this.compileRole('IamRoleLambdaExecution');
+      this.compileRole(newFunction, 'IamRoleLambdaExecution');
     }
 
     if (!functionObject.vpc) functionObject.vpc = {};
diff --git a/lib/plugins/aws/deploy/lib/iam-policy-lambda-execution-template.json b/lib/plugins/aws/deploy/lib/iam-policy-lambda-execution-template.json
index 171d3c65737..261043d118f 100644
--- a/lib/plugins/aws/deploy/lib/iam-policy-lambda-execution-template.json
+++ b/lib/plugins/aws/deploy/lib/iam-policy-lambda-execution-template.json
@@ -1,5 +1,8 @@
 {
   "Type": "AWS::IAM::Policy",
+  "DependsOn": [
+    "IamRoleLambdaExecution"
+  ],
   "Properties": {
     "PolicyName": "",
     "PolicyDocument": {


Test patch:
diff --git a/lib/plugins/aws/deploy/compile/functions/index.test.js b/lib/plugins/aws/deploy/compile/functions/index.test.js
index 80e4df9c2c6..0266baa0ebe 100644
--- a/lib/plugins/aws/deploy/compile/functions/index.test.js
+++ b/lib/plugins/aws/deploy/compile/functions/index.test.js
@@ -91,7 +91,7 @@ describe('AwsCompileFunctions', () => {
 
     it('should add an ARN provider role', () => {
       awsCompileFunctions.serverless.service.provider.name = 'aws';
-      awsCompileFunctions.serverless.service.provider.role = 'some:aws:arn:xxx:*:*';
+      awsCompileFunctions.serverless.service.provider.role = 'arn:aws:xxx:*:*';
       awsCompileFunctions.serverless.service.functions = {
         func: {
           handler: 'func.function.handler',
@@ -101,6 +101,8 @@ describe('AwsCompileFunctions', () => {
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.FuncLambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.provider.role);
@@ -118,6 +120,9 @@ describe('AwsCompileFunctions', () => {
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction.DependsOn
+      ).to.deep.equal(['LogicalNameRole']);
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.FuncLambdaFunction.Properties.Role
       ).to.deep.equal({
@@ -145,6 +150,9 @@ describe('AwsCompileFunctions', () => {
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction.DependsOn
+      ).to.deep.equal(['LogicalRoleName']);
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.FuncLambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.provider.role);
@@ -156,12 +164,14 @@ describe('AwsCompileFunctions', () => {
         func: {
           handler: 'func.function.handler',
           name: 'new-service-dev-func',
-          role: 'some:aws:arn:xxx:*:*',
+          role: 'arn:aws:xxx:*:*',
         },
       };
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.FuncLambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.functions.func.role);
@@ -179,6 +189,9 @@ describe('AwsCompileFunctions', () => {
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction.DependsOn
+      ).to.deep.equal(['LogicalRoleName']);
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.FuncLambdaFunction.Properties.Role
       ).to.deep.equal({
@@ -206,6 +219,30 @@ describe('AwsCompileFunctions', () => {
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction.DependsOn
+      ).to.deep.equal(['LogicalRoleName']);
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction.Properties.Role
+      ).to.deep.equal(awsCompileFunctions.serverless.service.functions.func.role);
+    });
+
+    it('should add a "Fn::ImportValue" Object function role', () => {
+      awsCompileFunctions.serverless.service.provider.name = 'aws';
+      awsCompileFunctions.serverless.service.functions = {
+        func: {
+          handler: 'func.function.handler',
+          name: 'new-service-dev-func',
+          role: {
+            'Fn::ImportValue': 'Foo',
+          },
+        },
+      };
+
+      awsCompileFunctions.compileFunctions();
+
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.FuncLambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.functions.func.role);
@@ -213,17 +250,19 @@ describe('AwsCompileFunctions', () => {
 
     it('should prefer function declared role over provider declared role', () => {
       awsCompileFunctions.serverless.service.provider.name = 'aws';
-      awsCompileFunctions.serverless.service.provider.role = 'some:provider:arn:xxx:*:*';
+      awsCompileFunctions.serverless.service.provider.role = 'arn:aws:xxx:*:*';
       awsCompileFunctions.serverless.service.functions = {
         func: {
           handler: 'func.function.handler',
           name: 'new-service-dev-func',
-          role: 'some:function:arn:xxx:*:*',
+          role: 'arn:aws:xxx:*:*',
         },
       };
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.FuncLambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.FuncLambdaFunction.Properties.Role
       ).to.equal(awsCompileFunctions.serverless.service.functions.func.role);
@@ -235,20 +274,25 @@ describe('AwsCompileFunctions', () => {
         func0: {
           handler: 'func.function.handler',
           name: 'new-service-dev-func0',
-          role: 'some:aws:arn:xx0:*:*',
+          role: 'arn:aws:xx0:*:*',
         },
         func1: {
           handler: 'func.function.handler',
           name: 'new-service-dev-func1',
-          role: 'some:aws:arn:xx1:*:*',
+          role: 'arn:aws:xx1:*:*',
         },
       };
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.Func0LambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.Func0LambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.functions.func0.role);
+
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.Func1LambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.Func1LambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.functions.func1.role);
@@ -256,7 +300,7 @@ describe('AwsCompileFunctions', () => {
 
     it('should add function declared role and fill in with provider role', () => {
       awsCompileFunctions.serverless.service.provider.name = 'aws';
-      awsCompileFunctions.serverless.service.provider.role = 'some:aws:arn:xxx:*:*';
+      awsCompileFunctions.serverless.service.provider.role = 'arn:aws:xxx:*:*';
       awsCompileFunctions.serverless.service.functions = {
         func0: {
           handler: 'func.function.handler',
@@ -265,15 +309,20 @@ describe('AwsCompileFunctions', () => {
         func1: {
           handler: 'func.function.handler',
           name: 'new-service-dev-func1',
-          role: 'some:aws:arn:xx1:*:*',
+          role: 'arn:aws:xx1:*:*',
         },
       };
 
       awsCompileFunctions.compileFunctions();
 
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.Func0LambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.Func0LambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.provider.role);
+
+      expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
+        .Resources.Func1LambdaFunction).not.to.have.property('DependsOn');
       expect(awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate
         .Resources.Func1LambdaFunction.Properties.Role
       ).to.deep.equal(awsCompileFunctions.serverless.service.functions.func1.role);
@@ -298,6 +347,10 @@ describe('AwsCompileFunctions', () => {
       };
       const compiledFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -334,6 +387,10 @@ describe('AwsCompileFunctions', () => {
       };
       const compliedFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -385,6 +442,10 @@ describe('AwsCompileFunctions', () => {
 
       const compliedFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -434,6 +495,10 @@ describe('AwsCompileFunctions', () => {
 
       const compliedFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -482,6 +547,10 @@ describe('AwsCompileFunctions', () => {
 
       const compliedFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -533,6 +602,10 @@ describe('AwsCompileFunctions', () => {
 
       const compliedFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -593,6 +666,10 @@ describe('AwsCompileFunctions', () => {
       };
       const compliedFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -630,6 +707,10 @@ describe('AwsCompileFunctions', () => {
 
       const compiledFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -661,6 +742,10 @@ describe('AwsCompileFunctions', () => {
       };
       const compiledFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -696,6 +781,10 @@ describe('AwsCompileFunctions', () => {
       };
       const compiledFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -733,6 +822,10 @@ describe('AwsCompileFunctions', () => {
       };
       const compiledFunction = {
         Type: 'AWS::Lambda::Function',
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          'IamRoleLambdaExecution',
+        ],
         Properties: {
           Code: {
             S3Key: `${awsCompileFunctions.serverless.service.package.artifactDirectoryName}/${
@@ -826,22 +919,113 @@ describe('AwsCompileFunctions', () => {
   });
 
   describe('#compileRole()', () => {
-    const logicalRoleName = 'LogicalRoleName';
-    const roleObject = {
-      'Fn::GetAtt': [logicalRoleName, 'Arn'],
-    };
-    const roleArn = 'some:aws:arn:xxx:*:*';
+    it('adds the default role with DependsOn values', () => {
+      const role = 'IamRoleLambdaExecution';
+      const resource = { Properties: {} };
+      awsCompileFunctions.compileRole(resource, role);
+
+      expect(resource).to.deep.equal({
+        DependsOn: [
+          'IamPolicyLambdaExecution',
+          role,
+        ],
+        Properties: {
+          Role: {
+            'Fn::GetAtt': [
+              role,
+              'Arn',
+            ],
+          },
+        },
+      });
+    });
 
-    it('compiles a logical role name into an reference object', () => {
-      expect(awsCompileFunctions.compileRole(logicalRoleName)).to.deep.equal(roleObject);
+    it('adds a role based on a logical name with DependsOn values', () => {
+      const role = 'LogicalRoleName';
+      const resource = { Properties: {} };
+      awsCompileFunctions.compileRole(resource, role);
+
+      expect(resource).to.deep.equal({
+        DependsOn: [
+          role,
+        ],
+        Properties: {
+          Role: {
+            'Fn::GetAtt': [
+              role,
+              'Arn',
+            ],
+          },
+        },
+      });
     });
 
-    it('returns a ARN string when given', () => {
-      expect(awsCompileFunctions.compileRole(roleArn)).to.equal(roleArn);
+    it('adds a role based on a Fn::GetAtt with DependsOn values', () => {
+      const role = { 'Fn::GetAtt': ['Foo', 'Arn'] };
+      const resource = { Properties: {} };
+      awsCompileFunctions.compileRole(resource, role);
+
+      expect(resource).to.deep.equal({
+        DependsOn: [
+          'Foo',
+        ],
+        Properties: {
+          Role: role,
+        },
+      });
     });
 
-    it('returns a reference object when given', () => {
-      expect(awsCompileFunctions.compileRole(roleObject)).to.deep.equal(roleObject);
+    it('adds a role based on a Fn::ImportValue', () => {
+      const role = { 'Fn::ImportValue': 'Foo' };
+      const resource = { Properties: {} };
+      awsCompileFunctions.compileRole(resource, role);
+
+      expect(resource).to.deep.equal({
+        Properties: {
+          Role: role,
+        },
+      });
+    });
+
+    it('adds a role based on a predefined arn string', () => {
+      const role = 'arn:aws:xxx:*:*';
+      const resource = { Properties: {} };
+      awsCompileFunctions.compileRole(resource, role);
+
+      expect(resource).to.deep.equal({
+        Properties: {
+          Role: role,
+        },
+      });
+    });
+
+    describe('Errors if unsupported object type is provided', () => {
+      it('should throw for object type { Ref: "Foo" }', () => {
+        const role = { Ref: 'Foo' };
+        const resource = { Properties: {} };
+
+        expect(() =>
+          awsCompileFunctions.compileRole(resource, role)
+        ).to.throw(Error);
+      });
+
+      it('should throw for object type Buffer', () => {
+        const role = new Buffer('Foo');
+        const resource = { Properties: {} };
+
+        expect(() =>
+          awsCompileFunctions.compileRole(resource, role)
+        ).to.throw(Error);
+      });
+
+      it('should throw for object type Array', () => {
+        const role = [1, 2, 3];
+        const resource = { Properties: {} };
+
+        expect(() =>
+          awsCompileFunctions.compileRole(resource, role)
+        ).to.throw(Error);
+      });
     });
   });
 });