Repository: aws/amazon-ecs-cli-v2
Pull request #454 (2019-11-21 21:07:48)

Fix Multiple Listener Rules
Right now, we pass in the Listener rule value of 1.
This works for 1 app, but not if we want to add another.
To solve this, we have to use a custom resource to generate
the next available rule priority by looking at the existing LB
Listener rules, and returning the max + 1.

This change also fixes a race condition with spinning up the
Listener and the ECS service (the service needs the Listener
to be created first). I had to do a little trick do to a conditional
depends on using a waiter.

This change, completely unrelated, also exposes the LB DNS path
as an env variable to the task definition we create.

<!-- Provide summary of changes -->

<!-- Issue number, if available. E.g. "Fixes #31", "Addresses #42, 77" -->

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.



Patch:
diff --git a/cf-custom-resources/lib/alb-rule-priority-generator.js b/cf-custom-resources/lib/alb-rule-priority-generator.js
new file mode 100644
index 00000000..7fe64dfa
--- /dev/null
+++ b/cf-custom-resources/lib/alb-rule-priority-generator.js
@@ -0,0 +1,141 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+const aws = require('aws-sdk');
+
+// These are used for test purposes only
+let defaultResponseURL;
+
+/**
+ * Upload a CloudFormation response object to S3.
+ *
+ * @param {object} event the Lambda event payload received by the handler function
+ * @param {object} context the Lambda context received by the handler function
+ * @param {string} responseStatus the response status, either 'SUCCESS' or 'FAILED'
+ * @param {string} physicalResourceId CloudFormation physical resource ID
+ * @param {object} [responseData] arbitrary response data object
+ * @param {string} [reason] reason for failure, if any, to convey to the user
+ * @returns {Promise} Promise that is resolved on success, or rejected on connection error or HTTP error response
+ */
+let report = function (event, context, responseStatus, physicalResourceId, responseData, reason) {
+    return new Promise((resolve, reject) => {
+        const https = require('https');
+        const {
+            URL
+        } = require('url');
+
+        var responseBody = JSON.stringify({
+            Status: responseStatus,
+            Reason: reason,
+            PhysicalResourceId: physicalResourceId || context.logStreamName,
+            StackId: event.StackId,
+            RequestId: event.RequestId,
+            LogicalResourceId: event.LogicalResourceId,
+            Data: responseData
+        });
+
+        const parsedUrl = new URL(event.ResponseURL || defaultResponseURL);
+        const options = {
+            hostname: parsedUrl.hostname,
+            port: 443,
+            path: parsedUrl.pathname + parsedUrl.search,
+            method: 'PUT',
+            headers: {
+                'Content-Type': '',
+                'Content-Length': responseBody.length
+            }
+        };
+
+        https.request(options)
+            .on('error', reject)
+            .on('response', res => {
+                res.resume();
+                if (res.statusCode >= 400) {
+                    reject(new Error(`Error ${res.statusCode}: ${res.statusMessage}`));
+                } else {
+                    resolve();
+                }
+            })
+            .end(responseBody, 'utf8');
+    });
+};
+
+/**
+ * Lists all the existing rules for a ALB Listener, finds the max of their
+ * priorities, and then returns max + 1.
+ *
+ * @param {string} listenerArn the ARN of the ALB listener.
+
+ * @returns {number} The next available ALB listener rule priority.
+ */
+const calculateNextRulePriority = async function (listenerArn) {
+    var elb = new aws.ELBv2();
+    // Grab all the rules for this listener
+    var marker;
+    var rules = []
+    do {
+        const rulesResponse = await elb.describeRules({
+            ListenerArn: listenerArn,
+            Marker: marker
+        }).promise();
+
+        rules = rules.concat(rulesResponse.Rules)
+        marker = rulesResponse.NextMarker;
+    } while (marker)
+
+    let nextRulePriority = 1;
+    if (rules.length > 0) {
+        // Take the max rule priority, and add 1 to it.
+        const rulePriorities = rules.map(rule => {
+            if (rule.Priority === "default") {
+                // We treat the default rule as having priority 0
+                return 0
+            }
+            return parseInt(rule.Priority);
+        });
+        const max = Math.max(...rulePriorities);
+        nextRulePriority = max + 1;
+    }
+
+    return nextRulePriority;
+};
+
+/**
+ * Next Available ALB Rule Priority handler, invoked by Lambda
+ */
+exports.nextAvailableRulePriorityHandler = async function(event, context) {
+    var responseData = {};
+    var physicalResourceId;
+    var rulePriority;
+
+    try {
+      switch (event.RequestType) {
+        case 'Create':
+          rulePriority = await calculateNextRulePriority(event.ResourceProperties.ListenerArn);
+          responseData.Priority = rulePriority;
+          physicalResourceId = `alb-rule-priority-${event.LogicalResourceId}`
+          break;
+        // Do nothing on update and delete, since this isn't a "real" resource.
+        case 'Update':
+        case 'Delete':
+          physicalResourceId = event.PhysicalResourceId
+          break;
+        default:
+          throw new Error(`Unsupported request type ${event.RequestType}`);
+      }
+
+      await report(event, context, 'SUCCESS', physicalResourceId, responseData);
+    } catch (err) {
+      console.log(`Caught error ${err}.`);
+      await report(event, context, 'FAILED', physicalResourceId, null, err.message);
+    }
+  };
+
+
+/**
+ * @private
+ */
+exports.withDefaultResponseURL = function(url) {
+  defaultResponseURL = url;
+};
diff --git a/internal/pkg/deploy/cloudformation/stack/lb_fargate_app.go b/internal/pkg/deploy/cloudformation/stack/lb_fargate_app.go
index 4a1aa6cc..42363ba9 100644
--- a/internal/pkg/deploy/cloudformation/stack/lb_fargate_app.go
+++ b/internal/pkg/deploy/cloudformation/stack/lb_fargate_app.go
@@ -18,8 +18,9 @@ import (
 )
 
 const (
-	lbFargateAppTemplatePath = "lb-fargate-service/cf.yml"
-	lbFargateAppParamsPath   = "lb-fargate-service/params.json"
+	lbFargateAppTemplatePath              = "lb-fargate-service/cf.yml"
+	lbFargateAppParamsPath                = "lb-fargate-service/params.json"
+	lbFargateAppRulePriorityGeneratorPath = "custom-resources/alb-rule-priority-generator.js"
 )
 
 const (
@@ -77,16 +78,32 @@ func (c *LBFargateStackConfig) StackName() string {
 
 // Template returns the CloudFormation template for the application parametrized for the environment.
 func (c *LBFargateStackConfig) Template() (string, error) {
+
+	rulePriority, err := c.box.FindString(lbFargateAppRulePriorityGeneratorPath)
+	if err != nil {
+		return "", &ErrTemplateNotFound{templateLocation: lbFargateAppRulePriorityGeneratorPath, parentErr: err}
+	}
+
 	content, err := c.box.FindString(lbFargateAppTemplatePath)
 	if err != nil {
 		return "", &ErrTemplateNotFound{templateLocation: lbFargateAppTemplatePath, parentErr: err}
 	}
+
 	tpl, err := template.New("template").Parse(content)
 	if err != nil {
 		return "", fmt.Errorf("parse CloudFormation template for %s: %w", c.App.Type, err)
 	}
+
+	templateData := struct {
+		RulePriorityLambda string
+		*lbFargateTemplateParams
+	}{
+		RulePriorityLambda:      rulePriority,
+		lbFargateTemplateParams: c.toTemplateParams(),
+	}
+
 	var buf bytes.Buffer
-	if err := tpl.Execute(&buf, c.toTemplateParams()); err != nil {
+	if err := tpl.Execute(&buf, templateData); err != nil {
 		return "", fmt.Errorf("execute CloudFormation template for %s: %w", c.App.Type, err)
 	}
 	return buf.String(), nil
diff --git a/templates/environment/cf.yml b/templates/environment/cf.yml
index eca0a59c..f871e9f6 100644
--- a/templates/environment/cf.yml
+++ b/templates/environment/cf.yml
@@ -130,7 +130,6 @@ Resources:
       RouteTableId: !Ref PublicRouteTable
       SubnetId: !Ref PublicSubnet2
 
-
   Cluster:
     Type: AWS::ECS::Cluster
 
@@ -205,6 +204,7 @@ Resources:
 
   CloudformationExecutionRole:
     Type: AWS::IAM::Role
+    DependsOn: VPC
     # This role should not be deleted while CloudFormation tries to delete the stack itself.
     DeletionPolicy: Retain
     Properties:
diff --git a/templates/lb-fargate-service/cf.yml b/templates/lb-fargate-service/cf.yml
index 0bca2ee1..bca8bdd7 100644
--- a/templates/lb-fargate-service/cf.yml
+++ b/templates/lb-fargate-service/cf.yml
@@ -48,6 +48,7 @@ Resources:
     Type: AWS::Logs::LogGroup
     Properties:
       LogGroupName: !Join ['', [/ecs/, !Ref ProjectName, '-', !Ref EnvName, '-', !Ref AppName]]
+
   TaskDefinition:
     Type: AWS::ECS::TaskDefinition
     DependsOn: LogGroup
@@ -64,8 +65,15 @@ Resources:
         - Name: !Ref AppName
           Image: !Ref ContainerImage
           PortMappings:
-            - ContainerPort: !Ref ContainerPort {{if .App.Variables}}
-          Environment:{{range $name, $value := .App.Variables}}
+            - ContainerPort: !Ref ContainerPort
+          # We pipe certain environment variables directly into the task definition.
+          # This lets customers have access to, for example, their LB endpoint - which they'd
+          # have no way of otherwise determining.
+          Environment:
+          - Name: ECS_ENV_LB_DNS
+            Value:
+              Fn::ImportValue:
+                !Sub "${ProjectName}-${EnvName}-PublicLoadBalancerDNS" {{if .App.Variables}}{{range $name, $value := .App.Variables}}
           - Name: {{$name}}
             Value: {{$value}}{{end}}{{end}}{{if .App.Secrets}}
           Secrets:{{range $name, $valueFrom := .App.Secrets}}
@@ -77,6 +85,7 @@ Resources:
               awslogs-region: !Ref AWS::Region
               awslogs-group: !Ref LogGroup
               awslogs-stream-prefix: ecs
+
   ExecutionRole:
     Type: AWS::IAM::Role
     Properties:
@@ -102,6 +111,7 @@ Resources:
                   - !Sub 'arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/*'
       ManagedPolicyArns:
         - 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy'
+
   TaskRole:
     Type: AWS::IAM::Role
     Properties:
@@ -121,6 +131,7 @@ Resources:
       VpcId:
         Fn::ImportValue:
           !Sub "${ProjectName}-${EnvName}-VpcId"
+
   ContainerSecurityGroupIngressFromPublicALB:
     Type: AWS::EC2::SecurityGroupIngress
     Properties:
@@ -130,6 +141,7 @@ Resources:
       SourceSecurityGroupId:
         Fn::ImportValue:
           !Sub "${ProjectName}-${EnvName}-PublicLoadBalancerSecurityGroupId"
+
   ContainerSecurityGroupIngressFromSelf:
     Type: AWS::EC2::SecurityGroupIngress
     Properties:
@@ -140,6 +152,7 @@ Resources:
 
   Service:
     Type: AWS::ECS::Service
+    DependsOn: WaitUntilListenerRuleIsCreated
     Properties:
       Cluster:
         Fn::ImportValue:
@@ -172,6 +185,7 @@ Resources:
         - ContainerName: !Ref AppName
           ContainerPort: !Ref ContainerPort
           TargetGroupArn: !Ref TargetGroup
+
   TargetGroup:
     Type: AWS::ElasticLoadBalancingV2::TargetGroup
     Properties:
@@ -188,6 +202,7 @@ Resources:
       VpcId:
         Fn::ImportValue:
           !Sub "${ProjectName}-${EnvName}-VpcId"
+
   LoadBalancerDNSAlias:
     Type: AWS::Route53::RecordSetGroup
     Condition: HTTPSLoadBalancer
@@ -212,22 +227,54 @@ Resources:
           DNSName:
             Fn::ImportValue:
               !Sub "${ProjectName}-${EnvName}-PublicLoadBalancerDNS"
-  HTTPListenerRule:
-    Type: AWS::ElasticLoadBalancingV2::ListenerRule
-    Condition: HTTPLoadBalancer
+
+  RulePriorityFunction:
+    Type: AWS::Lambda::Function
     Properties:
-      Actions:
-        - TargetGroupArn: !Ref TargetGroup
-          Type: forward
-      Conditions:
-        - Field: 'path-pattern'
-          PathPatternConfig:
-            Values:
-            - !Ref RulePath
+      Code:
+        ZipFile: |
+          {{.RulePriorityLambda}}
+      Handler: "index.nextAvailableRulePriorityHandler"
+      Timeout: 600
+      MemorySize: 512
+      Role: !GetAtt 'CustomResourceRole.Arn'
+      Runtime: nodejs10.x
+
+  CustomResourceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          -
+            Effect: Allow
+            Principal:
+              Service:
+                - lambda.amazonaws.com
+            Action:
+              - sts:AssumeRole
+      Path: /
+      Policies:
+        - PolicyName: "DNSandACMAccess"
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+                - elasticloadbalancing:DescribeRules
+              Resource: "*"
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+
+  HTTPSRulePriorityAction:
+    Condition: HTTPSLoadBalancer
+    Type: Custom::RulePriorityFunction
+    Properties:
+      ServiceToken: !GetAtt RulePriorityFunction.Arn
       ListenerArn:
         Fn::ImportValue:
-          !Sub "${ProjectName}-${EnvName}-HTTPListenerArn"
-      Priority: !Ref RulePriority
+          !Sub "${ProjectName}-${EnvName}-HTTPSListenerArn"
+
   HTTPSListenerRule:
     Type: AWS::ElasticLoadBalancingV2::ListenerRule
     Condition: HTTPSLoadBalancer
@@ -247,4 +294,60 @@ Resources:
       ListenerArn:
         Fn::ImportValue:
           !Sub "${ProjectName}-${EnvName}-HTTPSListenerArn"
-      Priority: !Ref RulePriority
+      Priority: !GetAtt HTTPSRulePriorityAction.Priority
+
+  HTTPRulePriorityAction:
+    Condition: HTTPLoadBalancer
+    Type: Custom::RulePriorityFunction
+    Properties:
+      ServiceToken: !GetAtt RulePriorityFunction.Arn
+      ListenerArn:
+        Fn::ImportValue:
+          !Sub "${ProjectName}-${EnvName}-HTTPListenerArn"
+
+  HTTPListenerRule:
+    Type: AWS::ElasticLoadBalancingV2::ListenerRule
+    Condition: HTTPLoadBalancer
+    Properties:
+      Actions:
+        - TargetGroupArn: !Ref TargetGroup
+          Type: forward
+      Conditions:
+        - Field: 'path-pattern'
+          PathPatternConfig:
+            Values:
+            - !Ref RulePath
+      ListenerArn:
+        Fn::ImportValue:
+          !Sub "${ProjectName}-${EnvName}-HTTPListenerArn"
+      Priority: !GetAtt HTTPRulePriorityAction.Priority
+
+  # Force a conditional dependency from the ECS service on the listener rules.
+  # Our service depends on our HTTP/S listener to be set up before it can
+  # be created. But, since our environment is either HTTPS or not, we
+  # have a conditional dependency (we have to wait for the HTTPS listener
+  # to be created or the HTTP listener to be created). In order to have a
+  # conditional dependency, we use the WaitHandle resource as a way to force
+  # a single dependency. The Ref in the WaitCondition implicitly creates a conditional
+  # dependency - if the condition is satisfied (HTTPLoadBalancer) - the ref resolves
+  # the HTTPWaitHandle, which depends on the HTTPListenerRule.
+
+  HTTPSWaitHandle:
+    Condition: HTTPSLoadBalancer
+    DependsOn: HTTPSListenerRule
+    Type: AWS::CloudFormation::WaitConditionHandle
+
+  HTTPWaitHandle:
+    Condition: HTTPLoadBalancer
+    DependsOn: HTTPListenerRule
+    Type: AWS::CloudFormation::WaitConditionHandle
+
+  # We don't actually need to wait for the condition to
+  # be completed, that's why we set a count of 0. The timeout
+  # is a required field, but useless, so we set it to one.
+  WaitUntilListenerRuleIsCreated:
+    Type: AWS::CloudFormation::WaitCondition
+    Properties:
+      Handle: !If [HTTPLoadBalancer, !Ref HTTPWaitHandle, !Ref HTTPSWaitHandle]
+      Timeout: "1"
+      Count: 0


Test patch:
diff --git a/cf-custom-resources/test/alb-rule-priority-generator-test.js b/cf-custom-resources/test/alb-rule-priority-generator-test.js
new file mode 100644
index 00000000..c7e08d25
--- /dev/null
+++ b/cf-custom-resources/test/alb-rule-priority-generator-test.js
@@ -0,0 +1,274 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+'use strict';
+
+describe('ALB Rule Priority Generator', () => {
+  const AWS = require('aws-sdk-mock');
+  const LambdaTester = require('lambda-tester').noVersionCheck();
+  const sinon = require('sinon');
+  const albRulePriorityHandler = require('../lib/alb-rule-priority-generator');
+  const nock = require('nock');
+  const ResponseURL = 'https://cloudwatch-response-mock.example.com/';
+
+  let origLog = console.log;
+
+  const testRequestId = 'f4ef1b10-c39a-44e3-99c0-fbf7e53c3943';
+  const testALBListenerArn = 'arn:aws:elasticloadbalancing:us-west-2:00000000:listener/app/lblistner';
+
+  beforeEach(() => {
+    albRulePriorityHandler.withDefaultResponseURL(ResponseURL);
+    console.log = function() { };
+  });
+  afterEach(() => {
+    AWS.restore();
+    console.log = origLog;
+  });
+
+  test('Empty event payload fails', () => {
+    const request = nock(ResponseURL).put('/', body => {
+      return body.Status === 'FAILED' && body.Reason === 'Unsupported request type undefined';
+    }).reply(200);
+    return LambdaTester(albRulePriorityHandler.nextAvailableRulePriorityHandler)
+      .event({})
+      .expectResolve(() => {
+        expect(request.isDone()).toBe(true);
+      });
+  });
+
+  test('Bogus operation fails', () => {
+    const bogusType = 'bogus';
+    const request = nock(ResponseURL).put('/', body => {
+      return body.Status === 'FAILED' && body.Reason === 'Unsupported request type ' + bogusType;
+    }).reply(200);
+    return LambdaTester(albRulePriorityHandler.nextAvailableRulePriorityHandler)
+      .event({
+        RequestType: bogusType
+      })
+      .expectResolve(() => {
+        expect(request.isDone()).toBe(true);
+      });
+  });
+
+  test('Delete event is a no-op', () => {
+    const describeRulesFake = sinon.fake.resolves(
+        {
+            "Rules": []
+    });
+
+    AWS.mock('ELBv2', 'describeRules', describeRulesFake);
+
+    const requestType = 'Delete';
+    const request = nock(ResponseURL).put('/', body => {
+      return body.Status === 'SUCCESS';
+    }).reply(200);
+    return LambdaTester(albRulePriorityHandler.nextAvailableRulePriorityHandler)
+      .event({
+        RequestType: requestType
+      })
+      .expectResolve(() => {
+        sinon.assert.notCalled(describeRulesFake)
+        expect(request.isDone()).toBe(true);
+      });
+  });
+
+  test('Update event is a no-op', () => {
+    const describeRulesFake = sinon.fake.resolves(
+        {
+            "Rules": []
+    });
+
+    AWS.mock('ELBv2', 'describeRules', describeRulesFake);
+
+    const requestType = 'Update';
+    const request = nock(ResponseURL).put('/', body => {
+      return body.Status === 'SUCCESS';
+    }).reply(200);
+    return LambdaTester(albRulePriorityHandler.nextAvailableRulePriorityHandler)
+      .event({
+        RequestType: requestType
+      })
+      .expectResolve(() => {
+        sinon.assert.notCalled(describeRulesFake)
+        expect(request.isDone()).toBe(true);
+      });
+  });
+
+  test('Create operation returns rule priority 1 when only the default rule is present', () => {
+
+    const describeRulesFake = sinon.fake.resolves(
+        {
+            "Rules": [
+                {
+                    "Priority": "default",
+                    "Conditions": [],
+                    "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:listener-rule/app/rule",
+                    "IsDefault": true,
+                    "Actions": [
+                        {
+                            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:targetgroup/tg",
+                            "Type": "forward"
+                        }
+                    ]
+                }
+            ]
+    });
+
+    AWS.mock('ELBv2', 'describeRules', describeRulesFake);
+    const request = nock(ResponseURL).put('/', body => {
+      return body.Status === 'SUCCESS' && body.Data.Priority == 1;
+    }).reply(200);
+
+    return LambdaTester(albRulePriorityHandler.nextAvailableRulePriorityHandler)
+      .event({
+        RequestType: 'Create',
+        RequestId: testRequestId,
+        ResourceProperties: {
+          ListenerArn: testALBListenerArn
+        }
+      })
+      .expectResolve(() => {
+        sinon.assert.calledWith(describeRulesFake, sinon.match({
+            ListenerArn: testALBListenerArn,
+        }));
+        expect(request.isDone()).toBe(true);
+      });
+  });
+
+
+  test('Create operation returns rule priority max + 1', () => {
+    // This set of rules has the default, 3 and 5 rule priorities. We don't try to fill
+    // in the gaps, we just create one that is 1 + the max. In this case, 6.
+    const describeRulesFake = sinon.fake.resolves(
+        {
+            "Rules": [
+                {
+                    "Priority": "default",
+                    "Conditions": [],
+                    "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:listener-rule/app/rule",
+                    "IsDefault": true,
+                    "Actions": [
+                        {
+                            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:targetgroup/tg",
+                            "Type": "forward"
+                        }
+                    ]
+                },
+                {
+                    "Priority": "3",
+                    "Conditions": [],
+                    "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:listener-rule/app/rule",
+                    "IsDefault": true,
+                    "Actions": [
+                        {
+                            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:targetgroup/tg",
+                            "Type": "forward"
+                        }
+                    ]
+                },
+                {
+                    "Priority": "5",
+                    "Conditions": [],
+                    "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:listener-rule/app/rule",
+                    "IsDefault": true,
+                    "Actions": [
+                        {
+                            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:targetgroup/tg",
+                            "Type": "forward"
+                        }
+                    ]
+                },
+            ]
+    });
+
+    AWS.mock('ELBv2', 'describeRules', describeRulesFake);
+    const request = nock(ResponseURL).put('/', body => {
+      return body.Status === 'SUCCESS' && body.Data.Priority == 6;
+    }).reply(200);
+
+    return LambdaTester(albRulePriorityHandler.nextAvailableRulePriorityHandler)
+      .event({
+        RequestType: 'Create',
+        RequestId: testRequestId,
+        ResourceProperties: {
+          ListenerArn: testALBListenerArn
+        }
+      })
+      .expectResolve(() => {
+        sinon.assert.calledWith(describeRulesFake, sinon.match({
+            ListenerArn: testALBListenerArn,
+        }));
+        expect(request.isDone()).toBe(true);
+      });
+  });
+
+  test('Create operation returns rule priority max + 1 for paginated response', () => {
+    // This set of rules has the default, 3 and 5 rule priorities. We don't try to fill
+    // in the gaps, we just create one that is 1 + the max. In this case, 6.
+    const describeRulesFake = sinon.stub();
+    const testNextMarkerToken = "12345";
+    describeRulesFake.onCall(0).resolves(
+        {
+            "NextMarker" : testNextMarkerToken,
+            "Rules": [
+                {
+                    "Priority": "default",
+                    "Conditions": [],
+                    "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:listener-rule/app/rule",
+                    "IsDefault": true,
+                    "Actions": [
+                        {
+                            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:targetgroup/tg",
+                            "Type": "forward"
+                        }
+                    ]
+                },
+            ]
+    });
+
+    describeRulesFake.onCall(1).resolves(
+        {
+            "Rules": [
+                {
+                    "Priority": "100",
+                    "Conditions": [],
+                    "RuleArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:listener-rule/app/rule",
+                    "IsDefault": true,
+                    "Actions": [
+                        {
+                            "TargetGroupArn": "arn:aws:elasticloadbalancing:us-west-2:000000000:targetgroup/tg",
+                            "Type": "forward"
+                        }
+                    ]
+                },
+            ]
+    });
+
+    AWS.mock('ELBv2', 'describeRules', describeRulesFake);
+    const request = nock(ResponseURL).put('/', body => {
+      return body.Status === 'SUCCESS' && body.Data.Priority == 101;
+    }).reply(200);
+
+    return LambdaTester(albRulePriorityHandler.nextAvailableRulePriorityHandler)
+      .event({
+        RequestType: 'Create',
+        RequestId: testRequestId,
+        ResourceProperties: {
+          ListenerArn: testALBListenerArn
+        }
+      })
+      .expectResolve(() => {
+        sinon.assert.calledWith(describeRulesFake.firstCall, sinon.match({
+            ListenerArn: testALBListenerArn,
+        }));
+
+        sinon.assert.calledWith(describeRulesFake.secondCall, sinon.match({
+            ListenerArn: testALBListenerArn,
+            Marker: testNextMarkerToken
+        }));
+
+        expect(request.isDone()).toBe(true);
+      });
+  });
+
+});
+
diff --git a/internal/pkg/deploy/cloudformation/stack/lb_fargate_app_test.go b/internal/pkg/deploy/cloudformation/stack/lb_fargate_app_test.go
index b343d5a9..a88cc41a 100644
--- a/internal/pkg/deploy/cloudformation/stack/lb_fargate_app_test.go
+++ b/internal/pkg/deploy/cloudformation/stack/lb_fargate_app_test.go
@@ -76,15 +76,25 @@ func TestLBFargateStackConfig_Template(t *testing.T) {
 		wantedTemplate string
 		wantedError    error
 	}{
-		"unavailable template": {
-			mockBox: func(box *packd.MemoryBox) {}, // empty box where template does not exist
-
+		"unavailable app template": {
+			mockBox: func(box *packd.MemoryBox) {
+				box.AddString(lbFargateAppRulePriorityGeneratorPath, "javascript")
+			},
 			wantedTemplate: "",
 			wantedError: &ErrTemplateNotFound{
 				templateLocation: lbFargateAppTemplatePath,
 				parentErr:        os.ErrNotExist,
 			},
 		},
+		"unavailable custom resource template": {
+			mockBox: func(box *packd.MemoryBox) {},
+
+			wantedTemplate: "",
+			wantedError: &ErrTemplateNotFound{
+				templateLocation: lbFargateAppRulePriorityGeneratorPath,
+				parentErr:        os.ErrNotExist,
+			},
+		},
 		"render default template": {
 			in: &deploy.CreateLBFargateAppInput{
 				App: manifest.NewLoadBalancedFargateManifest("frontend", "frontend/Dockerfile"),
@@ -99,6 +109,7 @@ func TestLBFargateStackConfig_Template(t *testing.T) {
 				ImageTag:     "manual-bf3678c",
 			},
 			mockBox: func(box *packd.MemoryBox) {
+				box.AddString(lbFargateAppRulePriorityGeneratorPath, "javascript")
 				box.AddString(lbFargateAppTemplatePath, `Parameters:
   ProjectName: {{.Env.Project}}
   EnvName: {{.Env.Name}}