Repository: openaddresses/machine
Pull request #478 (2016-12-06 05:45:42)

Configure CloudWatch namespace
`openaddr.ci` namespace was used all over the place; this change configures it explicitly.

Part of #472.


Patch:
diff --git a/chef/crontab/recipes/default.rb b/chef/crontab/recipes/default.rb
index 9f9b255..bb9847e 100644
--- a/chef/crontab/recipes/default.rb
+++ b/chef/crontab/recipes/default.rb
@@ -10,6 +10,7 @@ aws_access_id = local['aws_access_id']
 aws_secret_key = local['aws_secret_key']
 aws_s3_bucket = local['aws_s3_bucket']
 aws_sns_arn = local['aws_sns_arn']
+aws_cloudwatch_ns = local['aws_cloudwatch_ns']
 github_token = local['github_token']
 mapbox_key = local['mapbox_key']
 slack_url = local['slack_url']
@@ -153,6 +154,7 @@ LC_ALL=C.UTF-8
     -s "#{aws_secret_key}" \
     -b "#{aws_s3_bucket}" \
     --sns-arn "#{aws_sns_arn}" \
+    --cloudwatch-ns "#{aws_cloudwatch_ns}" \
   >> /var/log/openaddr_crontab/enqueue-sources.log 2>&1
 CRONTAB
 end
diff --git a/chef/data/local.json b/chef/data/local.json
index 97020d1..41acc6e 100644
--- a/chef/data/local.json
+++ b/chef/data/local.json
@@ -13,6 +13,7 @@
     "aws_secret_key": "{Amazon Secret Access Key}",
     "aws_sns_arn": "{Amazon SNS Subscription ID}",
     "aws_s3_bucket": "{Amazon S3 Bucket Name}",
+    "aws_cloudwatch_ns": "{Amazon Cloudwatch Namespace}",
 
     "github_token": "{Github Token}",
     "github_callback": "{Github Callback URL}",
diff --git a/chef/webhooks/recipes/default.rb b/chef/webhooks/recipes/default.rb
index a14798f..320245d 100644
--- a/chef/webhooks/recipes/default.rb
+++ b/chef/webhooks/recipes/default.rb
@@ -11,6 +11,7 @@ aws_access_id = local['aws_access_id']
 aws_secret_key = local['aws_secret_key']
 aws_s3_bucket = local['aws_s3_bucket']
 aws_sns_arn = local['aws_sns_arn']
+aws_cloudwatch_ns = local['aws_cloudwatch_ns']
 webhook_secrets = local['webhook_secrets']
 
 gag_github_status = local['gag_github_status']
@@ -40,6 +41,7 @@ GAG_GITHUB_STATUS=#{gag_github_status}
 AWS_ACCESS_KEY_ID=#{aws_access_id}
 AWS_SECRET_ACCESS_KEY=#{aws_secret_key}
 AWS_SNS_ARN=#{aws_sns_arn}
+AWS_CLOUDWATCH_NS=#{aws_cloudwatch_ns}
 AWS_S3_BUCKET=#{aws_s3_bucket}
 WEBHOOK_SECRETS=#{webhook_secrets}
 LC_ALL=C.UTF-8
diff --git a/openaddr/ci/enqueue.py b/openaddr/ci/enqueue.py
index 31d70b3..e1a1c24 100644
--- a/openaddr/ci/enqueue.py
+++ b/openaddr/ci/enqueue.py
@@ -40,6 +40,9 @@ parser.add_argument('-s', '--secret-key', help='Deprecated option provided for b
 parser.add_argument('--sns-arn', default=environ.get('AWS_SNS_ARN', None),
                     help='Optional AWS Simple Notification Service (SNS) resource. Defaults to value of AWS_SNS_ARN environment variable.')
 
+parser.add_argument('--cloudwatch-ns', default=environ.get('AWS_CLOUDWATCH_NS', None),
+                    help='Optional AWS CloudWatch namespace. Defaults to value of AWS_CLOUDWATCH_NS environment variable.')
+
 parser.add_argument('-v', '--verbose', help='Turn on verbose logging',
                     action='store_const', dest='loglevel',
                     const=logging.DEBUG, default=logging.INFO)
@@ -84,7 +87,7 @@ def main():
                 try:
                     if time() >= next_autoscale_grow:
                         next_autoscale_grow = time() + next_autoscale_interval
-                        set_autoscale_capacity(autoscale, cloudwatch, next(minimum_capacity))
+                        set_autoscale_capacity(autoscale, cloudwatch, args.cloudwatch_ns, next(minimum_capacity))
                 except Exception as e:
                     _L.error('Problem during autoscale', exc_info=True)
                 if expected_count:
diff --git a/openaddr/ci/run_dequeue.py b/openaddr/ci/run_dequeue.py
index c14ec82..693cb96 100644
--- a/openaddr/ci/run_dequeue.py
+++ b/openaddr/ci/run_dequeue.py
@@ -46,11 +46,12 @@ def main():
                 _L.info('{workers_n} active workers; queue lengths: {task_n} tasks, {done_n} done, {due_n} due'.format(**locals()))
                 
                 if cw:
-                    cw.put_metric_data('openaddr.ci', 'tasks queue', task_n, unit='Count')
-                    cw.put_metric_data('openaddr.ci', 'done queue', done_n, unit='Count')
-                    cw.put_metric_data('openaddr.ci', 'due queue', due_n, unit='Count')
-                    cw.put_metric_data('openaddr.ci', 'expected results', task_n + workers_n, unit='Count')
-                    cw.put_metric_data('openaddr.ci', 'active workers', workers_n, unit='Count')
+                    ns = environ.get('AWS_CLOUDWATCH_NS')
+                    cw.put_metric_data(ns, 'tasks queue', task_n, unit='Count')
+                    cw.put_metric_data(ns, 'done queue', done_n, unit='Count')
+                    cw.put_metric_data(ns, 'due queue', due_n, unit='Count')
+                    cw.put_metric_data(ns, 'expected results', task_n + workers_n, unit='Count')
+                    cw.put_metric_data(ns, 'active workers', workers_n, unit='Count')
 
                 checkin_time = time() + 30
 
diff --git a/openaddr/util/__init__.py b/openaddr/util/__init__.py
index 0da66a4..e33d7fe 100644
--- a/openaddr/util/__init__.py
+++ b/openaddr/util/__init__.py
@@ -33,12 +33,12 @@ def prepare_db_kwargs(dsn):
     
     return kwargs
 
-def set_autoscale_capacity(autoscale, cloudwatch, capacity):
+def set_autoscale_capacity(autoscale, cloudwatch, cloudwatch_ns, capacity):
     '''
     '''
     span, now = 60 * 60 * 3, datetime.now()
     start, end = now - timedelta(seconds=span), now
-    args = 'tasks queue', 'openaddr.ci', 'Maximum'
+    args = 'tasks queue', cloudwatch_ns, 'Maximum'
 
     (measure, ) = cloudwatch.get_metric_statistics(span, start, end, *args)
 


Test patch:
diff --git a/openaddr/tests/util.py b/openaddr/tests/util.py
index 52b5acb..4fdca0f 100644
--- a/openaddr/tests/util.py
+++ b/openaddr/tests/util.py
@@ -47,7 +47,7 @@ class TestUtilities (unittest.TestCase):
         autoscale.get_all_groups.return_value = [as_group]
         
         as_group.desired_capacity = 2
-        util.set_autoscale_capacity(autoscale, cloudwatch, 1)
+        util.set_autoscale_capacity(autoscale, cloudwatch, 'ns', 1)
         
         # The right group name was used.
         autoscale.get_all_groups.assert_called_once_with([group_name])
@@ -56,23 +56,28 @@ class TestUtilities (unittest.TestCase):
         as_group.set_capacity.assert_not_called()
 
         as_group.desired_capacity = 1
-        util.set_autoscale_capacity(autoscale, cloudwatch, 1)
+        util.set_autoscale_capacity(autoscale, cloudwatch, 'ns', 1)
         
         as_group.desired_capacity = 0
         cloudwatch.get_metric_statistics.return_value = [{'Maximum': 0}]
-        util.set_autoscale_capacity(autoscale, cloudwatch, 1)
+        util.set_autoscale_capacity(autoscale, cloudwatch, 'ns', 1)
         
         cloudwatch.get_metric_statistics.return_value = [{'Maximum': 1}]
-        util.set_autoscale_capacity(autoscale, cloudwatch, 1)
+        util.set_autoscale_capacity(autoscale, cloudwatch, 'ns', 1)
         
         # Capacity had to be increased to 1.
         as_group.set_capacity.assert_called_once_with(1)
 
         as_group.desired_capacity = 1
-        util.set_autoscale_capacity(autoscale, cloudwatch, 2)
+        util.set_autoscale_capacity(autoscale, cloudwatch, 'ns', 2)
         
         # Capacity had to be increased to 2.
         as_group.set_capacity.assert_called_with(2)
+
+        # The right namespace was used.
+        for mock_call in cloudwatch.mock_calls:
+            self.assertEqual(mock_call[1][:1], (10800, ))
+            self.assertEqual(mock_call[1][-3:], ('tasks queue', 'ns', 'Maximum'))
     
     def test_task_instance(self):
         '''