Repository: openaddresses/machine
Pull request #545 (2017-02-10 01:29:14)

Switch Slack status updates to SNS
Instead of talking to Slack directly, I'm connecting SNS to Slack via [a new AWS Lambda function](https://console.aws.amazon.com/lambda/home?region=us-east-1#/functions/OA-Slack?tab=code). This should soon remove the need for a Slack configuration setting and allow AWS configuration to handle messaging.


Patch:
diff --git a/chef/crontab/recipes/default.rb b/chef/crontab/recipes/default.rb
index 0af2106..17f2508 100644
--- a/chef/crontab/recipes/default.rb
+++ b/chef/crontab/recipes/default.rb
@@ -11,7 +11,6 @@ aws_sns_arn = local['aws_sns_arn']
 aws_cloudwatch_ns = local['aws_cloudwatch_ns']
 github_token = local['github_token']
 mapbox_key = local['mapbox_key']
-slack_url = local['slack_url']
 
 database_url = "postgres://#{db_user}:#{db_pass}@#{db_host}/#{db_name}?sslmode=require"
 
@@ -71,7 +70,6 @@ LC_ALL=C.UTF-8
   --hours 18 \
   -b "#{aws_s3_bucket}" \
   --sns-arn "#{aws_sns_arn}" \
-  --slack-url "#{slack_url}" \
   --verbose \
   -- \
     openaddr-collect-extracts \
@@ -93,7 +91,6 @@ LC_ALL=C.UTF-8
   --hours 9 \
   -b "#{aws_s3_bucket}" \
   --sns-arn "#{aws_sns_arn}" \
-  --slack-url "#{slack_url}" \
   --verbose \
   -- \
     openaddr-index-tiles \
@@ -117,7 +114,6 @@ LC_ALL=C.UTF-8
   --instance-type r3.large \
   -b "#{aws_s3_bucket}" \
   --sns-arn "#{aws_sns_arn}" \
-  --slack-url "#{slack_url}" \
   --verbose \
   -- \
     openaddr-update-dotmap \
@@ -139,7 +135,6 @@ LC_ALL=C.UTF-8
   --instance-type t2.nano \
   -b "#{aws_s3_bucket}" \
   --sns-arn "#{aws_sns_arn}" \
-  --slack-url "#{slack_url}" \
   --verbose \
   -- \
     openaddr-enqueue-sources \
diff --git a/chef/data/local.json b/chef/data/local.json
index d7e7b2a..c8cd206 100644
--- a/chef/data/local.json
+++ b/chef/data/local.json
@@ -21,7 +21,6 @@
     "webhook_secrets": "{Comma-delimited list of secrets}",
     "mapbox_key": "{Mapbox Key}",
     "mapzen_key": "{Mapzen Key}",
-    "slack_url": "{Slack URL}",
     "gag_github_status": "No",
 
     "worker_kind": null,
diff --git a/chef/worker/recipes/default.rb b/chef/worker/recipes/default.rb
index b0488e6..4398c3c 100644
--- a/chef/worker/recipes/default.rb
+++ b/chef/worker/recipes/default.rb
@@ -13,49 +13,6 @@ gag_github_status = local['gag_github_status']
 database_url = "postgres://#{db_user}:#{db_pass}@#{db_host}/#{db_name}?sslmode=require"
 web_docroot = local['web_docroot']
 mapzen_key = local['mapzen_key']
-slack_url = local['slack_url']
-
-#
-# Announce status to the world.
-#
-file '/etc/init.d/openaddr-worker' do
-  mode '0755'
-  content <<-INITD
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides:          openaddr-worker
-# Required-Start:    $local_fs $network
-# Required-Stop:     $local_fs $network
-# Should-Start:      $named
-# Should-Stop:       $named
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Start OpenAddresses worker.
-# Description:       Start OpenAddresses worker.
-### END INIT INFO
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-
-case "$1" in
-  start)
-    curl -X POST -d '{"text": "Worker started on `'`hostname --long`'`."}' '#{slack_url}'
-    ;;
-  stop)
-    curl -X POST -d '{"text": "Worker stopped on `'`hostname --long`'`."}' '#{slack_url}'
-    ;;
-  restart|reload|force-reload)
-    echo "Error: argument '$1' not supported" >&2
-    exit 3
-    ;;
-  *)
-    echo "Usage: $0 start|stop" >&2
-    exit 3
-    ;;
-esac
-INITD
-end
-
-execute "update-rc.d openaddr-worker defaults"
-execute "/etc/init.d/openaddr-worker start"
 
 env_file = "/tmp/#{app_name}.conf"
 procfile = File.join(File.dirname(__FILE__), '..', '..', 'Procfile-worker')
diff --git a/lambda.py b/lambda.py
new file mode 100755
index 0000000..89f53ee
--- /dev/null
+++ b/lambda.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python2.7
+from __future__ import print_function
+import json
+import os
+try:
+    from http.client import HTTPSConnection
+    from urllib.parse import urlparse
+except ImportError:
+    from httplib import HTTPSConnection
+    from urlparse import urlparse
+
+def lambda_handler(event, context):
+    '''
+    '''
+    parsed = urlparse(os.environ['SLACK_URL'])
+    conn = HTTPSConnection(parsed.hostname)
+    
+    for message in summarize_messages(event):
+        body = json.dumps(dict(text=message))
+        conn.request('POST', parsed.path, body)
+        resp = conn.getresponse()
+
+        print('HTTP {} for message {}'.format(resp.status, message))
+
+def summarize_messages(event):
+    '''
+    '''
+    records = event.get('Records', [])
+    messages = []
+    
+    for record in records:
+        if 'Sns' in record:
+            try:
+                message = json.loads(record['Sns']['Message'])['Cause']
+            except:
+                message = record['Sns']['Message']
+            messages.append('*{}*\n```\n{}\n```'.format(record['Sns']['Subject'], message))
+        else:
+            print('Unknown record type:', record)
+            messages.append('Mysterious message from {}'.format(record.get('EventSource', '???')))
+    
+    return messages
+
+if __name__ == '__main__':
+    print(summarize_messages(json.loads('''
+        {
+          "Records": [
+            {
+              "EventVersion": "1.0",
+              "EventSource": "aws:sns",
+              "EventSubscriptionArn": "arn:aws:sns:us-east-1:847904970422:CI-Events:380ff93e-d662-4028-8659-c5fb820673cd",
+              "Sns": {
+                "UnsubscribeUrl": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:847904970422:CI-Events:380ff93e-d662-4028-8659-c5fb820673cd",
+                "Type": "Notification",
+                "SignatureVersion": "1",
+                "Timestamp": "2017-02-09T00:31:48.572Z",
+                "Signature": "bT6VrGYf/O+GwELg62I27uTHwymYnSPSUGvRwJbaYQO1StIo9r2gOPu6qvuC7u75jSl7gdxsM2xJ/bJAgLP2BV15Eiv8S0bx7pInSZLiQXbe+40Tk3K1ydcKUkW5XMu8PP2OdRSktz2PB65Tkau+37R1VlID4C6fCcTd3oiD0xQGW1dsjzeJmGW8mONh/G3fUpnx8jbKOZdRGlaqzkhEZWA4zuGgBqXXoLShWokIPzS15WJYL2NUJ5EmE9G6dTibd8rkbRWIJh5Qhta/QBQ67ipNXZG5eqv+0alMef11/Ww92cMdS4z22p/oR+yWgw3OGzh+W8YQBOn45zr2PbXJEA==",
+                "Message": "{\\"Progress\\":50,\\"AccountId\\":\\"847904970422\\",\\"Description\\":\\"Launching a new EC2 instance: i-0154e7d1fba186b38\\",\\"RequestId\\":\\"510cf431-7cab-42e8-9ce9-91d7ebc8cb33\\",\\"EndTime\\":\\"2017-02-09T00:31:48.519Z\\",\\"AutoScalingGroupARN\\":\\"arn:aws:autoscaling:us-east-1:847904970422:autoScalingGroup:2d9cc8fc-b822-4266-afa1-c40ba5f0863c:autoScalingGroupName/CI Crontab 4.x\\",\\"ActivityId\\":\\"510cf431-7cab-42e8-9ce9-91d7ebc8cb33\\",\\"StartTime\\":\\"2017-02-09T00:31:16.159Z\\",\\"Service\\":\\"AWS Auto Scaling\\",\\"Time\\":\\"2017-02-09T00:31:48.519Z\\",\\"EC2InstanceId\\":\\"i-0154e7d1fba186b38\\",\\"StatusCode\\":\\"InProgress\\",\\"StatusMessage\\":\\"\\",\\"Details\\":{\\"Subnet ID\\":\\"subnet-35d87242\\",\\"Availability Zone\\":\\"us-east-1b\\"},\\"AutoScalingGroupName\\":\\"CI Crontab 4.x\\",\\"Cause\\":\\"At 2017-02-09T00:31:13Z an instance was started in response to a difference between desired and actual capacity, increasing the capacity from 0 to 1.\\",\\"Event\\":\\"autoscaling:EC2_INSTANCE_LAUNCH\\"}",
+                "SigningCertUrl": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-b95095beb82e8f6a046b3aafc7f4149a.pem",
+                "MessageAttributes": {},
+                "Subject": "Auto Scaling: launch for group \\"CI Crontab 4.x\\"",
+                "TopicArn": "arn:aws:sns:us-east-1:847904970422:CI-Events",
+                "MessageId": "90cced31-040b-58a6-9538-bc24bebf2c8b"
+              }
+            },
+            {
+              "EventVersion": "1.0", 
+              "EventSource": "aws:sns", 
+              "EventSubscriptionArn": "arn:aws:sns:us-east-1:847904970422:CI-Events:543efcac-0802-4fdd-9eb1-d6d6c8f76799", 
+              "Sns": {
+                "MessageId": "d0f9ec0a-e542-5c24-863e-bd1f2993ce6b", 
+                "Signature": "mTD5mrUzok2eE1UmJR7Le/D0eOveczZ39wXC7bxxg8IMOchSNwa6+KtKV4D+oD26uC4WSmCH5z92b09hX6vaTTpdc7G1DPywInUiwrLXYgrPgFKVG1Tj1JJZqTp+14JH/XiaaQ5WQ9sxPSQ7u1Iczd86jHtdkdOs7LBmWgzuFjFAJrkJz41JpgYuiEhDR0K07Syz/EKBtao3hd3QlG2CvJNNqhgxaYvn98GMDsfbtO0OZcoZX4TiAHkQslpyj3v0B/7IpuRKsmIC7wXL0NMNH2S8TetLqVLPpssezOYOHJj/Lu53ojeYJ3y3hwdQGBHL8yH6gOuUT7G/x6wFNDdKKQ==", 
+                "Type": "Notification", 
+                "TopicArn": "arn:aws:sns:us-east-1:847904970422:CI-Events", 
+                "MessageAttributes": {}, 
+                "SignatureVersion": "1", 
+                "Timestamp": "2017-02-09T01:49:05.702Z", 
+                "SigningCertUrl": "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-b95095beb82e8f6a046b3aafc7f4149a.pem", 
+                "Message": "And this is the test message", 
+                "UnsubscribeUrl": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:847904970422:CI-Events:543efcac-0802-4fdd-9eb1-d6d6c8f76799", 
+                "Subject": "Test Subject"
+              }
+            }
+          ]
+        }
+        ''')))
diff --git a/openaddr/run_ec2_ami.py b/openaddr/run_ec2_ami.py
index e0364ab..3f27595 100644
--- a/openaddr/run_ec2_ami.py
+++ b/openaddr/run_ec2_ami.py
@@ -14,6 +14,7 @@ parser = ArgumentParser(description='Run a process on a remote EC2 instance.')
 
 parser.add_argument('-a', '--access-key', help='Deprecated option provided for backwards compatibility.')
 parser.add_argument('-s', '--secret-key', help='Deprecated option provided for backwards compatibility.')
+parser.add_argument('--slack-url', help='Deprecated option provided for backwards compatibility.')
 
 parser.add_argument('-b', '--bucket', default=environ.get('AWS_S3_BUCKET', None),
                     help='S3 bucket name. Defaults to value of AWS_S3_BUCKET environment variable.')
@@ -30,9 +31,6 @@ parser.add_argument('--hours', default=12, type=float,
 parser.add_argument('--instance-type', default='m3.medium',
                     help='EC2 instance type. Defaults to "m3.medium".')
 
-parser.add_argument('--slack-url',
-                    help='Slack POST URL.')
-
 parser.add_argument('-v', '--verbose', help='Turn on verbose logging',
                     action='store_const', dest='loglevel',
                     const=logging.DEBUG, default=logging.INFO)
@@ -55,7 +53,7 @@ def main():
     autoscale = connect_autoscale(None, None)
     instance = request_task_instance(ec2, autoscale, args.instance_type,
                                      args.role, lifespan, args.command,
-                                     args.bucket, args.slack_url)
+                                     args.bucket, args.sns_arn)
     
     _L.info('instance {} is off to the races.'.format(instance))
 
diff --git a/openaddr/util/__init__.py b/openaddr/util/__init__.py
index e105442..1c48bd3 100644
--- a/openaddr/util/__init__.py
+++ b/openaddr/util/__init__.py
@@ -52,18 +52,7 @@ def set_autoscale_capacity(autoscale, cloudwatch, cloudwatch_ns, capacity):
     if measure['Maximum'] > .9:
         group.set_capacity(capacity)
 
-def _command_messages(command):
-    '''
-    '''
-    strings = dict(
-        message_starting = 'Starting {}...'.format(command),
-        message_complete = 'Completed {}'.format(command),
-        message_failed = 'Failed {}'.format(command),
-        )
-    
-    return { k: shlex.quote(json.dumps(dict(text=v))) for (k, v) in strings.items() }
-
-def request_task_instance(ec2, autoscale, instance_type, chef_role, lifespan, command, bucket, slack_url):
+def request_task_instance(ec2, autoscale, instance_type, chef_role, lifespan, command, bucket, aws_sns_arn):
     '''
     '''
     group_name = 'CI Workers {0}.x'.format(*get_version().split('.'))
@@ -83,9 +72,19 @@ def request_task_instance(ec2, autoscale, instance_type, chef_role, lifespan, co
             version = shlex.quote(get_version()),
             log_prefix = shlex.quote('logs/{}-{}'.format(yyyymmdd, command[0])),
             bucket = shlex.quote(bucket or 'data.openaddresses.io'),
-            slack_url = shlex.quote(slack_url or ''),
-            **_command_messages(command[0])
+            aws_sns_arn = '', aws_region = '',
+            command_name = command[0]
             )
+        
+        if aws_sns_arn:
+            try:
+                _, _, _, aws_region, _ = aws_sns_arn.split(':', 4)
+            except ValueError:
+                pass
+            else:
+                if aws_sns_arn.startswith('arn:aws:sns:'):
+                    userdata_kwargs.update(aws_sns_arn = shlex.quote(aws_sns_arn),
+                                           aws_region = shlex.quote(aws_region))
     
         run_kwargs = dict(instance_type=instance_type, security_groups=['default'],
                           instance_initiated_shutdown_behavior='terminate',
diff --git a/openaddr/util/templates/task-instance-userdata.sh b/openaddr/util/templates/task-instance-userdata.sh
index 22c0fe3..09970b2 100644
--- a/openaddr/util/templates/task-instance-userdata.sh
+++ b/openaddr/util/templates/task-instance-userdata.sh
@@ -1,22 +1,22 @@
 #!/bin/bash -ex
 
-# Tell Slack all about it
-function notify_slack
+# Tell SNS all about it
+function notify_sns
 {{
-    if [ {slack_url} ]; then
-        echo $1 | curl -s -o /dev/null -X POST -d @- {slack_url} || true
+    if [ {aws_sns_arn} ]; then
+        aws --region {aws_region} sns publish --topic-arn {aws_sns_arn} --subject 'Requested task instance' --message "$1"
     fi
 }}
 
-notify_slack {message_starting}
+notify_sns 'Starting {command_name}...'
 
 # Bail out with a log message
 function shutdown_with_log
 {{
     if [ $1 = 0 ]; then
-        notify_slack {message_complete}
+        notify_sns 'Completed {command_name}'
     else
-        notify_slack {message_failed}
+        notify_sns 'Failed {command_name}'
     fi
     
     mkdir /tmp/task


Test patch:
diff --git a/openaddr/tests/util.py b/openaddr/tests/util.py
index 1a15b1d..f3ac696 100644
--- a/openaddr/tests/util.py
+++ b/openaddr/tests/util.py
@@ -100,7 +100,7 @@ class TestUtilities (unittest.TestCase):
         image.run.return_value = reservation
         reservation.instances = [instance]
         
-        util.request_task_instance(ec2, autoscale, 'm3.medium', chef_role, 60, command, 'bucket-name', None)
+        util.request_task_instance(ec2, autoscale, 'm3.medium', chef_role, 60, command, 'bucket-name', 'arn:aws:sns:null-island:etc.')
         
         autoscale.get_all_groups.assert_called_once_with([expected_group_name])
         autoscale.get_all_launch_configurations.assert_called_once_with(names=[group.launch_config_name])
@@ -114,6 +114,9 @@ class TestUtilities (unittest.TestCase):
         
         self.assertIn('chef/run.sh {}'.format(quote(chef_role)), image_run_kwargs['user_data'])
         self.assertIn('s3://bucket-name/logs/', image_run_kwargs['user_data'])
+        self.assertIn("notify_sns 'Starting openaddr-good-times...'", image_run_kwargs['user_data'])
+        self.assertIn('aws --region null-island sns publish --topic-arn arn:aws:sns:null-island:etc.',
+                      image_run_kwargs['user_data'])
         for (arg1, arg2) in zip(command, command[1:]):
             self.assertIn(quote(arg1)+' '+quote(arg2), image_run_kwargs['user_data'])