GitHub PullRequestReviewCommentEvent at 2014-12-18 15:20:30
Repository: configuration
Actor: feanil
Payload: {"action":"created","comment":{"url":"https://api.github.com/repos/edx/configuration/pulls/comments/22047566","id":22047566,"diff_hunk":"@@ -0,0 +1,196 @@\n+\"\"\"\n+\n+For a given aws account, go through all un-attached volumes and tag them.\n+\n+\"\"\"\n+import boto\n+import boto.utils\n+import argparse\n+import logging\n+import subprocess\n+import time\n+import os\n+from os.path import join, exists, isdir, islink, realpath, basename, dirname\n+import yaml\n+# needs to be pip installed\n+import netaddr\n+\n+LOG_FORMAT = \"%(asctime)s %(levelname)s - %(filename)s:%(lineno)s - %(message)s\"\n+\n+log_level = logging.INFO\n+\n+def tags_for_hostname(hostname, mapping):\n+    logging.debug(\"Hostname is {}\".format(hostname))\n+    if not hostname.startswith('ip-'):\n+        return {}\n+\n+    octets = hostname.lstrip('ip-').split('-')\n+    tags = {}\n+\n+    # Update with env and deployment info\n+    tags.update(mapping['CIDR_SECOND_OCTET'][octets[1]])\n+\n+    ip_addr = netaddr.IPAddress(\".\".join(octets))\n+    for key, value in mapping['CIDR_REST'].items():\n+        cidr = \".\".join([\n+            mapping['CIDR_FIRST_OCTET'],\n+            octets[1],\n+            key])\n+\n+        cidrset = netaddr.IPSet([cidr])\n+\n+        if ip_addr in cidrset:\n+            tags.update(value)\n+\n+    return tags\n+\n+def potential_devices(root_device):\n+    device_dir = dirname(root_device)\n+    relevant_devices = lambda x: x.startswith(basename(root_device))\n+\n+    all_devices = os.listdir(device_dir)\n+    all_devices = filter( relevant_devices, all_devices)\n+\n+    logging.info(\"Potential devices on {}: {}\".format(root_device, all_devices))\n+    if len(all_devices) > 1:\n+        all_devices.remove(basename(root_device))\n+\n+    return map(lambda x: join(device_dir, x), all_devices)\n+\n+def get_tags_for_disk(mountpoint):\n+    tag_data = {}\n+    # Look at some files on it to determine:\n+    #  - hostname\n+    #  - environment\n+    #  - deployment\n+    #  - cluster\n+    #  - instance-id\n+    #  - date created\n+    hostname_file = join(mountpoint, \"etc\", \"hostname\")\n+    edx_dir = join(mountpoint, 'edx', 'app')\n+    if exists(hostname_file):\n+        # This means this was a root volume.\n+        with open(hostname_file, 'r') as f:\n+            hostname = f.readline().strip()\n+            tag_data['hostname'] = hostname\n+\n+        if exists(edx_dir) and isdir(edx_dir):\n+            # This is an ansible related ami, we'll try to map\n+            # the hostname to a knows deployment and cluster.\n+            cluster_tags = tags_for_hostname(hostname, mappings)\n+            tag_data.update(cluster_tags)\n+        else:\n+            # Not an ansible created root volume.\n+            tag_data['cluster'] = 'unknown'\n+    else:\n+        # Not a root volume\n+        tag_data['cluster'] = \"unknown\"\n+\n+    instance_file = join(mountpoint, \"var\", \"lib\", \"cloud\", \"instance\")\n+    if exists(instance_file) and islink(instance_file):\n+        resolved_path = realpath(instance_file)\n+        old_instance_id = basename(resolved_path)\n+        tag_data['instance-id'] = old_instance_id\n+\n+    return tag_data\n+\n+\n+if __name__ == \"__main__\":\n+    parser = argparse.ArgumentParser(description=\"Tag unattached ebs volumes.\")\n+    parser.add_argument(\"--profile\", '-p',\n+        help=\"AWS Profile to use with boto.\")\n+    parser.add_argument(\"--noop\", \"-n\", action=\"store_true\",\n+        help=\"Don't actually tag anything.\")\n+    parser.add_argument(\"--verbose\", \"-v\", action=\"store_true\",\n+        help=\"More verbose output.\")\n+    parser.add_argument(\"--device\", \"-d\", default=\"/dev/xvdf\",\n+        help=\"The /dev/??? where the volume should be mounted.\")\n+    parser.add_argument(\"--mountpoint\", \"-m\", default=\"/mnt\",\n+        help=\"Location to mount the new device.\")\n+    parser.add_argument(\"--config\", \"-c\", required=True,\n+        help=\"Configuration to map hostnames to tags.\")\n+    # The config should specify what tags to associate with the second\n+    # and this octet of the hostname which should be the ip address.\n+    # example:\n+\n+    args = parser.parse_args()\n+\n+    mappings = yaml.safe_load(open(args.config,'r'))\n+\n+    # Setup Logging\n+    if args.verbose:\n+        log_level = logging.DEBUG\n+\n+    logging.basicConfig(format=LOG_FORMAT, level=log_level)\n+\n+    # setup boto\n+    ec2 = boto.connect_ec2(profile_name=args.profile)\n+\n+    # get mounting args\n+    id_info = boto.utils.get_instance_identity()['document']\n+    instance_id = id_info['instanceId']\n+    az = id_info['availabilityZone']\n+    root_device = args.device\n+    mountpoint = args.mountpoint\n+\n+    # Find all unattached volumes\n+    filters = { \"status\": \"available\", \"availability-zone\": az }\n+    potential_volumes = ec2.get_all_volumes(filters=filters)\n+    logging.debug(\"Found {} unattached volumes in {}\".format(len(potential_volumes), az))\n+\n+    for vol in potential_volumes:\n+        if \"cluster\" in vol.tags:\n+            continue\n+        # Attach volume to the instance running this process\n+        logging.debug(\"Trying to attach {} to {} at {}\".format(\n+            vol.id, instance_id, root_device))\n+\n+        try:\n+            ec2.attach_volume(vol.id, instance_id, root_device)\n+            # Wait for the volume to finish attaching.\n+            waiting_msg = \"Waiting for {} to be available at {}\"\n+            while not exists(root_device):\n+                time.sleep(2)\n+                logging.debug(waiting_msg.format(vol.id, root_device))\n+\n+\n+            # Because a volume might have multiple mount points\n+            devices_on_volume = potential_devices(root_device)\n+            if len(devices_on_volume) != 1:\n+                vol.add_tag(\"devices_on_volume\", str(devices_on_volume))\n+                # Don't tag in this case because the different devices\n+                # may have conflicting tags.\n+                logging.info(\"Skipping {} because it has multiple mountpoints.\".format(vol.id))\n+                logging.info(\"{} has mountpoints {}\".format(vol.id, str(devices_on_volume)))\n+            else:\n+                device = devices_on_volume[0]\n+                try:\n+                    # Mount the volume\n+                    subprocess.check_call([\"sudo\", \"mount\", device, mountpoint])\n+\n+                    # Learn all tags we can know from content on disk.\n+                    tag_data = get_tags_for_disk(mountpoint)\n+                    tag_data['created'] = vol.create_time\n+\n+                    # If they are found tag the instance with them\n+                    if args.noop:\n+                        logging.info(\"Would have tagged {} with: \\n{}\".format(vol.id, str(tag_data)))\n+                    else:\n+                        logging.info(\"Tagging {} with: \\n{}\".format(vol.id, str(tag_data)))\n+                        vol.add_tags(tag_data)\n+                finally:\n+                    # Un-mount the volume\n+                    subprocess.check_call(['sudo', 'umount', mountpoint])\n+        finally:\n+            # Need this to be a function so we always re-check the API for status.\n+            is_attached = lambda vol_id: ec2.get_all_volumes(vol_id)[0].status != \"available\"\n+            while exists(root_device) or is_attached(vol.id):\n+                if is_attached(vol.id):\n+                    try:\n+                        # detach the volume\n+                        ec2.detach_volume(vol.id)\n+                    except boto.exception.EC2ResponseError as e:\n+                        logging.warning(\"Failed to detach volume.  Will try again in a bit.\")\n+                time.sleep(2)\n+                logging.debug(\"Waiting for {} to be detached.\".format(vol.id))","path":"util/vpc-tools/tag-old-ebs.py","position":195,"original_position":195,"commit_id":"dbd25668b549ab85f84a04fc0188da4ec6ef755e","original_commit_id":"dbd25668b549ab85f84a04fc0188da4ec6ef755e","user":{"login":"feanil","id":781561,"avatar_url":"https://avatars.githubusercontent.com/u/781561?v=3","gravatar_id":"","url":"https://api.github.com/users/feanil","html_url":"https://github.com/feanil","followers_url":"https://api.github.com/users/feanil/followers","following_url":"https://api.github.com/users/feanil/following{/other_user}","gists_url":"https://api.github.com/users/feanil/gists{/gist_id}","starred_url":"https://api.github.com/users/feanil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/feanil/subscriptions","organizations_url":"https://api.github.com/users/feanil/orgs","repos_url":"https://api.github.com/users/feanil/repos","events_url":"https://api.github.com/users/feanil/events{/privacy}","received_events_url":"https://api.github.com/users/feanil/received_events","type":"User","site_admin":false},"body":"I'll add a 5 minute timeout after which, I'll exit with an error. In most cases it does eventually detach but in the process of detaching the API calls are sometimes inconsistent.","created_at":"2014-12-18T15:20:30Z","updated_at":"2014-12-18T15:20:30Z","html_url":"https://github.com/edx/configuration/pull/1749#discussion_r22047566","pull_request_url":"https://api.github.com/repos/edx/configuration/pulls/1749","_links":{"self":{"href":"https://api.github.com/repos/edx/configuration/pulls/comments/22047566"},"html":{"href":"https://github.com/edx/configuration/pull/1749#discussion_r22047566"},"pull_request":{"href":"https://api.github.com/repos/edx/configuration/pulls/1749"}}},"pull_request":{"url":"https://api.github.com/repos/edx/configuration/pulls/1749","id":26097947,"html_url":"https://github.com/edx/configuration/pull/1749","diff_url":"https://github.com/edx/configuration/pull/1749.diff","patch_url":"https://github.com/edx/configuration/pull/1749.patch","issue_url":"https://api.github.com/repos/edx/configuration/issues/1749","number":1749,"state":"open","locked":false,"title":"Work for mounting currently detached volumes and inspecting them to tag them.","user":{"login":"feanil","id":781561,"avatar_url":"https://avatars.githubusercontent.com/u/781561?v=3","gravatar_id":"","url":"https://api.github.com/users/feanil","html_url":"https://github.com/feanil","followers_url":"https://api.github.com/users/feanil/followers","following_url":"https://api.github.com/users/feanil/following{/other_user}","gists_url":"https://api.github.com/users/feanil/gists{/gist_id}","starred_url":"https://api.github.com/users/feanil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/feanil/subscriptions","organizations_url":"https://api.github.com/users/feanil/orgs","repos_url":"https://api.github.com/users/feanil/repos","events_url":"https://api.github.com/users/feanil/events{/privacy}","received_events_url":"https://api.github.com/users/feanil/received_events","type":"User","site_admin":false},"body":"","created_at":"2014-12-15T20:18:10Z","updated_at":"2014-12-18T15:20:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5b72e3e1f9abf763ca0b378815ff9a2d240b8ff5","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/edx/configuration/pulls/1749/commits","review_comments_url":"https://api.github.com/repos/edx/configuration/pulls/1749/comments","review_comment_url":"https://api.github.com/repos/edx/configuration/pulls/comments/{number}","comments_url":"https://api.github.com/repos/edx/configuration/issues/1749/comments","statuses_url":"https://api.github.com/repos/edx/configuration/statuses/dbd25668b549ab85f84a04fc0188da4ec6ef755e","head":{"label":"edx:feanil/tag_old_ebs_vols","ref":"feanil/tag_old_ebs_vols","sha":"dbd25668b549ab85f84a04fc0188da4ec6ef755e","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":9478219,"name":"configuration","full_name":"edx/configuration","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/configuration","description":"a simple, but flexible, way for anyone to stand up an instance of the edX platform that is fully configured and ready-to-go","fork":false,"url":"https://api.github.com/repos/edx/configuration","forks_url":"https://api.github.com/repos/edx/configuration/forks","keys_url":"https://api.github.com/repos/edx/configuration/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/configuration/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/configuration/teams","hooks_url":"https://api.github.com/repos/edx/configuration/hooks","issue_events_url":"https://api.github.com/repos/edx/configuration/issues/events{/number}","events_url":"https://api.github.com/repos/edx/configuration/events","assignees_url":"https://api.github.com/repos/edx/configuration/assignees{/user}","branches_url":"https://api.github.com/repos/edx/configuration/branches{/branch}","tags_url":"https://api.github.com/repos/edx/configuration/tags","blobs_url":"https://api.github.com/repos/edx/configuration/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/configuration/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/configuration/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/configuration/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/configuration/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/configuration/languages","stargazers_url":"https://api.github.com/repos/edx/configuration/stargazers","contributors_url":"https://api.github.com/repos/edx/configuration/contributors","subscribers_url":"https://api.github.com/repos/edx/configuration/subscribers","subscription_url":"https://api.github.com/repos/edx/configuration/subscription","commits_url":"https://api.github.com/repos/edx/configuration/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/configuration/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/configuration/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/configuration/issues/comments/{number}","contents_url":"https://api.github.com/repos/edx/configuration/contents/{+path}","compare_url":"https://api.github.com/repos/edx/configuration/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/configuration/merges","archive_url":"https://api.github.com/repos/edx/configuration/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/configuration/downloads","issues_url":"https://api.github.com/repos/edx/configuration/issues{/number}","pulls_url":"https://api.github.com/repos/edx/configuration/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/configuration/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/configuration/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/configuration/labels{/name}","releases_url":"https://api.github.com/repos/edx/configuration/releases{/id}","created_at":"2013-04-16T17:23:42Z","updated_at":"2014-12-18T14:30:31Z","pushed_at":"2014-12-18T14:33:00Z","git_url":"git://github.com/edx/configuration.git","ssh_url":"git@github.com:edx/configuration.git","clone_url":"https://github.com/edx/configuration.git","svn_url":"https://github.com/edx/configuration","homepage":"","size":37861,"stargazers_count":308,"watchers_count":308,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":268,"mirror_url":null,"open_issues_count":28,"forks":268,"open_issues":28,"watchers":308,"default_branch":"master"}},"base":{"label":"edx:master","ref":"master","sha":"ef085f21c6041b53a8ff7ca0122061d2d2294702","user":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"repo":{"id":9478219,"name":"configuration","full_name":"edx/configuration","owner":{"login":"edx","id":3179841,"avatar_url":"https://avatars.githubusercontent.com/u/3179841?v=3","gravatar_id":"","url":"https://api.github.com/users/edx","html_url":"https://github.com/edx","followers_url":"https://api.github.com/users/edx/followers","following_url":"https://api.github.com/users/edx/following{/other_user}","gists_url":"https://api.github.com/users/edx/gists{/gist_id}","starred_url":"https://api.github.com/users/edx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/edx/subscriptions","organizations_url":"https://api.github.com/users/edx/orgs","repos_url":"https://api.github.com/users/edx/repos","events_url":"https://api.github.com/users/edx/events{/privacy}","received_events_url":"https://api.github.com/users/edx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/edx/configuration","description":"a simple, but flexible, way for anyone to stand up an instance of the edX platform that is fully configured and ready-to-go","fork":false,"url":"https://api.github.com/repos/edx/configuration","forks_url":"https://api.github.com/repos/edx/configuration/forks","keys_url":"https://api.github.com/repos/edx/configuration/keys{/key_id}","collaborators_url":"https://api.github.com/repos/edx/configuration/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/edx/configuration/teams","hooks_url":"https://api.github.com/repos/edx/configuration/hooks","issue_events_url":"https://api.github.com/repos/edx/configuration/issues/events{/number}","events_url":"https://api.github.com/repos/edx/configuration/events","assignees_url":"https://api.github.com/repos/edx/configuration/assignees{/user}","branches_url":"https://api.github.com/repos/edx/configuration/branches{/branch}","tags_url":"https://api.github.com/repos/edx/configuration/tags","blobs_url":"https://api.github.com/repos/edx/configuration/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/edx/configuration/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/edx/configuration/git/refs{/sha}","trees_url":"https://api.github.com/repos/edx/configuration/git/trees{/sha}","statuses_url":"https://api.github.com/repos/edx/configuration/statuses/{sha}","languages_url":"https://api.github.com/repos/edx/configuration/languages","stargazers_url":"https://api.github.com/repos/edx/configuration/stargazers","contributors_url":"https://api.github.com/repos/edx/configuration/contributors","subscribers_url":"https://api.github.com/repos/edx/configuration/subscribers","subscription_url":"https://api.github.com/repos/edx/configuration/subscription","commits_url":"https://api.github.com/repos/edx/configuration/commits{/sha}","git_commits_url":"https://api.github.com/repos/edx/configuration/git/commits{/sha}","comments_url":"https://api.github.com/repos/edx/configuration/comments{/number}","issue_comment_url":"https://api.github.com/repos/edx/configuration/issues/comments/{number}","contents_url":"https://api.github.com/repos/edx/configuration/contents/{+path}","compare_url":"https://api.github.com/repos/edx/configuration/compare/{base}...{head}","merges_url":"https://api.github.com/repos/edx/configuration/merges","archive_url":"https://api.github.com/repos/edx/configuration/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/edx/configuration/downloads","issues_url":"https://api.github.com/repos/edx/configuration/issues{/number}","pulls_url":"https://api.github.com/repos/edx/configuration/pulls{/number}","milestones_url":"https://api.github.com/repos/edx/configuration/milestones{/number}","notifications_url":"https://api.github.com/repos/edx/configuration/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/edx/configuration/labels{/name}","releases_url":"https://api.github.com/repos/edx/configuration/releases{/id}","created_at":"2013-04-16T17:23:42Z","updated_at":"2014-12-18T14:30:31Z","pushed_at":"2014-12-18T14:33:00Z","git_url":"git://github.com/edx/configuration.git","ssh_url":"git@github.com:edx/configuration.git","clone_url":"https://github.com/edx/configuration.git","svn_url":"https://github.com/edx/configuration","homepage":"","size":37861,"stargazers_count":308,"watchers_count":308,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":268,"mirror_url":null,"open_issues_count":28,"forks":268,"open_issues":28,"watchers":308,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/edx/configuration/pulls/1749"},"html":{"href":"https://github.com/edx/configuration/pull/1749"},"issue":{"href":"https://api.github.com/repos/edx/configuration/issues/1749"},"comments":{"href":"https://api.github.com/repos/edx/configuration/issues/1749/comments"},"review_comments":{"href":"https://api.github.com/repos/edx/configuration/pulls/1749/comments"},"review_comment":{"href":"https://api.github.com/repos/edx/configuration/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/edx/configuration/pulls/1749/commits"},"statuses":{"href":"https://api.github.com/repos/edx/configuration/statuses/dbd25668b549ab85f84a04fc0188da4ec6ef755e"}}}}