Repository: cloud-custodian/cloud-custodian

Issue:
Support cross region snapshot transfer
should support non-default cmk encryption (entails full transfer instead of incremental but that was the request), else support incremental transfer.



Discussion:

Resolving patch:
diff --git a/c7n/resources/ebs.py b/c7n/resources/ebs.py
--- a/c7n/resources/ebs.py
+++ b/c7n/resources/ebs.py
@@ -18,7 +18,8 @@
 
 from c7n.actions import ActionRegistry, BaseAction
 from c7n.filters import (
-    FilterRegistry, AgeFilter, ValueFilter, ANNOTATION_KEY)
+    FilterRegistry, AgeFilter, ValueFilter, ANNOTATION_KEY,
+    FilterValidationError)
 
 from c7n.manager import resources
 from c7n.query import QueryResourceManager, ResourceQuery
@@ -58,7 +59,7 @@ class SnapshotDelete(BaseAction):
 
     def process(self, snapshots):
         self.image_snapshots = snaps = set()
-        # Be careful re image snapshots, we do this by default
+         # Be careful re image snapshots, we do this by default
         # to keep things safe by default, albeit we'd get an error
         # if we did try to delete something associated to an image.
         if self.data.get('skip-ami-snapshots', True):
@@ -96,12 +97,63 @@ def process_snapshot_set(self, snapshots_set):
                 raise
 
 
+@Snapshot.action_registry.register('copy')
+class CopySnapshot(BaseAction):
+    """Copy a snapshot across regions
+
+    http://goo.gl/CP3dq
+    """
+
+    schema = type_schema(
+        'copy',
+        target_region={'type': 'string'},
+        target_key={'type': 'string'},
+        encrypted={'type': 'boolean'},
+    )
+
+    def validate(self):
+        if self.data.get('encrypted', True):
+            key = self.data.get('target_key')
+            if not key:
+                raise FilterValidationError(
+                    "Encrypted snapshot copy requires kms key")
+        return self
+
+    def process(self, resources):
+        if self.data['target_region'] == self.manager.config.region:
+            self.log.info(
+                "Source and destination region are the same, skipping")
+            return
+        with self.executor_factory(max_workers=2) as w:
+            list(w.map(self.process_resource_set, chunks(resources, 20)))
+
+    def process_resource_set(self, resource_set):
+        client = self.manager.session_factory(
+            region=self.data['target_region']).client('ec2')
+
+        params = {}
+        params['Encrypted'] = self.data.get('encrypted', True)
+        if params['Encrypted']:
+            params['KmsKeyId'] = self.data['target_key']
+
+        for r in resource_set:
+            snapshot_id = client.copy_snapshot(
+                SourceRegion=self.manager.config.region,
+                SourceSnapshotId=r['SnapshotId'],
+                Description=r.get('Description', ''),
+                **params)['SnapshotId']
+            client.create_tags(
+                Resources=[snapshot_id],
+                Tags=r['Tags'])
+            r['CopiedSnapshot'] = snapshot_id
+
+
 @resources.register('ebs')
 class EBS(QueryResourceManager):
 
     class resource_type(ResourceQuery.resolve("aws.ec2.volume")):
         default_namespace = 'AWS/EBS'
-    
+
     filter_registry = filters
     action_registry = actions
 


Test patch:
diff --git a/tests/data/placebo/test_ebs_snapshot_copy/ec2.CopySnapshot_1.json b/tests/data/placebo/test_ebs_snapshot_copy/ec2.CopySnapshot_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ebs_snapshot_copy/ec2.CopySnapshot_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "SnapshotId": "snap-7fc4f06d", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "bbd827ba-d0f1-436f-bffb-77ba112952f1", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Mon, 27 Jun 2016 13:03:28 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ebs_snapshot_copy/ec2.CreateTags_1.json b/tests/data/placebo/test_ebs_snapshot_copy/ec2.CreateTags_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ebs_snapshot_copy/ec2.CreateTags_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "047c2196-d54f-4a27-bf51-aa3a4430e739", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Mon, 27 Jun 2016 13:03:28 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ebs_snapshot_copy/ec2.DescribeSnapshots_1.json b/tests/data/placebo/test_ebs_snapshot_copy/ec2.DescribeSnapshots_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ebs_snapshot_copy/ec2.DescribeSnapshots_1.json
@@ -0,0 +1,53 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "72b653cc-1dd0-4bb4-840b-60500481aa17", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Mon, 27 Jun 2016 13:03:27 GMT"
+            }
+        }, 
+        "Snapshots": [
+            {
+                "Description": "Nance", 
+                "Tags": [
+                    {
+                        "Value": "BlockBiddy", 
+                        "Key": "Name"
+                    }, 
+                    {
+                        "Value": "RoadKill", 
+                        "Key": "ASV"
+                    }, 
+                    {
+                        "Value": "Toasty", 
+                        "Key": "Platforum"
+                    }
+                ], 
+                "Encrypted": true, 
+                "VolumeId": "vol-cc3fc645", 
+                "KmsKeyId": "arn:aws:kms:us-west-2:644160558196:key/d21dba12-ec3d-4115-a600-5779ae051580", 
+                "State": "completed", 
+                "VolumeSize": 5, 
+                "Progress": "100%", 
+                "StartTime": {
+                    "hour": 11, 
+                    "__class__": "datetime", 
+                    "month": 6, 
+                    "second": 36, 
+                    "microsecond": 0, 
+                    "year": 2016, 
+                    "day": 27, 
+                    "minute": 11
+                }, 
+                "SnapshotId": "snap-f8c07c02", 
+                "OwnerId": "644160558196"
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ebs_snapshot_copy/ec2.DescribeTags_1.json b/tests/data/placebo/test_ebs_snapshot_copy/ec2.DescribeTags_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ebs_snapshot_copy/ec2.DescribeTags_1.json
@@ -0,0 +1,36 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "596ad0b3-092f-4936-8f29-705475ce9b91", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Mon, 27 Jun 2016 13:03:28 GMT"
+            }
+        }, 
+        "Tags": [
+            {
+                "ResourceType": "snapshot", 
+                "ResourceId": "snap-7fc4f06d", 
+                "Value": "RoadKill", 
+                "Key": "ASV"
+            }, 
+            {
+                "ResourceType": "snapshot", 
+                "ResourceId": "snap-7fc4f06d", 
+                "Value": "BlockBiddy", 
+                "Key": "Name"
+            }, 
+            {
+                "ResourceType": "snapshot", 
+                "ResourceId": "snap-7fc4f06d", 
+                "Value": "Toasty", 
+                "Key": "Platforum"
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/tests/test_ebs.py b/tests/test_ebs.py
--- a/tests/test_ebs.py
+++ b/tests/test_ebs.py
@@ -15,13 +15,44 @@
 
 from .common import BaseTest
 from c7n.resources.ebs import (
-    CopyInstanceTags, EncryptInstanceVolumes)
+    CopyInstanceTags, EncryptInstanceVolumes, CopySnapshot)
 from c7n.executor import MainThreadExecutor
 
 
 logging.basicConfig(level=logging.DEBUG)
 
 
+class SnapshotCopyTest(BaseTest):
+
+    def test_snapshot_copy(self):
+        self.patch(CopySnapshot, 'executor_factory', MainThreadExecutor)
+        # DEFAULT_REGION needs to be set to west for recording
+        factory = self.replay_flight_data('test_ebs_snapshot_copy')
+        p = self.load_policy({
+            'name': 'snap-copy',
+            'resource': 'ebs-snapshot',
+            'filters': [
+                {'tag:ASV': 'RoadKill'}],
+            'actions': [
+                {'type': 'copy',
+                 'target_region': 'us-east-1',
+                 'target_key': '82645407-2faa-4d93-be71-7d6a8d59a5fc'}]
+            }, session_factory=factory)
+        resources = p.run()
+        # If test region is target region aka us-east-1, then the action
+        # skips, and so does the test
+        if factory().region_name == 'us-east-1':
+            return
+    
+        self.assertEqual(len(resources), 1)
+        client = factory(region="us-east-1").client('ec2')
+        tags = client.describe_tags(
+            Filters=[{'Name': 'resource-id',
+                       'Values': [resources[0]['CopiedSnapshot']]}])['Tags']
+        tags = {t['Key']: t['Value'] for t in tags}
+        self.assertEqual(tags['ASV'], 'RoadKill')
+
+
 class SnapshotTrimTest(BaseTest):
 
     def test_snapshot_trim(self):
@@ -93,12 +124,12 @@ def test_encrypt_volumes(self):
         output = self.capture_logging(level=logging.DEBUG)
 
         session_factory = self.replay_flight_data('test_encrypt_volumes')
-        
+
         policy = self.load_policy({
             'name': 'ebs-remediate-attached',
             'resource': 'ebs',
             'filters': [
-                {'Encrypted': False},                
+                {'Encrypted': False},
                 {'VolumeId': 'vol-fdd1f844'}],
             'actions': [
                 {'type': 'encrypt-instance-volumes',
diff --git a/tests/zpill.py b/tests/zpill.py
--- a/tests/zpill.py
+++ b/tests/zpill.py
@@ -161,6 +161,7 @@ def record_flight_data(self, test_case, zdata=False):
             os.makedirs(test_dir)
 
         session = boto3.Session()
+        default_region = session.region_name
         if not zdata:
             pill = placebo.attach(session, test_dir, debug=True)
         else:
@@ -169,7 +170,19 @@ def record_flight_data(self, test_case, zdata=False):
         pill.record()
         self.addCleanup(pill.stop)
         self.addCleanup(self.cleanUp)
+
+        def factory(region=None, assume=None):
+            if region and region != default_region:
+                new_session = boto3.Session(region_name=region)
+                assert not zdata
+                new_pill = placebo.attach(new_session, test_dir, debug=True)
+                new_pill.record()
+                self.addCleanup(new_pill.stop)
+                return new_session
+            return session
+
         # return session factory
+        return factory
         return lambda region=None, assume=None: session
 
     def replay_flight_data(self, test_case, zdata=False):