Repository: getmoto/moto

Issue:
ExpiredObjectDeleteMarker not supported in bucket lifecycle rules
I can do a put against a mocked BucketLifecycleConfiguration with ExpiredObjectDeleteMarker set in the Expiration, but then subsequent accessor to rule doesn't show.  

Looks like the LifecycleRule in models.py doesn't have include attributes to model this flag.
https://github.com/spulec/moto/blob/770281aef2132de8b4d8abb7ec3325a3038b6f09/moto/s3/models.py#L312

######
  mocked_s3 = boto3.resource('s3', region_name='us-east-1')
  mocked_s3.create_bucket(Bucket='mybucket')
  bucket_lifecycle_configuration = mocked_s3.BucketLifecycleConfiguration('mybucket')
  bucket_lifecycle_configuration.put(LifecycleConfiguration={
    'Rules': [
      {
        'Expiration': {
          'Days': 123,
          'ExpiredObjectDeleteMarker': True
        },
        'ID': 'id1',
        'Status': 'Enabled'
      }
    ]
  })

  bucket_lifecycle_configuration2 = mocked_s3.BucketLifecycleConfiguration('mybucket')
  assert bucket_lifecycle_configuration2.rules[0]['Expiration']['ExpiredObjectDeleteMarker'] == True


Discussion:

Resolving patch:
diff --git a/moto/s3/models.py b/moto/s3/models.py
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@@ -15,7 +15,7 @@
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
 from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \
-    InvalidNotificationDestination
+    InvalidNotificationDestination, MalformedXML
 from .utils import clean_key_name, _VersionedKeyStore
 
 UPLOAD_ID_BYTES = 43
@@ -311,18 +311,35 @@ def __init__(self, key, value=None):
         self.value = value
 
 
+class LifecycleFilter(BaseModel):
+
+    def __init__(self, prefix=None, tag=None, and_filter=None):
+        self.prefix = prefix or ''
+        self.tag = tag
+        self.and_filter = and_filter
+
+
+class LifecycleAndFilter(BaseModel):
+
+    def __init__(self, prefix=None, tags=None):
+        self.prefix = prefix or ''
+        self.tags = tags
+
+
 class LifecycleRule(BaseModel):
 
-    def __init__(self, id=None, prefix=None, status=None, expiration_days=None,
-                 expiration_date=None, transition_days=None,
+    def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None,
+                 expiration_date=None, transition_days=None, expired_object_delete_marker=None,
                  transition_date=None, storage_class=None):
         self.id = id
         self.prefix = prefix
+        self.filter = lc_filter
         self.status = status
         self.expiration_days = expiration_days
         self.expiration_date = expiration_date
         self.transition_days = transition_days
         self.transition_date = transition_date
+        self.expired_object_delete_marker = expired_object_delete_marker
         self.storage_class = storage_class
 
 
@@ -387,12 +404,50 @@ def set_lifecycle(self, rules):
         for rule in rules:
             expiration = rule.get('Expiration')
             transition = rule.get('Transition')
+
+            eodm = None
+            if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None:
+                # This cannot be set if Date or Days is set:
+                if expiration.get("Days") or expiration.get("Date"):
+                    raise MalformedXML()
+                eodm = expiration["ExpiredObjectDeleteMarker"]
+
+            # Pull out the filter:
+            lc_filter = None
+            if rule.get("Filter"):
+                # Can't have both `Filter` and `Prefix` (need to check for the presence of the key):
+                try:
+                    if rule["Prefix"] or not rule["Prefix"]:
+                        raise MalformedXML()
+                except KeyError:
+                    pass
+
+                and_filter = None
+                if rule["Filter"].get("And"):
+                    and_tags = []
+                    if rule["Filter"]["And"].get("Tag"):
+                        if not isinstance(rule["Filter"]["And"]["Tag"], list):
+                            rule["Filter"]["And"]["Tag"] = [rule["Filter"]["And"]["Tag"]]
+
+                        for t in rule["Filter"]["And"]["Tag"]:
+                            and_tags.append(FakeTag(t["Key"], t.get("Value", '')))
+
+                    and_filter = LifecycleAndFilter(prefix=rule["Filter"]["And"]["Prefix"], tags=and_tags)
+
+                filter_tag = None
+                if rule["Filter"].get("Tag"):
+                    filter_tag = FakeTag(rule["Filter"]["Tag"]["Key"], rule["Filter"]["Tag"].get("Value", ''))
+
+                lc_filter = LifecycleFilter(prefix=rule["Filter"]["Prefix"], tag=filter_tag, and_filter=and_filter)
+
             self.rules.append(LifecycleRule(
                 id=rule.get('ID'),
                 prefix=rule.get('Prefix'),
+                lc_filter=lc_filter,
                 status=rule['Status'],
                 expiration_days=expiration.get('Days') if expiration else None,
                 expiration_date=expiration.get('Date') if expiration else None,
+                expired_object_delete_marker=eodm,
                 transition_days=transition.get('Days') if transition else None,
                 transition_date=transition.get('Date') if transition else None,
                 storage_class=transition[
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -1176,7 +1176,30 @@ def _key_response_post(self, request, body, bucket_name, query, key_name, header
     {% for rule in rules %}
     <Rule>
         <ID>{{ rule.id }}</ID>
+        {% if rule.filter %}
+        <Filter>
+            <Prefix>{{ rule.filter.prefix }}</Prefix>
+            {% if rule.filter.tag %}
+            <Tag>
+                <Key>{{ rule.filter.tag.key }}</Key>
+                <Value>{{ rule.filter.tag.value }}</Value>
+            </Tag>
+            {% endif %}
+            {% if rule.filter.and_filter %}
+            <And>
+                <Prefix>{{ rule.filter.and_filter.prefix }}</Prefix>
+                {% for tag in rule.filter.and_filter.tags %}
+                <Tag>
+                    <Key>{{ tag.key }}</Key>
+                    <Value>{{ tag.value }}</Value>
+                </Tag>
+                {% endfor %}
+            </And>
+            {% endif %}
+        </Filter>
+        {% else %}
         <Prefix>{{ rule.prefix if rule.prefix != None }}</Prefix>
+        {% endif %}
         <Status>{{ rule.status }}</Status>
         {% if rule.storage_class %}
         <Transition>
@@ -1189,7 +1212,7 @@ def _key_response_post(self, request, body, bucket_name, query, key_name, header
            <StorageClass>{{ rule.storage_class }}</StorageClass>
         </Transition>
         {% endif %}
-        {% if rule.expiration_days or rule.expiration_date %}
+        {% if rule.expiration_days or rule.expiration_date or rule.expired_object_delete_marker %}
         <Expiration>
             {% if rule.expiration_days %}
                <Days>{{ rule.expiration_days }}</Days>
@@ -1197,6 +1220,9 @@ def _key_response_post(self, request, body, bucket_name, query, key_name, header
             {% if rule.expiration_date %}
                <Date>{{ rule.expiration_date }}</Date>
             {% endif %}
+            {% if rule.expired_object_delete_marker %}
+                <ExpiredObjectDeleteMarker>{{ rule.expired_object_delete_marker }}</ExpiredObjectDeleteMarker>
+            {% endif %}
         </Expiration>
         {% endif %}
     </Rule>
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -8,8 +8,8 @@
 install_requires = [
     "Jinja2>=2.7.3",
     "boto>=2.36.0",
-    "boto3>=1.2.1",
-    "botocore>=1.7.12",
+    "boto3>=1.6.16",
+    "botocore>=1.9.16",
     "cookies",
     "cryptography>=2.0.0",
     "requests>=2.5",


Test patch:
diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py
--- a/tests/test_s3/test_s3_lifecycle.py
+++ b/tests/test_s3/test_s3_lifecycle.py
@@ -1,12 +1,16 @@
 from __future__ import unicode_literals
 
 import boto
+import boto3
 from boto.exception import S3ResponseError
 from boto.s3.lifecycle import Lifecycle, Transition, Expiration, Rule
 
 import sure  # noqa
+from botocore.exceptions import ClientError
+from datetime import datetime
+from nose.tools import assert_raises
 
-from moto import mock_s3_deprecated
+from moto import mock_s3_deprecated, mock_s3
 
 
 @mock_s3_deprecated
@@ -26,6 +30,167 @@ def test_lifecycle_create():
     list(lifecycle.transition).should.equal([])
 
 
+@mock_s3
+def test_lifecycle_with_filters():
+    client = boto3.client("s3")
+    client.create_bucket(Bucket="bucket")
+
+    # Create a lifecycle rule with a Filter (no tags):
+    lfc = {
+        "Rules": [
+            {
+                "Expiration": {
+                    "Days": 7
+                },
+                "ID": "wholebucket",
+                "Filter": {
+                    "Prefix": ""
+                },
+                "Status": "Enabled"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ''
+    assert not result["Rules"][0]["Filter"].get("And")
+    assert not result["Rules"][0]["Filter"].get("Tag")
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # With a tag:
+    lfc["Rules"][0]["Filter"]["Tag"] = {
+        "Key": "mytag",
+        "Value": "mytagvalue"
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ''
+    assert not result["Rules"][0]["Filter"].get("And")
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # With And (single tag):
+    lfc["Rules"][0]["Filter"]["And"] = {
+        "Prefix": "some/prefix",
+        "Tags": [
+            {
+                "Key": "mytag",
+                "Value": "mytagvalue"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ""
+    assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix"
+    assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 1
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue"
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # With multiple And tags:
+    lfc["Rules"][0]["Filter"]["And"] = {
+        "Prefix": "some/prefix",
+        "Tags": [
+            {
+                "Key": "mytag",
+                "Value": "mytagvalue"
+            },
+            {
+                "Key": "mytag2",
+                "Value": "mytagvalue2"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ""
+    assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix"
+    assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 2
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue"
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Key"] == "mytag2"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Value"] == "mytagvalue2"
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # Can't have both filter and prefix:
+    lfc["Rules"][0]["Prefix"] = ''
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+
+    lfc["Rules"][0]["Prefix"] = 'some/path'
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+
+    # No filters -- just a prefix:
+    del lfc["Rules"][0]["Filter"]
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert not result["Rules"][0].get("Filter")
+    assert result["Rules"][0]["Prefix"] == "some/path"
+
+
+@mock_s3
+def test_lifecycle_with_eodm():
+    client = boto3.client("s3")
+    client.create_bucket(Bucket="bucket")
+
+    lfc = {
+        "Rules": [
+            {
+                "Expiration": {
+                    "ExpiredObjectDeleteMarker": True
+                },
+                "ID": "wholebucket",
+                "Filter": {
+                    "Prefix": ""
+                },
+                "Status": "Enabled"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"]
+
+    # Set to False:
+    lfc["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"] = False
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert not result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"]
+
+    # With failure:
+    lfc["Rules"][0]["Expiration"]["Days"] = 7
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+    del lfc["Rules"][0]["Expiration"]["Days"]
+
+    lfc["Rules"][0]["Expiration"]["Date"] = datetime(2015, 1, 1)
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+
+
 @mock_s3_deprecated
 def test_lifecycle_with_glacier_transition():
     conn = boto.s3.connect_to_region("us-west-1")