Repository: localstack/localstack

Issue:
S3 upload not triggering `s3:ObjectCreated:*` for Node.js uploads
It seems like I can't get `s3:ObjectCreated:*` to trigger for s3 events when making calls with the JavaScript AWS SDK. It works just fine when performing the same action using `boto3` and the `aws-cli`.

I peeked the source to check how this is done today, and noticed there's a todo referencing this issue: `# TODO: consider the case of hostname-based (as opposed to path-based) bucket addressing`. I'm working on a PR to fix it.


Discussion:

Resolving patch:
diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py
--- a/localstack/services/s3/s3_listener.py
+++ b/localstack/services/s3/s3_listener.py
@@ -10,6 +10,7 @@
 import botocore.config
 from requests.models import Response, Request
 from localstack.constants import DEFAULT_REGION
+from localstack.config import HOSTNAME, HOSTNAME_EXTERNAL
 from localstack.utils import persistence
 from localstack.utils.aws import aws_stack
 from localstack.utils.common import short_uid, timestamp, TIMESTAMP_FORMAT_MILLIS, to_str, to_bytes, clone
@@ -269,6 +270,47 @@ def expand_redirect_url(starting_url, key, bucket):
     return redirect_url
 
 
+def get_bucket_name(path, headers):
+    parsed = urlparse.urlparse(path)
+
+    # try pick the bucket_name from the path
+    bucket_name = parsed.path.split('/')[1]
+
+    host = headers['host']
+
+    # is the hostname not starting a bucket name?
+    if host.startswith(HOSTNAME) or host.startswith(HOSTNAME_EXTERNAL):
+        return bucket_name
+
+    # matches the common endpoints like
+    #     - '<bucket_name>.s3.<region>.amazonaws.com'
+    #     - '<bucket_name>.s3-<region>.amazonaws.com.cn'
+    common_pattern = re.compile(r'^(.+)\.s3[.\-][a-z]{2}-[a-z]+-[0-9]{1,}'
+                                r'\.amazonaws\.com(\.[a-z]+)?$')
+    # matches dualstack endpoints like
+    #     - <bucket_name>.s3.dualstack.<region>.amazonaws.com'
+    #     - <bucket_name>.s3.dualstack.<region>.amazonaws.com.cn'
+    dualstack_pattern = re.compile(r'^(.+)\.s3\.dualstack\.[a-z]{2}-[a-z]+-[0-9]{1,}'
+                                   r'\.amazonaws\.com(\.[a-z]+)?$')
+    # matches legacy endpoints like
+    #     - '<bucket_name>.s3.amazonaws.com'
+    #     - '<bucket_name>.s3-external-1.amazonaws.com.cn'
+    legacy_patterns = re.compile(r'^(.+)\.s3\.?(-external-1)?\.amazonaws\.com(\.[a-z]+)?$')
+
+    # if any of the above patterns match, the first captured group
+    # will be returned as the bucket name
+    for pattern in [common_pattern, dualstack_pattern, legacy_patterns]:
+        match = pattern.match(host)
+        if match:
+            bucket_name = match.groups()[0]
+
+            break
+
+    # we're either returning the original bucket_name,
+    # or a pattern matched the host and we're returning that name instead
+    return bucket_name
+
+
 class ProxyListenerS3(ProxyListener):
 
     def forward_request(self, method, path, data, headers):
@@ -368,9 +410,7 @@ def forward_request(self, method, path, data, headers):
 
     def return_response(self, method, path, data, headers, response):
 
-        parsed = urlparse.urlparse(path)
-        # TODO: consider the case of hostname-based (as opposed to path-based) bucket addressing
-        bucket_name = parsed.path.split('/')[1]
+        bucket_name = get_bucket_name(path, headers)
 
         # POST requests to S3 may include a success_action_redirect field,
         # which should be used to redirect a client to a new location.
@@ -381,16 +421,28 @@ def return_response(self, method, path, data, headers, response):
                 response.headers['Location'] = expand_redirect_url(redirect_url, key, bucket_name)
                 LOGGER.debug('S3 POST {} to {}'.format(response.status_code, response.headers['Location']))
 
-        # get subscribers and send bucket notifications
-        if method in ('PUT', 'DELETE') and '/' in path[1:]:
+        parsed = urlparse.urlparse(path)
+
+        bucket_name_in_host = headers['host'].startswith(bucket_name)
+
+        should_send_notifications = all([
+            method in ('PUT', 'DELETE'),
+            '/' in path[1:] or bucket_name_in_host,
             # check if this is an actual put object request, because it could also be
             # a put bucket request with a path like this: /bucket_name/
-            if len(path[1:].split('/')[1]) > 0:
+            bucket_name_in_host or (len(path[1:].split('/')) > 1 and len(path[1:].split('/')[1]) > 0),
+            # ignore bucket notification configuration requests
+            parsed.query != 'notification' and parsed.query != 'lifecycle',
+        ])
+
+        # get subscribers and send bucket notifications
+        if should_send_notifications:
+            if bucket_name_in_host:
+                object_path = parsed.path
+            else:
                 parts = parsed.path[1:].split('/', 1)
-                # ignore bucket notification configuration requests
-                if parsed.query != 'notification' and parsed.query != 'lifecycle':
-                    object_path = parts[1] if parts[1][0] == '/' else '/%s' % parts[1]
-                    send_notifications(method, bucket_name, object_path)
+                object_path = parts[1] if parts[1][0] == '/' else '/%s' % parts[1]
+            send_notifications(method, bucket_name, object_path)
 
         # publish event for creation/deletion of buckets:
         if method in ('PUT', 'DELETE') and ('/' not in path[1:] or len(path[1:].split('/')[1]) <= 0):


Test patch:
diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py
--- a/tests/integration/test_s3.py
+++ b/tests/integration/test_s3.py
@@ -1,7 +1,11 @@
+import os
 import json
+import requests
 from localstack.utils.aws import aws_stack
 
 TEST_BUCKET_NAME_WITH_POLICY = 'test_bucket_policy_1'
+TEST_BUCKET_WITH_NOTIFICATION = 'test_bucket_notification_1'
+TEST_QUEUE_FOR_BUCKET_WITH_NOTIFICATION = 'test_queue_for_bucket_notification_1'
 
 
 def test_bucket_policy():
@@ -33,3 +37,46 @@ def test_bucket_policy():
     # retrieve and check policy config
     saved_policy = s3_client.get_bucket_policy(Bucket=TEST_BUCKET_NAME_WITH_POLICY)['Policy']
     assert json.loads(saved_policy) == policy
+
+
+def test_s3_put_object_notification():
+
+    s3_client = aws_stack.connect_to_service('s3')
+    sqs_client = aws_stack.connect_to_service('sqs')
+
+    key_by_path = 'key-by-hostname'
+    key_by_host = 'key-by-host'
+
+    # create test queue
+    queue_url = sqs_client.create_queue(QueueName=TEST_QUEUE_FOR_BUCKET_WITH_NOTIFICATION)['QueueUrl']
+    queue_attributes = sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=['QueueArn'])
+
+    # create test bucket
+    s3_client.create_bucket(Bucket=TEST_BUCKET_WITH_NOTIFICATION)
+    s3_client.put_bucket_notification_configuration(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
+                                                    NotificationConfiguration={'QueueConfigurations': [
+                                                        {'QueueArn': queue_attributes['Attributes']['QueueArn'],
+                                                         'Events': ['s3:ObjectCreated:*']}]})
+
+    # put an object where the bucket_name is in the path
+    s3_client.put_object(Bucket=TEST_BUCKET_WITH_NOTIFICATION, Key=key_by_path, Body='something')
+
+    # put an object where the bucket_name is in the host
+    # it doesn't care about the authorization header as long as it's present
+    headers = {'Host': '{}.s3.amazonaws.com'.format(TEST_BUCKET_WITH_NOTIFICATION), 'authorization': 'some_token'}
+    url = '{}/{}'.format(os.getenv('TEST_S3_URL'), key_by_host)
+    # verify=False must be set as this test fails on travis because of an SSL error non-existent locally
+    response = requests.put(url, data='something else', headers=headers, verify=False)
+    assert response.ok
+
+    queue_attributes = sqs_client.get_queue_attributes(QueueUrl=queue_url,
+                                                       AttributeNames=['ApproximateNumberOfMessages'])
+    message_count = queue_attributes['Attributes']['ApproximateNumberOfMessages']
+    # the ApproximateNumberOfMessages attribute is a string
+    assert message_count == '2'
+
+    # clean up
+    sqs_client.delete_queue(QueueUrl=queue_url)
+    s3_client.delete_objects(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
+                             Delete={'Objects': [{'Key': key_by_path}, {'Key': key_by_host}]})
+    s3_client.delete_bucket(Bucket=TEST_BUCKET_WITH_NOTIFICATION)
diff --git a/tests/unit/test_s3.py b/tests/unit/test_s3.py
--- a/tests/unit/test_s3.py
+++ b/tests/unit/test_s3.py
@@ -1,5 +1,6 @@
 import unittest
 from localstack.services.s3 import s3_listener, multipart_content
+from requests.models import CaseInsensitiveDict
 
 
 class S3ListenerTest (unittest.TestCase):
@@ -100,3 +101,69 @@ def test_expand_multipart_filename(self):
         self.assertIsNot(expanded3, data3, 'Should have changed content of string data with filename to interpolate')
         self.assertIn(b'uploads/20170826T181315.679087009Z/upload/pixel.txt', expanded3,
             'Should see the interpolated filename')
+
+    def test_get_bucket_name(self):
+        bucket_name = 'test-bucket'
+        s3_key = '/some-folder/some-key.txt'
+
+        hosts = ['s3-ap-northeast-1.amazonaws.com',
+                 's3-ap-northeast-2.amazonaws.com',
+                 's3-ap-south-1.amazonaws.com',
+                 's3-ap-southeast-1.amazonaws.com',
+                 's3-ap-southeast-2.amazonaws.com',
+                 's3-ca-central-1.amazonaws.com',
+                 's3-eu-central-1.amazonaws.com',
+                 's3-eu-west-1.amazonaws.com',
+                 's3-eu-west-2.amazonaws.com',
+                 's3-eu-west-3.amazonaws.com',
+                 's3-external-1.amazonaws.com',
+                 's3-sa-east-1.amazonaws.com',
+                 's3-us-east-2.amazonaws.com',
+                 's3-us-west-1.amazonaws.com',
+                 's3-us-west-2.amazonaws.com',
+                 's3.amazonaws.com',
+                 's3.ap-northeast-1.amazonaws.com',
+                 's3.ap-northeast-2.amazonaws.com',
+                 's3.ap-south-1.amazonaws.com',
+                 's3.ap-southeast-1.amazonaws.com',
+                 's3.ap-southeast-2.amazonaws.com',
+                 's3.ca-central-1.amazonaws.com',
+                 's3.cn-north-1.amazonaws.com.cn',
+                 's3.cn-northwest-1.amazonaws.com.cn',
+                 's3.dualstack.ap-northeast-1.amazonaws.com',
+                 's3.dualstack.ap-northeast-2.amazonaws.com',
+                 's3.dualstack.ap-south-1.amazonaws.com',
+                 's3.dualstack.ap-southeast-1.amazonaws.com',
+                 's3.dualstack.ap-southeast-2.amazonaws.com',
+                 's3.dualstack.ca-central-1.amazonaws.com',
+                 's3.dualstack.eu-central-1.amazonaws.com',
+                 's3.dualstack.eu-west-1.amazonaws.com',
+                 's3.dualstack.eu-west-2.amazonaws.com',
+                 's3.dualstack.eu-west-3.amazonaws.com',
+                 's3.dualstack.sa-east-1.amazonaws.com',
+                 's3.dualstack.us-east-1.amazonaws.com',
+                 's3.dualstack.us-east-2.amazonaws.com',
+                 's3.dualstack.us-west-1.amazonaws.com',
+                 's3.dualstack.us-west-2.amazonaws.com',
+                 's3.eu-central-1.amazonaws.com',
+                 's3.eu-west-1.amazonaws.com',
+                 's3.eu-west-2.amazonaws.com',
+                 's3.eu-west-3.amazonaws.com',
+                 's3.sa-east-1.amazonaws.com',
+                 's3.us-east-1.amazonaws.com',
+                 's3.us-east-2.amazonaws.com',
+                 's3.us-west-1.amazonaws.com',
+                 's3.us-west-2.amazonaws.com']
+
+        # test all available hosts with the bucket_name in the path
+        bucket_path = '/{}/{}'.format(bucket_name, s3_key)
+        for host in hosts:
+            headers = CaseInsensitiveDict({'Host': hosts[0]})
+            returned_bucket_name = s3_listener.get_bucket_name(bucket_path, headers)
+            self.assertEqual(returned_bucket_name, bucket_name, 'Should match when bucket_name is in path')
+
+        # test all available hosts with the bucket_name in the host and the path is only the s3_key
+        for host in hosts:
+            headers = CaseInsensitiveDict({'Host': '{}.{}'.format(bucket_name, host)})
+            returned_bucket_name = s3_listener.get_bucket_name(s3_key, headers)
+            self.assertEqual(returned_bucket_name, bucket_name, 'Should match when bucket_name is in the host')