Repository: localstack/localstack

Issue:
Integrate S3 startup into multiserver.py
In [this PR](https://github.com/localstack/localstack/pull/1200/files) we introduced a new way of starting S3 in a separate process.

In the meantime, the service startup has been significantly reworked, and it seems that we can integrate the S3 starter into the [`multiserver.py`](https://github.com/localstack/localstack/blob/master/localstack/utils/server/multiserver.py) based loading approach that loads multiple APIs in a single process, for performance reasons. We should look into that.


Discussion:

Resolving patch:
diff --git a/localstack/services/cloudformation/cloudformation_starter.py b/localstack/services/cloudformation/cloudformation_starter.py
--- a/localstack/services/cloudformation/cloudformation_starter.py
+++ b/localstack/services/cloudformation/cloudformation_starter.py
@@ -31,7 +31,6 @@
 from localstack.utils.bootstrap import setup_logging
 from localstack.utils.cloudformation import template_deployer
 from localstack.services.cloudformation import service_models
-from localstack.services.awslambda.lambda_api import BUCKET_MARKER_LOCAL
 
 LOG = logging.getLogger(__name__)
 
@@ -152,17 +151,6 @@ def apply_patches():
 
     parsing.MODEL_MAP.update(MODEL_MAP)
 
-    # Patch S3Backend.get_key method in moto to use S3 API from LocalStack
-
-    def get_key(self, bucket_name, key_name, version_id=None):
-        s3_client = aws_stack.connect_to_service('s3')
-        value = b''
-        if bucket_name != BUCKET_MARKER_LOCAL:
-            value = s3_client.get_object(Bucket=bucket_name, Key=key_name)['Body'].read()
-        return s3_models.FakeKey(name=key_name, value=value)
-
-    s3_models.S3Backend.get_key = get_key
-
     # Patch clean_json in moto
 
     def clean_json(resource_json, resources_map):
diff --git a/localstack/services/s3/s3_starter.py b/localstack/services/s3/s3_starter.py
--- a/localstack/services/s3/s3_starter.py
+++ b/localstack/services/s3/s3_starter.py
@@ -1,22 +1,15 @@
-import sys
 import types
 import logging
 import traceback
 from moto.s3 import models as s3_models
 from moto.s3 import responses as s3_responses
 from moto.s3.responses import (
-    minidom,
-    MalformedXML,
-    undo_clean_key_name
-)
-from moto.server import main as moto_main
+    minidom, MalformedXML, undo_clean_key_name)
 from localstack import config
 from localstack.constants import DEFAULT_PORT_S3_BACKEND
 from localstack.utils.aws import aws_stack
 from localstack.utils.common import wait_for_port_open
-from localstack.services.infra import (
-    get_service_protocol, start_proxy_for_service, do_run)
-from localstack.utils.bootstrap import setup_logging
+from localstack.services.infra import start_moto_server
 
 LOG = logging.getLogger(__name__)
 
@@ -45,12 +38,11 @@ def check_s3(expect_shutdown=False, print_error=False):
 
 def start_s3(port=None, backend_port=None, asynchronous=None, update_listener=None):
     port = port or config.PORT_S3
-    backend_port = DEFAULT_PORT_S3_BACKEND
-    cmd = '%s "%s" s3 -p %s -H 0.0.0.0' % (sys.executable, __file__, backend_port)
-    print('Starting mock S3 (%s port %s)...' % (get_service_protocol(), port))
-    start_proxy_for_service('s3', port, backend_port, update_listener)
-    env_vars = {'PYTHONPATH': ':'.join(sys.path)}
-    return do_run(cmd, asynchronous, env_vars=env_vars)
+    backend_port = backend_port or DEFAULT_PORT_S3_BACKEND
+    apply_patches()
+    return start_moto_server(
+        key='s3', name='S3', asynchronous=asynchronous,
+        port=port, backend_port=backend_port, update_listener=update_listener)
 
 
 def apply_patches():
@@ -174,8 +166,7 @@ def s3_bucket_response_delete_keys(self, request, body, bucket_name):
             key_name = k['key_name']
             version_id = k['version_id']
             success = self.backend.delete_key(
-                bucket_name, undo_clean_key_name(key_name), version_id
-            )
+                bucket_name, undo_clean_key_name(key_name), version_id)
 
             if success:
                 deleted_names.append({
@@ -185,24 +176,8 @@ def s3_bucket_response_delete_keys(self, request, body, bucket_name):
             else:
                 error_names.append(key_name)
 
-        return (
-            200,
-            {},
-            template.render(deleted=deleted_names, delete_errors=error_names),
-        )
+        return (200, {},
+            template.render(deleted=deleted_names, delete_errors=error_names))
 
     s3_responses.S3ResponseInstance._bucket_response_delete_keys = types.MethodType(
-        s3_bucket_response_delete_keys, s3_responses.S3ResponseInstance
-    )
-
-
-def main():
-    setup_logging()
-    # patch moto implementation
-    apply_patches()
-    # start API
-    sys.exit(moto_main())
-
-
-if __name__ == '__main__':
-    main()
+        s3_bucket_response_delete_keys, s3_responses.S3ResponseInstance)


Test patch:
diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py
--- a/tests/integration/test_s3.py
+++ b/tests/integration/test_s3.py
@@ -419,7 +419,7 @@ def test_s3_delete_response_content_length_zero(self):
     def test_delete_object_tagging(self):
         bucket_name = 'test-%s' % short_uid()
         self.s3_client.create_bucket(Bucket=bucket_name, ACL='public-read')
-        object_key = 'test-key'
+        object_key = 'test-key-tagging'
         self.s3_client.put_object(Bucket=bucket_name, Key=object_key, Body='something')
         # get object and assert response
         url = '%s/%s/%s' % (config.TEST_S3_URL, bucket_name, object_key)
@@ -436,7 +436,7 @@ def test_delete_object_tagging(self):
     def test_delete_non_existing_keys(self):
         bucket_name = 'test-%s' % short_uid()
         self.s3_client.create_bucket(Bucket=bucket_name)
-        object_key = 'test-key'
+        object_key = 'test-key-nonexistent'
         self.s3_client.put_object(Bucket=bucket_name, Key=object_key, Body='something')
         response = self.s3_client.delete_objects(Bucket=bucket_name,
             Delete={'Objects': [{'Key': object_key}, {'Key': 'dummy1'}, {'Key': 'dummy2'}]})
@@ -686,16 +686,7 @@ def test_s3_event_notification_with_sqs(self):
         self.s3_client.put_bucket_versioning(Bucket=TEST_BUCKET_WITH_NOTIFICATION,
                                              VersioningConfiguration={'Status': 'Enabled'})
 
-        # flake8: noqa: W291
-        body = """ Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. 
-        Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. 
-        Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. 
-        Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a,. 
-        Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. 
-        Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. 
-        Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. 
-        Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. 
-        Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sed ipsum. """
+        body = 'Lorem ipsum dolor sit amet, ... ' * 30
 
         # put an object
         self.s3_client.put_object(Bucket=TEST_BUCKET_WITH_NOTIFICATION, Key=key_by_path, Body=body)
@@ -722,16 +713,7 @@ def test_s3_delete_object_with_version_id(self):
         test_1st_key = 'aws/s3/testkey1.txt'
         test_2nd_key = 'aws/s3/testkey2.txt'
 
-        # flake8: noqa: W291
-        body = """ Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. 
-        Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. 
-        Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. 
-        Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a,. 
-        Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. 
-        Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. 
-        Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. 
-        Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. 
-        Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sed ipsum. """
+        body = 'Lorem ipsum dolor sit amet, ... ' * 30
 
         self.s3_client.create_bucket(Bucket=TEST_BUCKET_WITH_VERSIONING)
         self.s3_client.put_bucket_versioning(Bucket=TEST_BUCKET_WITH_VERSIONING,