Repository: getmoto/moto
Pull request #6778 (2023-09-06 20:19:14)

S3: fix Range request handling
Hello,

It seems AWS has a weird way of handling HTTP `Range` request for S3. It seems it does not follow the RFC, and is instead very forgiving, failing only in certain circumstances. 

Basically, if the range request is badly formatted or not following the RFC, S3 will just treat the request as a non-range request and will return the entire object with a 200 status code.
Only certain conditions will trigger an exception and 416, mainly if the beginning of the request is above the end of the object and the request is a valid Range request (meaning it can be parsed), or if the request is a "suffix" request and it ends with 0.

I've thoroughly tested those changes against AWS, and it's been reported by one of our users.

I've updated the code and added the tests for most of the weird cases I could think of.
If you feel like the logic could be handled better in `_handle_range_header`, I'm all ears, I just tried to return early and return a regular response if AWS would do it too. 


Patch:
diff --git a/docs/docs/configuration/environment_variables.rst b/docs/docs/configuration/environment_variables.rst
index a717abe92..6b128a4be 100644
--- a/docs/docs/configuration/environment_variables.rst
+++ b/docs/docs/configuration/environment_variables.rst
@@ -28,6 +28,8 @@ The following is a non-exhaustive list of the environment variables that can be
 +-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
 |                               |          |           |                                                                                                 |
 +-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
+| MOTO_EVENTS_INVOKE_HTTP       | str      |           | See :ref:`events`.                                                                              |
++-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
 | MOTO_S3_CUSTOM_ENDPOINTS      | str      |           | See :ref:`s3`.                                                                                  |
 +-------------------------------+----------+-----------+-------------------------------------------------------------------------------------------------+
 
diff --git a/moto/events/models.py b/moto/events/models.py
index 3800218d5..9e03b0a09 100644
--- a/moto/events/models.py
+++ b/moto/events/models.py
@@ -1,6 +1,7 @@
 import copy
 import os
 import re
+import requests
 import json
 import sys
 import warnings
@@ -11,6 +12,8 @@ from operator import lt, le, eq, ge, gt
 from typing import Any, Dict, List, Optional, Tuple
 
 from collections import OrderedDict
+
+from moto import settings
 from moto.core.exceptions import JsonRESTError
 from moto.core import BaseBackend, BackendDict, CloudFormationModel, BaseModel
 from moto.core.utils import (
@@ -142,6 +145,23 @@ class Rule(CloudFormationModel):
                     "EventBusName": arn.resource_id,
                 }
                 cross_account_backend.put_events([new_event])
+            elif arn.service == "events" and arn.resource_type == "api-destination":
+                if settings.events_invoke_http():
+                    api_destination = self._find_api_destination(arn.resource_id)
+                    request_parameters = target.get("HttpParameters", {})
+                    headers = request_parameters.get("HeaderParameters", {})
+                    qs_params = request_parameters.get("QueryStringParameters", {})
+                    query_string = "&".join(
+                        [f"{key}={val}" for key, val in qs_params.items()]
+                    )
+                    url = api_destination.invocation_endpoint + (
+                        f"?{query_string}" if query_string else ""
+                    )
+                    requests.request(
+                        method=api_destination.http_method,
+                        url=url,
+                        headers=headers,
+                    )
             else:
                 raise NotImplementedError(f"Expr not defined for {type(self)}")
 
@@ -170,6 +190,11 @@ class Rule(CloudFormationModel):
         if archive.uuid == archive_uuid:
             archive.events.append(event)
 
+    def _find_api_destination(self, resource_id: str) -> "Destination":
+        backend: "EventsBackend" = events_backends[self.account_id][self.region_name]
+        destination_name = resource_id.split("/")[0]
+        return backend.destinations[destination_name]
+
     def _send_to_sqs_queue(
         self, resource_id: str, event: Dict[str, Any], group_id: Optional[str] = None
     ) -> None:
@@ -1245,6 +1270,7 @@ class EventsBackend(BaseBackend):
          - EventBridge Archive
          - SQS Queue + FIFO Queue
          - Cross-region/account EventBus
+         - HTTP requests (only enabled when MOTO_EVENTS_INVOKE_HTTP=true)
         """
         num_events = len(events)
 
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
index 701400b72..1510ce112 100644
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -1169,25 +1169,35 @@ class S3Response(BaseResponse):
     ) -> TYPE_RESPONSE:
         length = len(response_content)
         last = length - 1
+
         _, rspec = request.headers.get("range").split("=")
         if "," in rspec:
-            raise NotImplementedError("Multiple range specifiers not supported")
+            return 200, response_headers, response_content
+
+        try:
+            begin, end = [int(i) if i else None for i in rspec.split("-")]
+        except ValueError:
+            # if we can't parse the Range header, S3 just treat the request as a non-range request
+            return 200, response_headers, response_content
 
-        def toint(i: Any) -> Optional[int]:
-            return int(i) if i else None
+        if (begin is None and end == 0) or (begin is not None and begin > last):
+            raise InvalidRange(
+                actual_size=str(length), range_requested=request.headers.get("range")
+            )
 
-        begin, end = map(toint, rspec.split("-"))
         if begin is not None:  # byte range
             end = last if end is None else min(end, last)
         elif end is not None:  # suffix byte range
             begin = length - min(end, length)
             end = last
         else:
-            return 400, response_headers, ""
-        if begin < 0 or end > last or begin > min(end, last):
-            raise InvalidRange(
-                actual_size=str(length), range_requested=request.headers.get("range")
-            )
+            # Treat as non-range request
+            return 200, response_headers, response_content
+
+        if begin > min(end, last):
+            # Treat as non-range request if after the logic is applied, the start of the range is greater than the end
+            return 200, response_headers, response_content
+
         response_headers["content-range"] = f"bytes {begin}-{end}/{length}"
         content = response_content[begin : end + 1]
         response_headers["content-length"] = len(content)
diff --git a/moto/settings.py b/moto/settings.py
index 89c421372..63bf6bf03 100644
--- a/moto/settings.py
+++ b/moto/settings.py
@@ -79,6 +79,10 @@ def ecs_new_arn_format() -> bool:
     return os.environ.get("MOTO_ECS_NEW_ARN", "true").lower() != "false"
 
 
+def events_invoke_http() -> bool:
+    return os.environ.get("MOTO_EVENTS_INVOKE_HTTP", "false").lower() == "true"
+
+
 def allow_unknown_region() -> bool:
     return os.environ.get("MOTO_ALLOW_NONEXISTENT_REGION", "false").lower() == "true"
 


Test patch:
diff --git a/tests/test_events/test_events_http_integration.py b/tests/test_events/test_events_http_integration.py
new file mode 100644
index 000000000..6d33ffc4d
--- /dev/null
+++ b/tests/test_events/test_events_http_integration.py
@@ -0,0 +1,84 @@
+import boto3
+import json
+import os
+import responses
+
+from moto import mock_events, settings
+from unittest import mock, SkipTest
+
+
+@mock_events
+@mock.patch.dict(os.environ, {"MOTO_EVENTS_INVOKE_HTTP": "true"})
+def test_invoke_http_request_on_event():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't intercept HTTP requests in ServerMode")
+    events = boto3.client("events", region_name="eu-west-1")
+
+    #
+    # Create API endpoint to invoke
+    response = events.create_connection(
+        Name="test",
+        Description="test description",
+        AuthorizationType="API_KEY",
+        AuthParameters={
+            "ApiKeyAuthParameters": {"ApiKeyName": "test", "ApiKeyValue": "test"}
+        },
+    )
+
+    destination_response = events.create_api_destination(
+        Name="test",
+        Description="test-description",
+        ConnectionArn=response.get("ConnectionArn"),
+        InvocationEndpoint="https://www.google.com",
+        HttpMethod="GET",
+    )
+    destination_arn = destination_response["ApiDestinationArn"]
+
+    #
+    # Create Rules when to invoke the connection
+    pattern = {"source": ["test-source"], "detail-type": ["test-detail-type"]}
+    rule_name = "test-event-rule"
+    events.put_rule(
+        Name=rule_name,
+        State="ENABLED",
+        EventPattern=json.dumps(pattern),
+    )
+
+    events.put_targets(
+        Rule=rule_name,
+        Targets=[
+            {
+                "Id": "123",
+                "Arn": destination_arn,
+                "HttpParameters": {
+                    "HeaderParameters": {"header1": "value1"},
+                    "QueryStringParameters": {"qs1": "qv2"},
+                },
+            }
+        ],
+    )
+
+    #
+    # Ensure we intercept HTTP requests
+    with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
+        # test that both json and urlencoded body are empty in matcher and in request
+        rsps.add(
+            method=responses.GET,
+            url="https://www.google.com/",
+            match=[
+                responses.matchers.header_matcher({"header1": "value1"}),
+                responses.matchers.query_param_matcher({"qs1": "qv2"}),
+            ],
+        )
+
+        #
+        # Invoke HTTP requests
+        events.put_events(
+            Entries=[
+                {
+                    "Source": "test-source",
+                    "DetailType": "test-detail-type",
+                    "Detail": "{}",
+                }
+            ]
+        )
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
index 962ea8602..766f14de1 100644
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -1004,6 +1004,51 @@ def test_ranged_get():
 
     assert key.content_length == 100
 
+    assert key.get(Range="bytes=0-0")["Body"].read() == b"0"
+    assert key.get(Range="bytes=1-1")["Body"].read() == b"1"
+
+    range_req = key.get(Range="bytes=1-0")
+    assert range_req["Body"].read() == rep * 10
+    # assert that the request was not treated as a range request
+    assert range_req["ResponseMetadata"]["HTTPStatusCode"] == 200
+    assert "ContentRange" not in range_req
+
+    range_req = key.get(Range="bytes=-1-")
+    assert range_req["Body"].read() == rep * 10
+    assert range_req["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    range_req = key.get(Range="bytes=0--1")
+    assert range_req["Body"].read() == rep * 10
+    assert range_req["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    range_req = key.get(Range="bytes=0-1,3-4,7-9")
+    assert range_req["Body"].read() == rep * 10
+    assert range_req["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    range_req = key.get(Range="bytes=-")
+    assert range_req["Body"].read() == rep * 10
+    assert range_req["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    with pytest.raises(ClientError) as ex:
+        key.get(Range="bytes=-0")
+    assert ex.value.response["Error"]["Code"] == "InvalidRange"
+    assert (
+        ex.value.response["Error"]["Message"]
+        == "The requested range is not satisfiable"
+    )
+    assert ex.value.response["Error"]["ActualObjectSize"] == "100"
+    assert ex.value.response["Error"]["RangeRequested"] == "bytes=-0"
+
+    with pytest.raises(ClientError) as ex:
+        key.get(Range="bytes=101-200")
+    assert ex.value.response["Error"]["Code"] == "InvalidRange"
+    assert (
+        ex.value.response["Error"]["Message"]
+        == "The requested range is not satisfiable"
+    )
+    assert ex.value.response["Error"]["ActualObjectSize"] == "100"
+    assert ex.value.response["Error"]["RangeRequested"] == "bytes=101-200"
+
 
 @mock_s3
 def test_policy():