Repository: getmoto/moto

Issue:
[Logs] startFromHead has incorrect behavior
# Moto Version
1.3.9, though I see the issue on master

# Bug
The documented behavior for `startFromHead` in `GetLogEvents` is to return the _earliest_ events first, if the value is `True`. Moto inverts this behavior.
https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_GetLogEvents.html#API_GetLogEvents_RequestSyntax

# Repro:
```
    def test_fail(self, mock_logs):
        log_group = "1234"
        log_stream = "xyz"
        mock_logs.create_log_group(logGroupName=log_group)
        mock_logs.create_log_stream(logGroupName=log_group, logStreamName=log_stream)
        non_json_log = "normal non json message"
        json_log = {"json": "log"}
        mock_logs.put_log_events(
            logGroupName=log_group,
            logStreamName=log_stream,
            logEvents=[{"timestamp": 1234, "message": non_json_log}],
        )
        mock_logs.put_log_events(
            logGroupName=log_group,
            logStreamName=log_stream,
            logEvents=[{"timestamp": 1235, "message": json.dumps(json_log)}],
        )

        kwargs = {"logGroupName": log_group, "logStreamName": log_stream}
        kwargs["startFromHead"] = True
        response = mock_logs.get_log_events(**kwargs)
>       assert [e["message"] for e in response["events"]] == [
            non_json_log,
            json.dumps(json_log),
        ]
E       assert ['{"json":"lo...json message'] == ['normal non ...json":"log"}']
E         At index 0 diff: '{"json":"log"}' != 'normal non json message'
E         Full diff:
E         - ['{"json":"log"}', 'normal non json message']
E         + ['normal non json message', '{"json":"log"}']
```
# Line of code that's a problem:
https://github.com/spulec/moto/blob/master/moto/logs/models.py#L134

reverse should be `not startFromHead`

The problem is actually worse than this, because "reverse" pagination for CWL is actually iterating backwards by _page_ but having the log messages still forward ordered in time. So I'm not sure if `reverse` is event semantically correct.


Discussion:

Resolving patch:
diff --git a/moto/logs/models.py b/moto/logs/models.py
--- a/moto/logs/models.py
+++ b/moto/logs/models.py
@@ -1,7 +1,11 @@
 from moto.core import BaseBackend
 import boto.logs
 from moto.core.utils import unix_time_millis
-from .exceptions import ResourceNotFoundException, ResourceAlreadyExistsException
+from .exceptions import (
+    ResourceNotFoundException,
+    ResourceAlreadyExistsException,
+    InvalidParameterException,
+)
 
 
 class LogEvent:
@@ -118,41 +122,66 @@ def filter_func(event):
 
             return True
 
-        def get_paging_token_from_index(index, back=False):
-            if index is not None:
-                return "b/{:056d}".format(index) if back else "f/{:056d}".format(index)
-            return 0
-
-        def get_index_from_paging_token(token):
+        def get_index_and_direction_from_token(token):
             if token is not None:
-                return int(token[2:])
-            return 0
+                try:
+                    return token[0], int(token[2:])
+                except Exception:
+                    raise InvalidParameterException(
+                        "The specified nextToken is invalid."
+                    )
+            return None, 0
 
         events = sorted(
-            filter(filter_func, self.events),
-            key=lambda event: event.timestamp,
-            reverse=start_from_head,
+            filter(filter_func, self.events), key=lambda event: event.timestamp,
         )
-        next_index = get_index_from_paging_token(next_token)
-        back_index = next_index
 
-        events_page = [
-            event.to_response_dict()
-            for event in events[next_index : next_index + limit]
-        ]
-        if next_index + limit < len(self.events):
-            next_index += limit
+        direction, index = get_index_and_direction_from_token(next_token)
+        limit_index = limit - 1
+        final_index = len(events) - 1
+
+        if direction is None:
+            if start_from_head:
+                start_index = 0
+                end_index = start_index + limit_index
+            else:
+                end_index = final_index
+                start_index = end_index - limit_index
+        elif direction == "f":
+            start_index = index + 1
+            end_index = start_index + limit_index
+        elif direction == "b":
+            end_index = index - 1
+            start_index = end_index - limit_index
         else:
-            next_index = len(self.events)
+            raise InvalidParameterException("The specified nextToken is invalid.")
+
+        if start_index < 0:
+            start_index = 0
+        elif start_index > final_index:
+            return (
+                [],
+                "b/{:056d}".format(final_index),
+                "f/{:056d}".format(final_index),
+            )
 
-        back_index -= limit
-        if back_index <= 0:
-            back_index = 0
+        if end_index > final_index:
+            end_index = final_index
+        elif end_index < 0:
+            return (
+                [],
+                "b/{:056d}".format(0),
+                "f/{:056d}".format(0),
+            )
+
+        events_page = [
+            event.to_response_dict() for event in events[start_index : end_index + 1]
+        ]
 
         return (
             events_page,
-            get_paging_token_from_index(back_index, True),
-            get_paging_token_from_index(next_index),
+            "b/{:056d}".format(start_index),
+            "f/{:056d}".format(end_index),
         )
 
     def filter_log_events(


Test patch:
diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py
--- a/tests/test_logs/test_logs.py
+++ b/tests/test_logs/test_logs.py
@@ -166,70 +166,202 @@ def test_delete_retention_policy():
 
 @mock_logs
 def test_get_log_events():
-    conn = boto3.client("logs", "us-west-2")
+    client = boto3.client("logs", "us-west-2")
     log_group_name = "test"
     log_stream_name = "stream"
-    conn.create_log_group(logGroupName=log_group_name)
-    conn.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
+    client.create_log_group(logGroupName=log_group_name)
+    client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
 
     events = [{"timestamp": x, "message": str(x)} for x in range(20)]
 
-    conn.put_log_events(
+    client.put_log_events(
         logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=events
     )
 
-    resp = conn.get_log_events(
+    resp = client.get_log_events(
         logGroupName=log_group_name, logStreamName=log_stream_name, limit=10
     )
 
     resp["events"].should.have.length_of(10)
-    resp.should.have.key("nextForwardToken")
-    resp.should.have.key("nextBackwardToken")
+    for i in range(10):
+        resp["events"][i]["timestamp"].should.equal(i + 10)
+        resp["events"][i]["message"].should.equal(str(i + 10))
     resp["nextForwardToken"].should.equal(
-        "f/00000000000000000000000000000000000000000000000000000010"
+        "f/00000000000000000000000000000000000000000000000000000019"
     )
     resp["nextBackwardToken"].should.equal(
-        "b/00000000000000000000000000000000000000000000000000000000"
+        "b/00000000000000000000000000000000000000000000000000000010"
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextBackwardToken"],
+        limit=20,
     )
+
+    resp["events"].should.have.length_of(10)
     for i in range(10):
         resp["events"][i]["timestamp"].should.equal(i)
         resp["events"][i]["message"].should.equal(str(i))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000009"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000000"
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextBackwardToken"],
+        limit=10,
+    )
 
-    next_token = resp["nextForwardToken"]
+    resp["events"].should.have.length_of(0)
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000000"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000000"
+    )
 
-    resp = conn.get_log_events(
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextForwardToken"],
+        limit=1,
+    )
+
+    resp["events"].should.have.length_of(1)
+    resp["events"][0]["timestamp"].should.equal(1)
+    resp["events"][0]["message"].should.equal(str(1))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000001"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000001"
+    )
+
+
+@mock_logs
+def test_get_log_events_with_start_from_head():
+    client = boto3.client("logs", "us-west-2")
+    log_group_name = "test"
+    log_stream_name = "stream"
+    client.create_log_group(logGroupName=log_group_name)
+    client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
+
+    events = [{"timestamp": x, "message": str(x)} for x in range(20)]
+
+    client.put_log_events(
+        logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=events
+    )
+
+    resp = client.get_log_events(
         logGroupName=log_group_name,
         logStreamName=log_stream_name,
-        nextToken=next_token,
         limit=10,
+        startFromHead=True,  # this parameter is only relevant without the usage of nextToken
     )
 
     resp["events"].should.have.length_of(10)
-    resp.should.have.key("nextForwardToken")
-    resp.should.have.key("nextBackwardToken")
+    for i in range(10):
+        resp["events"][i]["timestamp"].should.equal(i)
+        resp["events"][i]["message"].should.equal(str(i))
     resp["nextForwardToken"].should.equal(
-        "f/00000000000000000000000000000000000000000000000000000020"
+        "f/00000000000000000000000000000000000000000000000000000009"
     )
     resp["nextBackwardToken"].should.equal(
         "b/00000000000000000000000000000000000000000000000000000000"
     )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextForwardToken"],
+        limit=20,
+    )
+
+    resp["events"].should.have.length_of(10)
     for i in range(10):
         resp["events"][i]["timestamp"].should.equal(i + 10)
         resp["events"][i]["message"].should.equal(str(i + 10))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000019"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000010"
+    )
 
-    resp = conn.get_log_events(
+    resp = client.get_log_events(
         logGroupName=log_group_name,
         logStreamName=log_stream_name,
-        nextToken=resp["nextBackwardToken"],
+        nextToken=resp["nextForwardToken"],
         limit=10,
     )
 
-    resp["events"].should.have.length_of(10)
-    resp.should.have.key("nextForwardToken")
-    resp.should.have.key("nextBackwardToken")
-    for i in range(10):
-        resp["events"][i]["timestamp"].should.equal(i)
-        resp["events"][i]["message"].should.equal(str(i))
+    resp["events"].should.have.length_of(0)
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000019"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000019"
+    )
+
+    resp = client.get_log_events(
+        logGroupName=log_group_name,
+        logStreamName=log_stream_name,
+        nextToken=resp["nextBackwardToken"],
+        limit=1,
+    )
+
+    resp["events"].should.have.length_of(1)
+    resp["events"][0]["timestamp"].should.equal(18)
+    resp["events"][0]["message"].should.equal(str(18))
+    resp["nextForwardToken"].should.equal(
+        "f/00000000000000000000000000000000000000000000000000000018"
+    )
+    resp["nextBackwardToken"].should.equal(
+        "b/00000000000000000000000000000000000000000000000000000018"
+    )
+
+
+@mock_logs
+def test_get_log_events_errors():
+    client = boto3.client("logs", "us-west-2")
+    log_group_name = "test"
+    log_stream_name = "stream"
+    client.create_log_group(logGroupName=log_group_name)
+    client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name)
+
+    with assert_raises(ClientError) as e:
+        client.get_log_events(
+            logGroupName=log_group_name,
+            logStreamName=log_stream_name,
+            nextToken="n/00000000000000000000000000000000000000000000000000000000",
+        )
+    ex = e.exception
+    ex.operation_name.should.equal("GetLogEvents")
+    ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    ex.response["Error"]["Code"].should.equal("InvalidParameterException")
+    ex.response["Error"]["Message"].should.contain(
+        "The specified nextToken is invalid."
+    )
+
+    with assert_raises(ClientError) as e:
+        client.get_log_events(
+            logGroupName=log_group_name,
+            logStreamName=log_stream_name,
+            nextToken="not-existing-token",
+        )
+    ex = e.exception
+    ex.operation_name.should.equal("GetLogEvents")
+    ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    ex.response["Error"]["Code"].should.equal("InvalidParameterException")
+    ex.response["Error"]["Message"].should.contain(
+        "The specified nextToken is invalid."
+    )
 
 
 @mock_logs