Repository: getsentry/sentry-python

Issue:
AWS Lambda under Python 3.8 is currently broken
It seems they have done some updates on the runtime that broke our integration.


Discussion:
Seems to work again
![image](https://user-images.githubusercontent.com/152008/88281277-e711ea00-ccd6-11ea-90af-46fbed351e33.png)


Resolving patch:
diff --git a/sentry_sdk/integrations/aws_lambda.py b/sentry_sdk/integrations/aws_lambda.py
--- a/sentry_sdk/integrations/aws_lambda.py
+++ b/sentry_sdk/integrations/aws_lambda.py
@@ -1,7 +1,6 @@
 from datetime import datetime, timedelta
 from os import environ
 import sys
-import json
 
 from sentry_sdk.hub import Hub, _should_send_default_pii
 from sentry_sdk._compat import reraise
@@ -42,19 +41,15 @@ def sentry_init_error(*args, **kwargs):
         if integration is None:
             return init_error(*args, **kwargs)
 
-        # Fetch Initialization error details from arguments
-        error = json.loads(args[1])
-
         # If an integration is there, a client has to be there.
         client = hub.client  # type: Any
 
-        with hub.push_scope() as scope:
-            with capture_internal_exceptions():
+        with capture_internal_exceptions():
+            with hub.configure_scope() as scope:
                 scope.clear_breadcrumbs()
-            # Checking if there is any error/exception which is raised in the runtime
-            # environment from arguments and, re-raising it to capture it as an event.
-            if error.get("errorType"):
-                exc_info = sys.exc_info()
+
+            exc_info = sys.exc_info()
+            if exc_info and all(exc_info):
                 event, hint = event_from_exception(
                     exc_info,
                     client_options=client.options,
@@ -140,25 +135,39 @@ def __init__(self, timeout_warning=False):
     @staticmethod
     def setup_once():
         # type: () -> None
-        import __main__ as lambda_bootstrap  # type: ignore
-
-        pre_37 = True  # Python 3.6 or 2.7
-
-        if not hasattr(lambda_bootstrap, "handle_http_request"):
-            try:
-                import bootstrap as lambda_bootstrap  # type: ignore
 
-                pre_37 = False  # Python 3.7
-            except ImportError:
-                pass
+        # Python 2.7: Everything is in `__main__`.
+        #
+        # Python 3.7: If the bootstrap module is *already imported*, it is the
+        # one we actually want to use (no idea what's in __main__)
+        #
+        # On Python 3.8 bootstrap is also importable, but will be the same file
+        # as __main__ imported under a different name:
+        #
+        #     sys.modules['__main__'].__file__ == sys.modules['bootstrap'].__file__
+        #     sys.modules['__main__'] is not sys.modules['bootstrap']
+        #
+        # Such a setup would then make all monkeypatches useless.
+        if "bootstrap" in sys.modules:
+            lambda_bootstrap = sys.modules["bootstrap"]  # type: Any
+        elif "__main__" in sys.modules:
+            lambda_bootstrap = sys.modules["__main__"]
+        else:
+            logger.warning(
+                "Not running in AWS Lambda environment, "
+                "AwsLambdaIntegration disabled (could not find bootstrap module)"
+            )
+            return
 
         if not hasattr(lambda_bootstrap, "handle_event_request"):
             logger.warning(
                 "Not running in AWS Lambda environment, "
-                "AwsLambdaIntegration disabled"
+                "AwsLambdaIntegration disabled (could not find handle_event_request)"
             )
             return
 
+        pre_37 = hasattr(lambda_bootstrap, "handle_http_request")  # Python 3.6 or 2.7
+
         if pre_37:
             old_handle_event_request = lambda_bootstrap.handle_event_request
 


Test patch:
diff --git a/tests/integrations/aws_lambda/client.py b/tests/integrations/aws_lambda/client.py
new file mode 100644
--- /dev/null
+++ b/tests/integrations/aws_lambda/client.py
@@ -0,0 +1,148 @@
+import sys
+import os
+import shutil
+import tempfile
+import subprocess
+import boto3
+import uuid
+import base64
+
+
+def get_boto_client():
+    return boto3.client(
+        "lambda",
+        aws_access_key_id=os.environ["SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID"],
+        aws_secret_access_key=os.environ["SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY"],
+        region_name="us-east-1",
+    )
+
+
+def run_lambda_function(
+    client,
+    runtime,
+    code,
+    payload,
+    add_finalizer,
+    syntax_check=True,
+    timeout=30,
+    subprocess_kwargs=(),
+):
+    subprocess_kwargs = dict(subprocess_kwargs)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        test_lambda_py = os.path.join(tmpdir, "test_lambda.py")
+        with open(test_lambda_py, "w") as f:
+            f.write(code)
+
+        if syntax_check:
+            # Check file for valid syntax first, and that the integration does not
+            # crash when not running in Lambda (but rather a local deployment tool
+            # such as chalice's)
+            subprocess.check_call([sys.executable, test_lambda_py])
+
+        setup_cfg = os.path.join(tmpdir, "setup.cfg")
+        with open(setup_cfg, "w") as f:
+            f.write("[install]\nprefix=")
+
+        subprocess.check_call(
+            [sys.executable, "setup.py", "sdist", "-d", os.path.join(tmpdir, "..")],
+            **subprocess_kwargs
+        )
+
+        # https://docs.aws.amazon.com/lambda/latest/dg/lambda-python-how-to-create-deployment-package.html
+        subprocess.check_call(
+            "pip install ../*.tar.gz -t .", cwd=tmpdir, shell=True, **subprocess_kwargs
+        )
+        shutil.make_archive(os.path.join(tmpdir, "ball"), "zip", tmpdir)
+
+        fn_name = "test_function_{}".format(uuid.uuid4())
+
+        with open(os.path.join(tmpdir, "ball.zip"), "rb") as zip:
+            client.create_function(
+                FunctionName=fn_name,
+                Runtime=runtime,
+                Timeout=timeout,
+                Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
+                Handler="test_lambda.test_handler",
+                Code={"ZipFile": zip.read()},
+                Description="Created as part of testsuite for getsentry/sentry-python",
+            )
+
+        @add_finalizer
+        def delete_function():
+            client.delete_function(FunctionName=fn_name)
+
+        response = client.invoke(
+            FunctionName=fn_name,
+            InvocationType="RequestResponse",
+            LogType="Tail",
+            Payload=payload,
+        )
+
+        assert 200 <= response["StatusCode"] < 300, response
+        return response
+
+
+_REPL_CODE = """
+import os
+
+def test_handler(event, context):
+    line = {line!r}
+    if line.startswith(">>> "):
+        exec(line[4:])
+    elif line.startswith("$ "):
+        os.system(line[2:])
+    else:
+        print("Start a line with $ or >>>")
+
+    return b""
+"""
+
+try:
+    import click
+except ImportError:
+    pass
+else:
+
+    @click.command()
+    @click.option(
+        "--runtime", required=True, help="name of the runtime to use, eg python3.8"
+    )
+    @click.option("--verbose", is_flag=True, default=False)
+    def repl(runtime, verbose):
+        """
+        Launch a "REPL" against AWS Lambda to inspect their runtime.
+        """
+
+        cleanup = []
+        client = get_boto_client()
+
+        print("Start a line with `$ ` to run shell commands, or `>>> ` to run Python")
+
+        while True:
+            line = input()
+
+            response = run_lambda_function(
+                client,
+                runtime,
+                _REPL_CODE.format(line=line),
+                b"",
+                cleanup.append,
+                subprocess_kwargs={
+                    "stdout": subprocess.DEVNULL,
+                    "stderr": subprocess.DEVNULL,
+                }
+                if not verbose
+                else {},
+            )
+
+            for line in base64.b64decode(response["LogResult"]).splitlines():
+                print(line.decode("utf8"))
+
+            for f in cleanup:
+                f()
+
+            cleanup = []
+
+    if __name__ == "__main__":
+        repl()
diff --git a/tests/integrations/aws_lambda/test_aws.py b/tests/integrations/aws_lambda/test_aws.py
--- a/tests/integrations/aws_lambda/test_aws.py
+++ b/tests/integrations/aws_lambda/test_aws.py
@@ -1,11 +1,23 @@
+"""
+# AWS Lambda system tests
+
+This testsuite uses boto3 to upload actual lambda functions to AWS, execute
+them and assert some things about the externally observed behavior. What that
+means for you is that those tests won't run without AWS access keys:
+
+    export SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID=..
+    export SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY=...
+    export SENTRY_PYTHON_TEST_AWS_IAM_ROLE="arn:aws:iam::920901907255:role/service-role/lambda"
+
+If you need to debug a new runtime, use this REPL to figure things out:
+
+    pip3 install click
+    python3 tests/integrations/aws_lambda/client.py --runtime=python4.0
+"""
 import base64
 import json
 import os
 import re
-import shutil
-import subprocess
-import sys
-import uuid
 from textwrap import dedent
 
 import pytest
@@ -15,24 +27,27 @@
 LAMBDA_PRELUDE = """
 from __future__ import print_function
 
-import time
-
 from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
 import sentry_sdk
 import json
+import time
+
 from sentry_sdk.transport import HttpTransport
 
-FLUSH_EVENT = True
+def event_processor(event):
+    # AWS Lambda truncates the log output to 4kb. If you only need a
+    # subsection of the event, override this function in your test
+    # to print less to logs.
+    return event
 
 class TestTransport(HttpTransport):
     def _send_event(self, event):
-        # Delay event output like this to test proper shutdown
-        # Note that AWS Lambda truncates the log output to 4kb, so you better
-        # pray that your events are smaller than that or else tests start
-        # failing.
-        if FLUSH_EVENT:
-            time.sleep(1)
-        print("\\nEVENT:", json.dumps(event))
+        event = event_processor(event)
+        # Writing a single string to stdout holds the GIL (seems like) and
+        # therefore cannot be interleaved with other threads. This is why we
+        # explicitly add a newline at the end even though `print` would provide
+        # us one.
+        print("\\nEVENT: {}\\n".format(json.dumps(event)))
 
 def init_sdk(timeout_warning=False, **extra_init_args):
     sentry_sdk.init(
@@ -50,64 +65,31 @@ def lambda_client():
     if "SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID" not in os.environ:
         pytest.skip("AWS environ vars not set")
 
-    return boto3.client(
-        "lambda",
-        aws_access_key_id=os.environ["SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID"],
-        aws_secret_access_key=os.environ["SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY"],
-        region_name="us-east-1",
-    )
+    from tests.integrations.aws_lambda.client import get_boto_client
+
+    return get_boto_client()
 
 
 @pytest.fixture(params=["python3.6", "python3.7", "python3.8", "python2.7"])
-def run_lambda_function(tmpdir, lambda_client, request, relay_normalize):
-    if request.param == "python3.8":
-        pytest.xfail("Python 3.8 is currently broken")
-
-    def inner(code, payload, syntax_check=True):
-        runtime = request.param
-        tmpdir.ensure_dir("lambda_tmp").remove()
-        tmp = tmpdir.ensure_dir("lambda_tmp")
-
-        tmp.join("test_lambda.py").write(code)
-
-        # Check file for valid syntax first, and that the integration does not
-        # crash when not running in Lambda (but rather a local deployment tool
-        # such as chalice's)
-        if syntax_check:
-            subprocess.check_call([sys.executable, str(tmp.join("test_lambda.py"))])
-
-        tmp.join("setup.cfg").write("[install]\nprefix=")
-        subprocess.check_call([sys.executable, "setup.py", "sdist", "-d", str(tmpdir)])
-
-        # https://docs.aws.amazon.com/lambda/latest/dg/lambda-python-how-to-create-deployment-package.html
-        subprocess.check_call("pip install ../*.tar.gz -t .", cwd=str(tmp), shell=True)
-        shutil.make_archive(tmpdir.join("ball"), "zip", str(tmp))
-
-        fn_name = "test_function_{}".format(uuid.uuid4())
-
-        lambda_client.create_function(
-            FunctionName=fn_name,
-            Runtime=runtime,
-            Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
-            Handler="test_lambda.test_handler",
-            Code={"ZipFile": tmpdir.join("ball.zip").read(mode="rb")},
-            Description="Created as part of testsuite for getsentry/sentry-python",
-            Timeout=4,
-        )
+def lambda_runtime(request):
+    return request.param
 
-        @request.addfinalizer
-        def delete_function():
-            lambda_client.delete_function(FunctionName=fn_name)
 
-        response = lambda_client.invoke(
-            FunctionName=fn_name,
-            InvocationType="RequestResponse",
-            LogType="Tail",
-            Payload=payload,
+@pytest.fixture
+def run_lambda_function(request, lambda_client, lambda_runtime):
+    def inner(code, payload, timeout=30, syntax_check=True):
+        from tests.integrations.aws_lambda.client import run_lambda_function
+
+        response = run_lambda_function(
+            client=lambda_client,
+            runtime=lambda_runtime,
+            code=code,
+            payload=payload,
+            add_finalizer=request.addfinalizer,
+            timeout=timeout,
+            syntax_check=syntax_check,
         )
 
-        assert 200 <= response["StatusCode"] < 300, response
-
         events = []
 
         for line in base64.b64decode(response["LogResult"]).splitlines():
@@ -116,7 +98,6 @@ def delete_function():
                 continue
             line = line[len(b"EVENT: ") :]
             events.append(json.loads(line.decode("utf-8")))
-            relay_normalize(events[-1])
 
         return events, response
 
@@ -130,6 +111,10 @@ def test_basic(run_lambda_function):
             """
         init_sdk()
 
+        def event_processor(event):
+            # Delay event output like this to test proper shutdown
+            time.sleep(1)
+            return event
 
         def test_handler(event, context):
             raise Exception("something went wrong")
@@ -246,25 +231,24 @@ def test_handler(event, context):
     }
 
 
-def test_init_error(run_lambda_function):
+def test_init_error(run_lambda_function, lambda_runtime):
+    if lambda_runtime == "python2.7":
+        pytest.skip("initialization error not supported on Python 2.7")
+
     events, response = run_lambda_function(
         LAMBDA_PRELUDE
-        + dedent(
-            """
-        init_sdk()
-        func()
-
-        def test_handler(event, context):
-            return 0
-        """
+        + (
+            "def event_processor(event):\n"
+            '    return event["exception"]["values"][0]["value"]\n'
+            "init_sdk()\n"
+            "func()"
         ),
         b'{"foo": "bar"}',
         syntax_check=False,
     )
 
-    log_result = (base64.b64decode(response["LogResult"])).decode("utf-8")
-    expected_text = "name 'func' is not defined"
-    assert expected_text in log_result
+    (event,) = events
+    assert "name 'func' is not defined" in event
 
 
 def test_timeout_error(run_lambda_function):
@@ -273,8 +257,6 @@ def test_timeout_error(run_lambda_function):
         + dedent(
             """
         init_sdk(timeout_warning=True)
-        FLUSH_EVENT=False
-
 
         def test_handler(event, context):
             time.sleep(10)
@@ -282,15 +264,16 @@ def test_handler(event, context):
         """
         ),
         b'{"foo": "bar"}',
+        timeout=3,
     )
 
     (event,) = events
     assert event["level"] == "error"
     (exception,) = event["exception"]["values"]
     assert exception["type"] == "ServerlessTimeoutWarning"
-    assert (
-        exception["value"]
-        == "WARNING : Function is expected to get timed out. Configured timeout duration = 4 seconds."
+    assert exception["value"] in (
+        "WARNING : Function is expected to get timed out. Configured timeout duration = 4 seconds.",
+        "WARNING : Function is expected to get timed out. Configured timeout duration = 3 seconds.",
     )
 
     assert exception["mechanism"] == {"type": "threading", "handled": False}