Repository: localstack/localstack

Issue:
bug: Lambda state does not reflect internal state
While a lambda is still processing (archive unpacking etc), the lambda state should be set to "Pending" (currently "Active") and invocations to that lambda should fail (currently, it can end up in a inconsistent state in this case, like lambdas being executed without layers).

Lambda state should reflect the current internal state, and invocations should fail if not active.


Discussion:

Resolving patch:
diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py
--- a/localstack/services/awslambda/lambda_api.py
+++ b/localstack/services/awslambda/lambda_api.py
@@ -63,7 +63,6 @@
     now_utc,
     parse_request_data,
     run,
-    run_for_max_seconds,
     safe_requests,
     save_file,
     short_uid,
@@ -78,6 +77,7 @@
 from localstack.utils.functions import run_safe
 from localstack.utils.http import canonicalize_headers, parse_chunked_data
 from localstack.utils.patch import patch
+from localstack.utils.run import run_for_max_seconds
 
 LOG = logging.getLogger(__name__)
 
@@ -539,6 +539,14 @@ def run_lambda(
             result = not_found_error(msg="The resource specified in the request does not exist.")
             return InvocationResult(result)
 
+        if lambda_function.state != "Active":
+            result = error_response(
+                f"The operation cannot be performed at this time. The function is currently in the following state: {lambda_function.state}",
+                409,
+                "ResourceConflictException",
+            )
+            raise ClientError(result)
+
         context = LambdaContext(lambda_function, version, context)
         result = LAMBDA_EXECUTOR.execute(
             func_arn,
@@ -551,7 +559,8 @@ def run_lambda(
             lock_discriminator=lock_discriminator,
         )
         return result
-
+    except ClientError:
+        raise
     except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         response = {
@@ -710,11 +719,22 @@ def set_archive_code(code: Dict, lambda_name: str, zip_file_content: bytes = Non
 
 
 def set_function_code(lambda_function: LambdaFunction):
-    def _set_and_configure():
-        do_set_function_code(lambda_function)
-        # initialize function code via plugins
-        for plugin in lambda_executors.LambdaExecutorPlugin.get_plugins():
-            plugin.init_function_code(lambda_function)
+    def _set_and_configure(*args, **kwargs):
+        try:
+            before = time.perf_counter()
+            do_set_function_code(lambda_function)
+            # initialize function code via plugins
+            for plugin in lambda_executors.LambdaExecutorPlugin.get_plugins():
+                plugin.init_function_code(lambda_function)
+            lambda_function.state = "Active"
+            LOG.debug(
+                "Function code initialization for function '%s' complete. State => Active (in %.3fs)",
+                lambda_function.name(),
+                time.perf_counter() - before,
+            )
+        except Exception:
+            lambda_function.state = "Failed"
+            raise
 
     # unzipping can take some time - limit the execution time to avoid client/network timeout issues
     run_for_max_seconds(config.LAMBDA_CODE_EXTRACT_TIME, _set_and_configure)
@@ -751,6 +771,8 @@ def store_and_get_lambda_code_archive(
     else:
         # override lambda archive with fresh code if we got an update
         save_file(archive_file, zip_file_content)
+    # remove content from code attribute, if present
+    lambda_function.code.pop("ZipFile", None)
     return lambda_cwd, archive_file, zip_file_content
 
 
@@ -807,8 +829,12 @@ def generic_handler(*_):
         # Unzip the Lambda archive contents
 
         if get_unzipped_size(archive_file) >= FUNCTION_MAX_UNZIPPED_SIZE:
-            raise Exception(
-                f"An error occurred (InvalidParameterValueException) when calling the CreateFunction operation: Unzipped size must be smaller than {FUNCTION_MAX_UNZIPPED_SIZE} bytes"
+            raise ClientError(
+                error_response(
+                    f"Unzipped size must be smaller than {FUNCTION_MAX_UNZIPPED_SIZE} bytes",
+                    code=400,
+                    error_type="InvalidParameterValueException",
+                )
             )
 
         unzip(archive_file, lambda_cwd)
@@ -933,7 +959,7 @@ def format_func_details(
         "LastModified": format_timestamp(lambda_function.last_modified),
         "TracingConfig": lambda_function.tracing_config or {"Mode": "PassThrough"},
         "RevisionId": func_version.get("RevisionId"),
-        "State": "Active",
+        "State": lambda_function.state,
         "LastUpdateStatus": "Successful",
         "PackageType": lambda_function.package_type,
         "ImageConfig": getattr(lambda_function, "image_config", None),
@@ -1155,12 +1181,11 @@ def create_function():
         lambda_function.image_config = data.get("ImageConfig", {})
         lambda_function.tracing_config = data.get("TracingConfig", {})
         lambda_function.set_dead_letter_config(data)
+        lambda_function.state = "Pending"
         result = set_function_code(lambda_function)
         if isinstance(result, Response):
             del region.lambdas[arn]
             return result
-        # remove content from code attribute, if present
-        lambda_function.code.pop("ZipFile", None)
         # prepare result
         result.update(format_func_details(lambda_function))
         if data.get("Publish"):
@@ -1615,17 +1640,26 @@ def _create_response(invocation_result, status_code=200, headers=None):
         context = {"client_context": request.headers.get("X-Amz-Client-Context")}
 
         time_before = time.perf_counter()
-        result = run_lambda(
-            func_arn=arn,
-            event=data,
-            context=context,
-            asynchronous=False,
-            version=qualifier,
-        )
-        LOG.debug("Lambda invocation duration: %0.2fms", (time.perf_counter() - time_before) * 1000)
+        try:
+            result = run_lambda(
+                func_arn=arn,
+                event=data,
+                context=context,
+                asynchronous=False,
+                version=qualifier,
+            )
+        except ClientError as e:
+            return e.get_response()
+        finally:
+            LOG.debug(
+                "Lambda invocation duration: %0.2fms", (time.perf_counter() - time_before) * 1000
+            )
         return _create_response(result)
     elif invocation_type == "Event":
-        run_lambda(func_arn=arn, event=data, context={}, asynchronous=True, version=qualifier)
+        try:
+            run_lambda(func_arn=arn, event=data, context={}, asynchronous=True, version=qualifier)
+        except ClientError as e:
+            return e.get_response()
         return _create_response("", status_code=202)
     elif invocation_type == "DryRun":
         # Assume the dry run always passes.
diff --git a/localstack/utils/aws/aws_models.py b/localstack/utils/aws/aws_models.py
--- a/localstack/utils/aws/aws_models.py
+++ b/localstack/utils/aws/aws_models.py
@@ -205,6 +205,7 @@ def __init__(self, arn):
         self.architectures = ["x86_64"]
         self.image_config = {}
         self.tracing_config = {}
+        self.state = None
 
     def set_dead_letter_config(self, data):
         config = data.get("DeadLetterConfig")


Test patch:
diff --git a/tests/integration/awslambda/test_lambda.py b/tests/integration/awslambda/test_lambda.py
--- a/tests/integration/awslambda/test_lambda.py
+++ b/tests/integration/awslambda/test_lambda.py
@@ -72,6 +72,8 @@
 from .functions import lambda_integration
 from .lambda_test_util import concurrency_update_done, get_invoke_init_type, update_done
 
+LOG = logging.getLogger(__name__)
+
 THIS_FOLDER = os.path.dirname(os.path.realpath(__file__))
 TEST_LAMBDA_PYTHON = os.path.join(THIS_FOLDER, "functions/lambda_integration.py")
 TEST_LAMBDA_PYTHON_ECHO = os.path.join(THIS_FOLDER, "functions/lambda_echo.py")
@@ -798,6 +800,50 @@ def test_large_payloads(self, caplog, lambda_client, create_lambda_function, sna
         result_data = json.loads(to_str(result_data))
         assert payload == result_data
 
+    @pytest.mark.aws_validated
+    @pytest.mark.skip_snapshot_verify  # skipped - the diff is too big
+    @pytest.mark.skipif(
+        os.environ.get("TEST_TARGET") != "AWS_CLOUD",
+        reason="Lambda function state pending for small lambdas not supported on localstack",
+    )
+    def test_function_state(self, lambda_client, lambda_su_role, snapshot):
+        """Tests if a lambda starts in state "Pending" but moves to "Active" at some point"""
+        snapshot.add_transformer(snapshot.transform.lambda_api())
+        # necessary due to changing zip timestamps. We might need a reproducible archive method for this
+        snapshot.add_transformer(snapshot.transform.key_value("CodeSha256"))
+        function_name = f"test-function-{short_uid()}"
+        zip_file = create_lambda_archive(load_file(TEST_LAMBDA_PYTHON_ECHO), get_content=True)
+
+        try:
+
+            def create_function():
+                return lambda_client.create_function(
+                    FunctionName=function_name,
+                    Runtime="python3.9",
+                    Handler="handler.handler",
+                    Role=lambda_su_role,
+                    Code={"ZipFile": zip_file},
+                )
+
+            response = retry(create_function, sleep=3, retries=5)
+
+            snapshot.match("create-fn-response", response)
+            assert response["State"] == "Pending"
+
+            # lambda has to get active at some point
+            def _check_lambda_state():
+                response = lambda_client.get_function(FunctionName=function_name)
+                assert response["Configuration"]["State"] == "Active"
+                return response
+
+            response = retry(_check_lambda_state)
+            snapshot.match("get-fn-response", response)
+        finally:
+            try:
+                lambda_client.delete_function(FunctionName=function_name)
+            except Exception:
+                LOG.debug("Unable to delete function %s", function_name)
+
 
 parametrize_python_runtimes = pytest.mark.parametrize(
     "runtime",
diff --git a/tests/integration/awslambda/test_lambda.snapshot.json b/tests/integration/awslambda/test_lambda.snapshot.json
--- a/tests/integration/awslambda/test_lambda.snapshot.json
+++ b/tests/integration/awslambda/test_lambda.snapshot.json
@@ -3769,5 +3769,79 @@
         }
       }
     }
+  },
+  "tests/integration/awslambda/test_lambda.py::TestLambdaBaseFeatures::test_function_state": {
+    "recorded-date": "02-08-2022, 12:58:39",
+    "recorded-content": {
+      "create-fn-response": {
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 201
+        },
+        "FunctionName": "<function-name:1>",
+        "FunctionArn": "arn:aws:lambda:<region>:111111111111:function:<function-name:1>",
+        "Runtime": "python3.9",
+        "Role": "arn:aws:iam::111111111111:role/<resource:1>",
+        "Handler": "handler.handler",
+        "CodeSize": 264,
+        "Description": "",
+        "Timeout": 3,
+        "MemorySize": 128,
+        "LastModified": "date",
+        "CodeSha256": "<code-sha256:1>",
+        "Version": "$LATEST",
+        "TracingConfig": {
+          "Mode": "PassThrough"
+        },
+        "RevisionId": "<uuid:1>",
+        "State": "Pending",
+        "StateReason": "The function is being created.",
+        "StateReasonCode": "Creating",
+        "PackageType": "Zip",
+        "Architectures": [
+          "x86_64"
+        ],
+        "EphemeralStorage": {
+          "Size": 512
+        }
+      },
+      "get-fn-response": {
+        "ResponseMetadata": {
+          "HTTPHeaders": {},
+          "HTTPStatusCode": 200
+        },
+        "Configuration": {
+          "FunctionName": "<function-name:1>",
+          "FunctionArn": "arn:aws:lambda:<region>:111111111111:function:<function-name:1>",
+          "Runtime": "python3.9",
+          "Role": "arn:aws:iam::111111111111:role/<resource:1>",
+          "Handler": "handler.handler",
+          "CodeSize": 264,
+          "Description": "",
+          "Timeout": 3,
+          "MemorySize": 128,
+          "LastModified": "date",
+          "CodeSha256": "<code-sha256:1>",
+          "Version": "$LATEST",
+          "TracingConfig": {
+            "Mode": "PassThrough"
+          },
+          "RevisionId": "<uuid:2>",
+          "State": "Active",
+          "LastUpdateStatus": "Successful",
+          "PackageType": "Zip",
+          "Architectures": [
+            "x86_64"
+          ],
+          "EphemeralStorage": {
+            "Size": 512
+          }
+        },
+        "Code": {
+          "RepositoryType": "S3",
+          "Location": "<location>"
+        }
+      }
+    }
   }
 }
diff --git a/tests/integration/awslambda/test_lambda_size_lims.py b/tests/integration/awslambda/test_lambda_size_lims.py
--- a/tests/integration/awslambda/test_lambda_size_lims.py
+++ b/tests/integration/awslambda/test_lambda_size_lims.py
@@ -2,6 +2,7 @@
 from io import BytesIO
 
 import pytest
+from botocore.exceptions import ClientError
 
 from localstack.services.awslambda.lambda_api import LAMBDA_DEFAULT_HANDLER
 from localstack.services.awslambda.lambda_utils import LAMBDA_RUNTIME_PYTHON37
@@ -38,7 +39,7 @@ def test_oversized_lambda(self, lambda_client, s3_client, s3_bucket, lambda_su_r
         s3_client.upload_fileobj(BytesIO(zip_file), s3_bucket, bucket_key)
 
         # create lambda function
-        with pytest.raises(Exception) as e:
+        with pytest.raises(ClientError) as e:
             lambda_client.create_function(
                 FunctionName=function_name,
                 Runtime=LAMBDA_RUNTIME_PYTHON37,
@@ -47,6 +48,8 @@ def test_oversized_lambda(self, lambda_client, s3_client, s3_bucket, lambda_su_r
                 Code={"S3Bucket": s3_bucket, "S3Key": bucket_key},
                 Timeout=10,
             )
+        assert e
+        assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 400
         e.match(
             r"An error occurred \(InvalidParameterValueException\) when calling the CreateFunction operation\: Unzipped size must be smaller than [0-9]* bytes"
         )
diff --git a/tests/integration/awslambda/test_lambda_whitebox.py b/tests/integration/awslambda/test_lambda_whitebox.py
--- a/tests/integration/awslambda/test_lambda_whitebox.py
+++ b/tests/integration/awslambda/test_lambda_whitebox.py
@@ -1,14 +1,18 @@
 import base64
 import json
+import logging
 import os
+import threading
 import time
 import unittest
 
 import pytest
+from botocore.exceptions import ClientError
 
+import localstack.services.awslambda.lambda_api
 from localstack import config
 from localstack.services.awslambda import lambda_api, lambda_executors
-from localstack.services.awslambda.lambda_api import use_docker
+from localstack.services.awslambda.lambda_api import do_set_function_code, use_docker
 from localstack.services.awslambda.lambda_utils import LAMBDA_RUNTIME_PYTHON36
 from localstack.services.generic_proxy import ProxyListener
 from localstack.services.infra import start_proxy
@@ -24,8 +28,14 @@
     to_bytes,
     to_str,
 )
+from localstack.utils.testutil import create_lambda_archive
 
-from .test_lambda import TEST_LAMBDA_ENV, TEST_LAMBDA_LIBS, TEST_LAMBDA_PYTHON
+from .test_lambda import (
+    TEST_LAMBDA_ENV,
+    TEST_LAMBDA_LIBS,
+    TEST_LAMBDA_PYTHON,
+    TEST_LAMBDA_PYTHON_ECHO,
+)
 
 # TestLocalLambda variables
 THIS_FOLDER = os.path.dirname(os.path.realpath(__file__))
@@ -36,6 +46,8 @@
     THIS_FOLDER, "functions", "python3", "lambda2", "lambda2.zip"
 )
 
+LOG = logging.getLogger(__name__)
+
 
 class TestLambdaFallbackUrl(unittest.TestCase):
     @staticmethod
@@ -370,3 +382,57 @@ def test_python3_runtime_multiple_create_with_conflicting_module(self):
             testutil.delete_lambda_function(lambda_name2)
         finally:
             lambda_api.DO_USE_DOCKER = original_do_use_docker
+
+
+class TestFunctionStates:
+    def test_invoke_failure_when_state_pending(self, lambda_client, lambda_su_role, monkeypatch):
+        """Tests if a lambda invocation fails if state is pending"""
+        function_name = f"test-function-{short_uid()}"
+        zip_file = create_lambda_archive(load_file(TEST_LAMBDA_PYTHON_ECHO), get_content=True)
+
+        function_code_set = threading.Event()
+
+        def _do_set_function_code(*args, **kwargs):
+            result = do_set_function_code(*args, **kwargs)
+            function_code_set.wait()
+            return result
+
+        monkeypatch.setattr(
+            localstack.services.awslambda.lambda_api, "do_set_function_code", _do_set_function_code
+        )
+        try:
+            response = lambda_client.create_function(
+                FunctionName=function_name,
+                Runtime="python3.9",
+                Handler="handler.handler",
+                Role=lambda_su_role,
+                Code={"ZipFile": zip_file},
+            )
+
+            assert response["State"] == "Pending"
+
+            with pytest.raises(ClientError) as e:
+                lambda_client.invoke(FunctionName=function_name, Payload=b"{}")
+
+            assert e.value.response["ResponseMetadata"]["HTTPStatusCode"] == 409
+            assert e.match("ResourceConflictException")
+            assert e.match(
+                "The operation cannot be performed at this time. The function is currently in the following state: Pending"
+            )
+
+            # let function move to active
+            function_code_set.set()
+
+            # lambda has to get active at some point
+            def _check_lambda_state():
+                response = lambda_client.get_function(FunctionName=function_name)
+                assert response["Configuration"]["State"] == "Active"
+                return response
+
+            retry(_check_lambda_state)
+            lambda_client.invoke(FunctionName=function_name, Payload=b"{}")
+        finally:
+            try:
+                lambda_client.delete_function(FunctionName=function_name)
+            except Exception:
+                LOG.debug("Unable to delete function %s", function_name)
diff --git a/tests/unit/test_lambda.py b/tests/unit/test_lambda.py
--- a/tests/unit/test_lambda.py
+++ b/tests/unit/test_lambda.py
@@ -1027,6 +1027,7 @@ def _create_function(self, function_name, tags=None):
         region.lambdas[arn].last_modified = self.LAST_MODIFIED
         region.lambdas[arn].role = self.ROLE
         region.lambdas[arn].memory_size = self.MEMORY_SIZE
+        region.lambdas[arn].state = "Active"
 
     def _update_function_code(self, function_name, tags=None):
         if tags is None: