Repository: getmoto/moto
Pull request: Speed up exception handling by re-using Environments
Address https://github.com/getmoto/moto/issues/7142

By taking advantage of the caching logic in [Environment](https://jinja.palletsprojects.com/en/3.1.x/api/#jinja2.Environment)

Problem statement:
Slow s3 exception raising due to usage pattern of jinja2
```python
import time

from moto.s3.exceptions import MissingBucket

start = time.time()
for _ in range(1000):
    try:
        raise MissingBucket(bucket="foo")
    except MissingBucket:
        pass

print(round(time.time() - start, 2))
```

with  moto 4.2.12: `1.58`

cProfile data shows it's related to jinja2 usage. Some refactoring should be able to significantly speed this up.
```pstats% stats 30
Tue Dec 19 00:09:04 2023    pstats

         4532388 function calls (4276064 primitive calls) in 2.932 seconds

   Ordered by: cumulative time
   List reduced from 4319 to 30 due to restriction <30>

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
   2658/1    0.010    0.000    2.975    2.975 {built-in method builtins.exec}
        1    0.003    0.003    2.975    2.975 foo.py:1(<module>)
     1000    0.002    0.000    2.501    0.003 /usr/local/lib/python3.8/dist-packages/moto/s3/exceptions.py:103(__init__)
     1000    0.002    0.000    2.499    0.002 /usr/local/lib/python3.8/dist-packages/moto/s3/exceptions.py:76(__init__)
     1000    0.002    0.000    2.496    0.002 /usr/local/lib/python3.8/dist-packages/moto/s3/exceptions.py:44(__init__)
     1000    0.009    0.000    2.494    0.002 /usr/local/lib/python3.8/dist-packages/moto/core/exceptions.py:58(__init__)
     2000    0.002    0.000    2.418    0.001 /usr/local/lib/python3.8/dist-packages/jinja2/environment.py:964(get_template)
     2000    0.007    0.000    2.415    0.001 /usr/local/lib/python3.8/dist-packages/jinja2/environment.py:939(_load_template)
     2000    0.007    0.000    2.393    0.001 /usr/local/lib/python3.8/dist-packages/jinja2/loaders.py:106(load)
     2000    0.008    0.000    2.359    0.001 /usr/local/lib/python3.8/dist-packages/jinja2/environment.py:717(compile)
     1000    0.003    0.000    1.433    0.001 /usr/local/lib/python3.8/dist-packages/jinja2/environment.py:1256(render)
26226/16158    0.014    0.000    1.411    0.000 {method 'join' of 'str' objects}
```

This means in code that has lots of calls to Moto for buckets that aren't set, there is a noticeable performance impact.

Hints from issue discussion:

Commit message: Speed up exception handling by re-using Environments

Address https://github.com/getmoto/moto/issues/7142

Patch to review:
diff --git a/moto/cloudfront/exceptions.py b/moto/cloudfront/exceptions.py
index 584d999e4d7a..65cd219d534c 100644
--- a/moto/cloudfront/exceptions.py
+++ b/moto/cloudfront/exceptions.py
@@ -1,5 +1,6 @@
 from typing import Any
 
+from jinja2 import DictLoader, Environment
 from moto.core.exceptions import RESTError
 
 EXCEPTION_RESPONSE = """<?xml version="1.0"?>
@@ -15,10 +16,11 @@
 
 class CloudFrontException(RESTError):
     code = 400
+    extended_templates = {"cferror": EXCEPTION_RESPONSE}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
 
     def __init__(self, error_type: str, message: str, **kwargs: Any):
         kwargs.setdefault("template", "cferror")
-        self.templates["cferror"] = EXCEPTION_RESPONSE
         super().__init__(error_type, message, **kwargs)
 
 
diff --git a/moto/core/exceptions.py b/moto/core/exceptions.py
index ee9a07820a0b..30e77084a34b 100644
--- a/moto/core/exceptions.py
+++ b/moto/core/exceptions.py
@@ -54,6 +54,7 @@ class RESTError(HTTPException):
         "wrapped_single_error": WRAPPED_SINGLE_ERROR_RESPONSE,
         "error": ERROR_RESPONSE,
     }
+    env = Environment(loader=DictLoader(templates))
 
     def __init__(
         self, error_type: str, message: str, template: str = "error", **kwargs: Any
@@ -63,8 +64,7 @@ def __init__(
         self.message = message
 
         if template in self.templates.keys():
-            env = Environment(loader=DictLoader(self.templates))
-            self.description: str = env.get_template(template).render(
+            self.description: str = self.env.get_template(template).render(
                 error_type=error_type,
                 message=message,
                 request_id_tag=self.request_id_tag_name,
diff --git a/moto/ec2/exceptions.py b/moto/ec2/exceptions.py
index c9354d3fb1ea..e103dcb6e4c4 100644
--- a/moto/ec2/exceptions.py
+++ b/moto/ec2/exceptions.py
@@ -1,5 +1,6 @@
 from typing import Any, Iterable, List, Optional, Union
 
+from jinja2 import DictLoader, Environment
 from moto.core.exceptions import RESTError
 
 # EC2 has a custom root-tag - <Response> vs <ErrorResponse>
@@ -22,10 +23,11 @@ class EC2ClientError(RESTError):
     code = 400
     # EC2 uses <RequestID> as tag name in the XML response
     request_id_tag_name = "RequestID"
+    extended_templates = {"custom_response": EC2_ERROR_RESPONSE}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
 
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "custom_response")
-        self.templates["custom_response"] = EC2_ERROR_RESPONSE
         super().__init__(*args, **kwargs)
 
 
diff --git a/moto/elasticache/exceptions.py b/moto/elasticache/exceptions.py
index f5e7f5b65413..ab5e2cd4153d 100644
--- a/moto/elasticache/exceptions.py
+++ b/moto/elasticache/exceptions.py
@@ -1,5 +1,6 @@
 from typing import Any
 
+from jinja2 import DictLoader, Environment
 from moto.core.exceptions import RESTError
 
 EXCEPTION_RESPONSE = """<?xml version="1.0"?>
@@ -16,6 +17,8 @@
 class ElastiCacheException(RESTError):
 
     code = 400
+    extended_templates = {"ecerror": EXCEPTION_RESPONSE}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
 
     def __init__(self, code: str, message: str, **kwargs: Any):
         kwargs.setdefault("template", "ecerror")
diff --git a/moto/elasticbeanstalk/exceptions.py b/moto/elasticbeanstalk/exceptions.py
index 632080fb5f7b..c210b7aaea9b 100644
--- a/moto/elasticbeanstalk/exceptions.py
+++ b/moto/elasticbeanstalk/exceptions.py
@@ -1,5 +1,6 @@
 from typing import Any
 
+from jinja2 import DictLoader, Environment
 from moto.core.exceptions import RESTError
 
 EXCEPTION_RESPONSE = """<?xml version="1.0"?>
@@ -16,6 +17,8 @@
 class ElasticBeanstalkException(RESTError):
 
     code = 400
+    extended_templates = {"ecerror": EXCEPTION_RESPONSE}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
 
     def __init__(self, code: str, message: str, **kwargs: Any):
         kwargs.setdefault("template", "ecerror")
diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py
index b86d56995c50..fbbce0f02b1f 100644
--- a/moto/s3/exceptions.py
+++ b/moto/s3/exceptions.py
@@ -1,5 +1,6 @@
 from typing import TYPE_CHECKING, Any, Optional, Union
 
+from jinja2 import DictLoader, Environment
 from moto.core.exceptions import RESTError
 
 if TYPE_CHECKING:
@@ -41,6 +42,17 @@ class S3ClientError(RESTError):
     # S3 API uses <RequestID> as the XML tag in response messages
     request_id_tag_name = "RequestID"
 
+    extended_templates = {
+        "bucket_error": ERROR_WITH_BUCKET_NAME,
+        "key_error": ERROR_WITH_KEY_NAME,
+        "argument_error": ERROR_WITH_ARGUMENT,
+        "error_uploadid": ERROR_WITH_UPLOADID,
+        "condition_error": ERROR_WITH_CONDITION_NAME,
+        "range_error": ERROR_WITH_RANGE,
+        "storage_error": ERROR_WITH_STORAGE_CLASS,
+    }
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
+
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "single_error")
         self.templates["bucket_error"] = ERROR_WITH_BUCKET_NAME
diff --git a/moto/s3control/exceptions.py b/moto/s3control/exceptions.py
index 05d962f50d60..fa2b1389b7ef 100644
--- a/moto/s3control/exceptions.py
+++ b/moto/s3control/exceptions.py
@@ -1,5 +1,6 @@
 from typing import Any
 
+from jinja2 import DictLoader, Environment
 from moto.core.exceptions import RESTError
 
 ERROR_WITH_ACCESS_POINT_NAME = """{% extends 'wrapped_single_error' %}
@@ -13,6 +14,12 @@
 
 
 class S3ControlError(RESTError):
+    extended_templates = {
+        "ap_not_found": ERROR_WITH_ACCESS_POINT_NAME,
+        "apf_not_found": ERROR_WITH_ACCESS_POINT_POLICY,
+    }
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
+
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "single_error")
         super().__init__(*args, **kwargs)
@@ -24,7 +31,6 @@ class AccessPointNotFound(S3ControlError):
     def __init__(self, name: str, **kwargs: Any):
         kwargs.setdefault("template", "ap_not_found")
         kwargs["name"] = name
-        self.templates["ap_not_found"] = ERROR_WITH_ACCESS_POINT_NAME
         super().__init__(
             "NoSuchAccessPoint", "The specified accesspoint does not exist", **kwargs
         )
@@ -36,7 +42,6 @@ class AccessPointPolicyNotFound(S3ControlError):
     def __init__(self, name: str, **kwargs: Any):
         kwargs.setdefault("template", "apf_not_found")
         kwargs["name"] = name
-        self.templates["apf_not_found"] = ERROR_WITH_ACCESS_POINT_POLICY
         super().__init__(
             "NoSuchAccessPointPolicy",
             "The specified accesspoint policy does not exist",
diff --git a/moto/sagemaker/exceptions.py b/moto/sagemaker/exceptions.py
index 77e9308627d6..febb7f712f7c 100644
--- a/moto/sagemaker/exceptions.py
+++ b/moto/sagemaker/exceptions.py
@@ -1,5 +1,6 @@
 from typing import Any
 
+from jinja2 import DictLoader, Environment
 from moto.core.exceptions import AWSError, JsonRESTError, RESTError
 
 ERROR_WITH_MODEL_NAME = """{% extends 'single_error' %}
@@ -8,9 +9,11 @@
 
 
 class SagemakerClientError(RESTError):
+    extended_templates = {"model_error": ERROR_WITH_MODEL_NAME}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
+
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "single_error")
-        self.templates["model_error"] = ERROR_WITH_MODEL_NAME
         super().__init__(*args, **kwargs)
 
 
diff --git a/moto/sdb/exceptions.py b/moto/sdb/exceptions.py
index 9d90ecddc899..2315c9914574 100644
--- a/moto/sdb/exceptions.py
+++ b/moto/sdb/exceptions.py
@@ -2,6 +2,7 @@
 from typing import Any
 
 from moto.core.exceptions import RESTError
+from jinja2 import DictLoader, Environment
 
 SDB_ERROR = """<?xml version="1.0"?>
 <Response>
@@ -18,6 +19,8 @@
 
 class InvalidParameterError(RESTError):
     code = 400
+    extended_templates = {"sdb_error": SDB_ERROR}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
 
     def __init__(self, **kwargs: Any):
         kwargs.setdefault("template", "sdb_error")
@@ -37,10 +40,11 @@ def __init__(self, domain_name: str):
 
 class UnknownDomainName(RESTError):
     code = 400
+    extended_templates = {"sdb_error": SDB_ERROR}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
 
     def __init__(self, **kwargs: Any):
         kwargs.setdefault("template", "sdb_error")
-        self.templates["sdb_error"] = SDB_ERROR
         kwargs["error_type"] = "NoSuchDomain"
         kwargs["message"] = "The specified domain does not exist."
         super().__init__(**kwargs)


Code review comments:
diff_hunk: @@ -63,8 +64,7 @@ def __init__(
         self.message = message
 
         if template in self.templates.keys():
-            env = Environment(loader=DictLoader(self.templates))
-            self.description: str = env.get_template(template).render(
+            self.description: str = self.env.get_template(template).render(
path: moto/core/exceptions.py
text: @user1:
```suggestion
            self.description: str = self.__class__.env.get_template(template).render(
```
Functionally equivalent (in this case), but this would make it more obvious to a developer that we're calling a class-attribute.

diff_hunk: @@ -16,6 +17,8 @@
 class ElastiCacheException(RESTError):
 
     code = 400
+    extended_templates = {"ecerror": EXCEPTION_RESPONSE}
path: moto/elasticache/exceptions.py
text: @user1:
The `self.templates["ecerror"] = ERROR...` can now be removed in the `__init__`-method, right?

I saw the same in the `sdb`, `elasticbeanstalk` and `s3` modules.

@author:
done

diff_hunk: @@ -15,10 +16,11 @@
 
 class CloudFrontException(RESTError):
     code = 400
+    extended_templates = {"cferror": EXCEPTION_RESPONSE}
+    env = Environment(loader=DictLoader(RESTError.templates | extended_templates))
path: moto/cloudfront/exceptions.py
text: @user1:
The `|`-operator is not supported in Python 3.7, as far as I know. I'll kick off the linter anyway, but I expect that to fail.

@author:
good catch,  this is a 3.9 feature it turns out