Repository: getmoto/moto
Pull request #7143 (2023-12-19 00:56:36)

Speed up exception handling by re-using Environments
Address https://github.com/getmoto/moto/issues/7142

By taking advantage of the caching logic in [Environment](https://jinja.palletsprojects.com/en/3.1.x/api/#jinja2.Environment)

jogo: test code from #7142 before:

Before: `1.58`
After: `0.02`


cProfile data confirms jinja2 call times drop significantly

Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5a83285ab..eefb42076 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -82,6 +82,10 @@ jobs:
     needs: lint
     uses: ./.github/workflows/tests_sdk_ruby.yml
 
+  terraformexamplestest:
+    needs: lint
+    uses: ./.github/workflows/tests_terraform_examples.yml
+
   test:
     needs: [lint]
     if: "!contains(github.event.pull_request.labels.*.name, 'java')"
diff --git a/.gitignore b/.gitignore
index 690cc0cff..e34bcfd32 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,5 @@ other_langs/tests_java/target
 other_langs/tests_dotnet/ExampleTestProject/bin
 other_langs/tests_dotnet/ExampleTestProject/obj
 other_langs/tests_ruby/Gemfile.lock
+other_langs/terraform/*/.terraform*
+other_langs/terraform/*/terraform*
diff --git a/moto/acm/models.py b/moto/acm/models.py
index 522061436..8d7bcf4c8 100644
--- a/moto/acm/models.py
+++ b/moto/acm/models.py
@@ -337,25 +337,32 @@ class CertBundle(BaseModel):
                 "ExtendedKeyUsages": [],
                 "RenewalEligibility": "INELIGIBLE",
                 "Options": {"CertificateTransparencyLoggingPreference": "ENABLED"},
-                "DomainValidationOptions": [{"DomainName": self.common_name}],
             }
         }
 
+        domain_names = set(sans + [self.common_name])
+        validation_options = []
+
         if self.status == "PENDING_VALIDATION":
-            result["Certificate"]["DomainValidationOptions"][0][
-                "ValidationDomain"
-            ] = self.common_name
-            result["Certificate"]["DomainValidationOptions"][0][
-                "ValidationStatus"
-            ] = self.status
-            result["Certificate"]["DomainValidationOptions"][0]["ResourceRecord"] = {
-                "Name": f"_d930b28be6c5927595552b219965053e.{self.common_name}.",
-                "Type": "CNAME",
-                "Value": "_c9edd76ee4a0e2a74388032f3861cc50.ykybfrwcxw.acm-validations.aws.",
-            }
-            result["Certificate"]["DomainValidationOptions"][0][
-                "ValidationMethod"
-            ] = "DNS"
+            for san in domain_names:
+                resource_record = {
+                    "Name": f"_d930b28be6c5927595552b219965053e.{san}.",
+                    "Type": "CNAME",
+                    "Value": "_c9edd76ee4a0e2a74388032f3861cc50.ykybfrwcxw.acm-validations.aws.",
+                }
+                validation_options.append(
+                    {
+                        "DomainName": san,
+                        "ValidationDomain": san,
+                        "ValidationStatus": self.status,
+                        "ValidationMethod": "DNS",
+                        "ResourceRecord": resource_record,
+                    }
+                )
+        else:
+            validation_options = [{"DomainName": name} for name in domain_names]
+        result["Certificate"]["DomainValidationOptions"] = validation_options
+
         if self.type == "IMPORTED":
             result["Certificate"]["ImportedAt"] = datetime_to_epoch(self.created_at)
         else:
diff --git a/moto/cloudfront/exceptions.py b/moto/cloudfront/exceptions.py
index 584d999e4..b8e7f634d 100644
--- a/moto/cloudfront/exceptions.py
+++ b/moto/cloudfront/exceptions.py
@@ -15,10 +15,11 @@ EXCEPTION_RESPONSE = """<?xml version="1.0"?>
 
 class CloudFrontException(RESTError):
     code = 400
+    extended_templates = {"cferror": EXCEPTION_RESPONSE}
+    env = RESTError.extended_environment(extended_templates)
 
     def __init__(self, error_type: str, message: str, **kwargs: Any):
         kwargs.setdefault("template", "cferror")
-        self.templates["cferror"] = EXCEPTION_RESPONSE
         super().__init__(error_type, message, **kwargs)
 
 
diff --git a/moto/core/exceptions.py b/moto/core/exceptions.py
index ee9a07820..7be3b5979 100644
--- a/moto/core/exceptions.py
+++ b/moto/core/exceptions.py
@@ -1,5 +1,5 @@
 import json
-from typing import Any, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 from jinja2 import DictLoader, Environment
 from werkzeug.exceptions import HTTPException
@@ -54,6 +54,7 @@ class RESTError(HTTPException):
         "wrapped_single_error": WRAPPED_SINGLE_ERROR_RESPONSE,
         "error": ERROR_RESPONSE,
     }
+    env = Environment(loader=DictLoader(templates))
 
     def __init__(
         self, error_type: str, message: str, template: str = "error", **kwargs: Any
@@ -62,9 +63,8 @@ class RESTError(HTTPException):
         self.error_type = error_type
         self.message = message
 
-        if template in self.templates.keys():
-            env = Environment(loader=DictLoader(self.templates))
-            self.description: str = env.get_template(template).render(
+        if template in self.env.list_templates():
+            self.description: str = self.__class__.env.get_template(template).render(
                 error_type=error_type,
                 message=message,
                 request_id_tag=self.request_id_tag_name,
@@ -95,6 +95,13 @@ class RESTError(HTTPException):
         err.code = self.code
         return err
 
+    @classmethod
+    def extended_environment(cls, extended_templates: Dict[str, str]) -> Environment:
+        # Can be simplified to cls.templates | extended_templates when we drop Python 3.8 support
+        # https://docs.python.org/3/library/stdtypes.html#mapping-types-dict
+        templates = dict(cls.templates.items() | extended_templates.items())
+        return Environment(loader=DictLoader(templates))
+
 
 class DryRunClientError(RESTError):
     code = 412
diff --git a/moto/ec2/exceptions.py b/moto/ec2/exceptions.py
index c9354d3fb..8b8fc7f33 100644
--- a/moto/ec2/exceptions.py
+++ b/moto/ec2/exceptions.py
@@ -22,10 +22,11 @@ class EC2ClientError(RESTError):
     code = 400
     # EC2 uses <RequestID> as tag name in the XML response
     request_id_tag_name = "RequestID"
+    extended_templates = {"custom_response": EC2_ERROR_RESPONSE}
+    env = RESTError.extended_environment(extended_templates)
 
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "custom_response")
-        self.templates["custom_response"] = EC2_ERROR_RESPONSE
         super().__init__(*args, **kwargs)
 
 
diff --git a/moto/elasticache/exceptions.py b/moto/elasticache/exceptions.py
index f5e7f5b65..0d63efb88 100644
--- a/moto/elasticache/exceptions.py
+++ b/moto/elasticache/exceptions.py
@@ -16,10 +16,11 @@ EXCEPTION_RESPONSE = """<?xml version="1.0"?>
 class ElastiCacheException(RESTError):
 
     code = 400
+    extended_templates = {"ecerror": EXCEPTION_RESPONSE}
+    env = RESTError.extended_environment(extended_templates)
 
     def __init__(self, code: str, message: str, **kwargs: Any):
         kwargs.setdefault("template", "ecerror")
-        self.templates["ecerror"] = EXCEPTION_RESPONSE
         super().__init__(code, message)
 
 
diff --git a/moto/elasticbeanstalk/exceptions.py b/moto/elasticbeanstalk/exceptions.py
index 632080fb5..619adbaa1 100644
--- a/moto/elasticbeanstalk/exceptions.py
+++ b/moto/elasticbeanstalk/exceptions.py
@@ -16,10 +16,11 @@ EXCEPTION_RESPONSE = """<?xml version="1.0"?>
 class ElasticBeanstalkException(RESTError):
 
     code = 400
+    extended_templates = {"ecerror": EXCEPTION_RESPONSE}
+    env = RESTError.extended_environment(extended_templates)
 
     def __init__(self, code: str, message: str, **kwargs: Any):
         kwargs.setdefault("template", "ecerror")
-        self.templates["ecerror"] = EXCEPTION_RESPONSE
         super().__init__(code, message)
 
 
diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py
index b86d56995..337a9492e 100644
--- a/moto/s3/exceptions.py
+++ b/moto/s3/exceptions.py
@@ -41,9 +41,19 @@ class S3ClientError(RESTError):
     # S3 API uses <RequestID> as the XML tag in response messages
     request_id_tag_name = "RequestID"
 
+    extended_templates = {
+        "bucket_error": ERROR_WITH_BUCKET_NAME,
+        "key_error": ERROR_WITH_KEY_NAME,
+        "argument_error": ERROR_WITH_ARGUMENT,
+        "error_uploadid": ERROR_WITH_UPLOADID,
+        "condition_error": ERROR_WITH_CONDITION_NAME,
+        "range_error": ERROR_WITH_RANGE,
+        "storage_error": ERROR_WITH_STORAGE_CLASS,
+    }
+    env = RESTError.extended_environment(extended_templates)
+
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "single_error")
-        self.templates["bucket_error"] = ERROR_WITH_BUCKET_NAME
         super().__init__(*args, **kwargs)
 
 
@@ -54,7 +64,6 @@ class InvalidArgumentError(S3ClientError):
         kwargs.setdefault("template", "argument_error")
         kwargs["name"] = name
         kwargs["value"] = value
-        self.templates["argument_error"] = ERROR_WITH_ARGUMENT
         super().__init__("InvalidArgument", message, *args, **kwargs)
 
 
@@ -75,7 +84,6 @@ class BadRequest(S3ClientError):
 class BucketError(S3ClientError):
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "bucket_error")
-        self.templates["bucket_error"] = ERROR_WITH_BUCKET_NAME
         super().__init__(*args, **kwargs)
 
 
@@ -84,7 +92,6 @@ class BucketAlreadyExists(BucketError):
 
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "bucket_error")
-        self.templates["bucket_error"] = ERROR_WITH_BUCKET_NAME
         super().__init__(
             "BucketAlreadyExists",
             (
@@ -111,7 +118,6 @@ class MissingKey(S3ClientError):
 
     def __init__(self, **kwargs: Any):
         kwargs.setdefault("template", "key_error")
-        self.templates["key_error"] = ERROR_WITH_KEY_NAME
         super().__init__("NoSuchKey", "The specified key does not exist.", **kwargs)
 
 
@@ -129,7 +135,6 @@ class InvalidVersion(S3ClientError):
         kwargs.setdefault("template", "argument_error")
         kwargs["name"] = "versionId"
         kwargs["value"] = version_id
-        self.templates["argument_error"] = ERROR_WITH_ARGUMENT
         super().__init__(
             "InvalidArgument", "Invalid version id specified", *args, **kwargs
         )
@@ -436,7 +441,6 @@ class NoSuchUpload(S3ClientError):
     def __init__(self, upload_id: Union[int, str], *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "error_uploadid")
         kwargs["upload_id"] = upload_id
-        self.templates["error_uploadid"] = ERROR_WITH_UPLOADID
         super().__init__(
             "NoSuchUpload",
             "The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
@@ -450,7 +454,6 @@ class PreconditionFailed(S3ClientError):
 
     def __init__(self, failed_condition: str, **kwargs: Any):
         kwargs.setdefault("template", "condition_error")
-        self.templates["condition_error"] = ERROR_WITH_CONDITION_NAME
         super().__init__(
             "PreconditionFailed",
             "At least one of the pre-conditions you specified did not hold",
@@ -464,7 +467,6 @@ class InvalidRange(S3ClientError):
 
     def __init__(self, range_requested: str, actual_size: str, **kwargs: Any):
         kwargs.setdefault("template", "range_error")
-        self.templates["range_error"] = ERROR_WITH_RANGE
         super().__init__(
             "InvalidRange",
             "The requested range is not satisfiable",
@@ -488,7 +490,6 @@ class InvalidObjectState(BucketError):
 
     def __init__(self, storage_class: Optional[str], **kwargs: Any):
         kwargs.setdefault("template", "storage_error")
-        self.templates["storage_error"] = ERROR_WITH_STORAGE_CLASS
         super().__init__(
             error_type="InvalidObjectState",
             message="The operation is not valid for the object's storage class",
diff --git a/moto/s3control/exceptions.py b/moto/s3control/exceptions.py
index 05d962f50..cd69cdcbd 100644
--- a/moto/s3control/exceptions.py
+++ b/moto/s3control/exceptions.py
@@ -13,6 +13,12 @@ ERROR_WITH_ACCESS_POINT_POLICY = """{% extends 'wrapped_single_error' %}
 
 
 class S3ControlError(RESTError):
+    extended_templates = {
+        "ap_not_found": ERROR_WITH_ACCESS_POINT_NAME,
+        "apf_not_found": ERROR_WITH_ACCESS_POINT_POLICY,
+    }
+    env = RESTError.extended_environment(extended_templates)
+
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "single_error")
         super().__init__(*args, **kwargs)
@@ -24,7 +30,6 @@ class AccessPointNotFound(S3ControlError):
     def __init__(self, name: str, **kwargs: Any):
         kwargs.setdefault("template", "ap_not_found")
         kwargs["name"] = name
-        self.templates["ap_not_found"] = ERROR_WITH_ACCESS_POINT_NAME
         super().__init__(
             "NoSuchAccessPoint", "The specified accesspoint does not exist", **kwargs
         )
@@ -36,7 +41,6 @@ class AccessPointPolicyNotFound(S3ControlError):
     def __init__(self, name: str, **kwargs: Any):
         kwargs.setdefault("template", "apf_not_found")
         kwargs["name"] = name
-        self.templates["apf_not_found"] = ERROR_WITH_ACCESS_POINT_POLICY
         super().__init__(
             "NoSuchAccessPointPolicy",
             "The specified accesspoint policy does not exist",
diff --git a/moto/sagemaker/exceptions.py b/moto/sagemaker/exceptions.py
index 77e930862..8f5750573 100644
--- a/moto/sagemaker/exceptions.py
+++ b/moto/sagemaker/exceptions.py
@@ -8,16 +8,20 @@ ERROR_WITH_MODEL_NAME = """{% extends 'single_error' %}
 
 
 class SagemakerClientError(RESTError):
+    extended_templates = {"model_error": ERROR_WITH_MODEL_NAME}
+    env = RESTError.extended_environment(extended_templates)
+
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "single_error")
-        self.templates["model_error"] = ERROR_WITH_MODEL_NAME
         super().__init__(*args, **kwargs)
 
 
 class ModelError(RESTError):
+    extended_templates = {"model_error": ERROR_WITH_MODEL_NAME}
+    env = RESTError.extended_environment(extended_templates)
+
     def __init__(self, *args: Any, **kwargs: Any):
         kwargs.setdefault("template", "model_error")
-        self.templates["model_error"] = ERROR_WITH_MODEL_NAME
         super().__init__(*args, **kwargs)
 
 
diff --git a/moto/sdb/exceptions.py b/moto/sdb/exceptions.py
index 9d90ecddc..07eb4e099 100644
--- a/moto/sdb/exceptions.py
+++ b/moto/sdb/exceptions.py
@@ -18,10 +18,11 @@ SDB_ERROR = """<?xml version="1.0"?>
 
 class InvalidParameterError(RESTError):
     code = 400
+    extended_templates = {"sdb_error": SDB_ERROR}
+    env = RESTError.extended_environment(extended_templates)
 
     def __init__(self, **kwargs: Any):
         kwargs.setdefault("template", "sdb_error")
-        self.templates["sdb_error"] = SDB_ERROR
         kwargs["error_type"] = "InvalidParameterValue"
         super().__init__(**kwargs)
 
@@ -37,10 +38,11 @@ class InvalidDomainName(InvalidParameterError):
 
 class UnknownDomainName(RESTError):
     code = 400
+    extended_templates = {"sdb_error": SDB_ERROR}
+    env = RESTError.extended_environment(extended_templates)
 
     def __init__(self, **kwargs: Any):
         kwargs.setdefault("template", "sdb_error")
-        self.templates["sdb_error"] = SDB_ERROR
         kwargs["error_type"] = "NoSuchDomain"
         kwargs["message"] = "The specified domain does not exist."
         super().__init__(**kwargs)
diff --git a/other_langs/terraform/acm/acm.tf b/other_langs/terraform/acm/acm.tf
new file mode 100644
index 000000000..1e43932f6
--- /dev/null
+++ b/other_langs/terraform/acm/acm.tf
@@ -0,0 +1,42 @@
+locals {
+  domain_name = "test.co"
+}
+
+resource "aws_route53_zone" "test" {
+  name = local.domain_name
+}
+
+resource "aws_acm_certificate" "certificate" {
+  domain_name       = local.domain_name
+  validation_method = "DNS"
+
+  subject_alternative_names = ["*.${local.domain_name}"]
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_route53_record" "certificate_validation" {
+  for_each = {
+    for dvo in aws_acm_certificate.certificate.domain_validation_options : dvo.domain_name => {
+      name   = dvo.resource_record_name
+      record = dvo.resource_record_value
+      type   = dvo.resource_record_type
+    }
+  }
+
+  allow_overwrite = true
+  name            = each.value.name
+  records         = [each.value.record]
+  ttl             = 60
+  type            = each.value.type
+  zone_id         = aws_route53_zone.test.zone_id
+}
+
+resource "aws_acm_certificate_validation" "validation" {
+  certificate_arn = aws_acm_certificate.certificate.arn
+  validation_record_fqdns = [
+    for record in aws_route53_record.certificate_validation : record.fqdn
+  ]
+}
\ No newline at end of file
diff --git a/other_langs/terraform/acm/providers.tf b/other_langs/terraform/acm/providers.tf
new file mode 100644
index 000000000..3b900e069
--- /dev/null
+++ b/other_langs/terraform/acm/providers.tf
@@ -0,0 +1,15 @@
+provider "aws" {
+  region                      = "us-east-1"
+  s3_use_path_style           = true
+  skip_credentials_validation = true
+  skip_metadata_api_check     = true
+  skip_requesting_account_id  = true
+
+  endpoints {
+    acm     = "http://localhost:5000"
+    route53 = "http://localhost:5000"
+  }
+
+  access_key = "my-access-key"
+  secret_key = "my-secret-key"
+}
\ No newline at end of file


Test patch:
diff --git a/.github/workflows/tests_terraform_examples.yml b/.github/workflows/tests_terraform_examples.yml
new file mode 100644
index 000000000..46fa804d2
--- /dev/null
+++ b/.github/workflows/tests_terraform_examples.yml
@@ -0,0 +1,33 @@
+# Small, self contained Terraform examples
+# Scripts should be placed in:
+#    other_langs/terraform/service
+
+name: Terraform Examples
+on: [workflow_call]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        service: ["acm"]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.8"
+    - name: Start MotoServer
+      run: |
+        pip install build
+        python -m build
+        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh &
+        python scripts/ci_wait_for_server.py
+    - name: Run tests
+      run: |
+        mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
+        cd other_langs/terraform/${{ matrix.service }} && terraform init && terraform apply --auto-approve
diff --git a/tests/test_acm/test_acm.py b/tests/test_acm/test_acm.py
index 14db902b6..d0c46d20c 100644
--- a/tests/test_acm/test_acm.py
+++ b/tests/test_acm/test_acm.py
@@ -385,7 +385,7 @@ def test_request_certificate():
 
 
 @mock_acm
-def test_request_certificate_with_tags():
+def test_request_certificate_with_optional_arguments():
     client = boto3.client("acm", region_name="eu-central-1")
 
     token = str(uuid.uuid4())
@@ -402,6 +402,13 @@ def test_request_certificate_with_tags():
     assert "CertificateArn" in resp
     arn_1 = resp["CertificateArn"]
 
+    cert = client.describe_certificate(CertificateArn=arn_1)["Certificate"]
+    assert len(cert["SubjectAlternativeNames"]) == 3
+    assert len(cert["DomainValidationOptions"]) == 3
+    assert {option["DomainName"] for option in cert["DomainValidationOptions"]} == set(
+        cert["SubjectAlternativeNames"]
+    )
+
     resp = client.list_tags_for_certificate(CertificateArn=arn_1)
     tags = {item["Key"]: item.get("Value", "__NONE__") for item in resp["Tags"]}
     assert len(tags) == 2