Repository: getmoto/moto
Pull request #7824 (2024-07-05 11:52:04)

SESv2: send_email() response fix
Resolves https://github.com/getmoto/moto/issues/7816


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9a238238b..35d14179b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,31 @@
 Moto Changelog
 ==============
 
+5.0.11
+-----
+Docker Digest for 5.0.11: _sha256:438f7fbb5fa1dff2cf0887c59466ff78bed5aaca9ea7b5cf54d6a41fc2418e28_
+
+    New Methods:
+        * ServiceDiscovery:
+            * deregister_instance()
+            * discover_instances()
+            * discover_instances_revision()
+            * get_instance()
+            * get_instances_health_status()
+            * list_instances()
+            * register_instance()
+            * update_http_namespace()
+            * update_instance_custom_health_status()
+
+    Miscellaneous:
+        * DynamoDB: transact_write_items() no longer throws a ValidationException when passing multiple set actions, only when passing multiple set clauses
+        * DynamoDB: transact_write_items() no longer throws an Exception when ExpressionAttributeNames are not provided
+          (Both bugs were introduced in 5.0.10)
+
+        * IOT-data: update_thing_shadow() now calculates the delta correctly
+        * ResourceGroupsTagging: get_resources() now supports EFS resources
+
+
 5.0.10
 -----
 Docker Digest for 5.0.10: _sha256:bfb9cd2a437fc7c754b3a6a66b7fb528ec1a53e0c683e8b75514bff81543cf55_
diff --git a/docs/docs/services/athena.rst b/docs/docs/services/athena.rst
index 12145c85a..c90ccaf82 100644
--- a/docs/docs/services/athena.rst
+++ b/docs/docs/services/athena.rst
@@ -88,6 +88,8 @@ athena
 
         .. note:: The exact QueryExecutionId is not relevant here, but will likely be whatever value is returned by start_query_execution
 
+        Query results will also be stored in the S3 output location (in CSV format).
+
         
 
 - [ ] get_query_runtime_statistics
diff --git a/moto/__init__.py b/moto/__init__.py
index 5ef3b79fe..c20d45070 100644
--- a/moto/__init__.py
+++ b/moto/__init__.py
@@ -1,4 +1,4 @@
 from moto.core.decorator import mock_aws as mock_aws
 
 __title__ = "moto"
-__version__ = "5.0.11.dev"
+__version__ = "5.0.12.dev"
diff --git a/moto/athena/models.py b/moto/athena/models.py
index 93a563ae2..1ca9d35bd 100644
--- a/moto/athena/models.py
+++ b/moto/athena/models.py
@@ -5,6 +5,8 @@ from typing import Any, Dict, List, Optional
 from moto.core.base_backend import BackendDict, BaseBackend
 from moto.core.common_models import BaseModel
 from moto.moto_api._internal import mock_random
+from moto.s3.models import s3_backends
+from moto.s3.utils import bucket_and_name_from_url
 from moto.utilities.paginator import paginate
 from moto.utilities.utils import get_partition
 
@@ -232,8 +234,17 @@ class AthenaBackend(BaseBackend):
             execution_parameters=execution_parameters,
         )
         self.executions[execution.id] = execution
+
+        self._store_predefined_query_results(execution.id)
+
         return execution.id
 
+    def _store_predefined_query_results(self, exec_id: str) -> None:
+        if exec_id not in self.query_results and self.query_results_queue:
+            self.query_results[exec_id] = self.query_results_queue.pop(0)
+
+            self._store_query_result_in_s3(exec_id)
+
     def get_query_execution(self, exec_id: str) -> Execution:
         return self.executions[exec_id]
 
@@ -285,9 +296,11 @@ class AthenaBackend(BaseBackend):
 
         .. note:: The exact QueryExecutionId is not relevant here, but will likely be whatever value is returned by start_query_execution
 
+        Query results will also be stored in the S3 output location (in CSV format).
+
         """
-        if exec_id not in self.query_results and self.query_results_queue:
-            self.query_results[exec_id] = self.query_results_queue.pop(0)
+        self._store_predefined_query_results(exec_id)
+
         results = (
             self.query_results[exec_id]
             if exec_id in self.query_results
@@ -295,6 +308,31 @@ class AthenaBackend(BaseBackend):
         )
         return results
 
+    def _store_query_result_in_s3(self, exec_id: str) -> None:
+        try:
+            output_location = self.executions[exec_id].config["OutputLocation"]
+            bucket, key = bucket_and_name_from_url(output_location)
+
+            query_result = ""
+            for row in self.query_results[exec_id].rows:
+                query_result += ",".join(
+                    [
+                        f"\"{r['VarCharValue']}\"" if "VarCharValue" in r else ""
+                        for r in row["Data"]
+                    ]
+                )
+                query_result += "\n"
+
+            s3_backends[self.account_id][self.partition].put_object(
+                bucket_name=bucket,  # type: ignore
+                key_name=key,  # type: ignore
+                value=query_result.encode("utf-8"),
+            )
+        except:  # noqa
+            # Execution may not exist
+            # OutputLocation may not exist
+            pass
+
     def stop_query_execution(self, exec_id: str) -> None:
         execution = self.executions[exec_id]
         execution.status = "CANCELLED"
diff --git a/moto/dynamodb/parsing/ast_nodes.py b/moto/dynamodb/parsing/ast_nodes.py
index 81afbfae5..95875d90d 100644
--- a/moto/dynamodb/parsing/ast_nodes.py
+++ b/moto/dynamodb/parsing/ast_nodes.py
@@ -40,7 +40,8 @@ class Node(metaclass=abc.ABCMeta):
             if len(set_attributes) != len(set(set_attributes)):
                 raise DuplicateUpdateExpression(set_attributes)
 
-            if limit_set_actions and len(set_actions) > 1:
+            set_clauses = self.find_clauses([UpdateExpressionSetClause])
+            if limit_set_actions and len(set_clauses) > 1:
                 raise MockValidationException(
                     'Invalid UpdateExpression: The "SET" section can only be used once in an update expression;'
                 )
diff --git a/moto/ec2/models/instances.py b/moto/ec2/models/instances.py
index 615be6807..91efe920a 100644
--- a/moto/ec2/models/instances.py
+++ b/moto/ec2/models/instances.py
@@ -935,5 +935,5 @@ class InstanceBackend:
             )[0]
         )
         version = launch_template_arg.get("Version", template.latest_version_number)
-        template_version = template.get_version(int(version))
+        template_version = template.get_version(version)
         return template_version
diff --git a/moto/moto_proxy/certificate_creator.py b/moto/moto_proxy/certificate_creator.py
index c5699a0b4..db0c29dbe 100644
--- a/moto/moto_proxy/certificate_creator.py
+++ b/moto/moto_proxy/certificate_creator.py
@@ -64,10 +64,13 @@ class CertificateCreator:
             # All requests that match this wildcard certificate will reuse it
 
             wildcard_name = f"*.{'.'.join(full_name.split('.')[1:])}"
-            server_csr = f"{self.certdir.rstrip('/')}/{wildcard_name}.csr"
+            # In Linux we could use asterisk as part of the filename, but not on Windows
+            # So we need to pick a different (unique) filename that works everywhere
+            wildcard_filename = f"_.{'.'.join(full_name.split('.')[1:])}"
+            server_csr = f"{self.certdir.rstrip('/')}/{wildcard_filename}.csr"
 
             # Verify if the certificate already exists
-            certpath = f"{self.certdir.rstrip('/')}/{wildcard_name}.crt"
+            certpath = f"{self.certdir.rstrip('/')}/{wildcard_filename}.crt"
             if not os.path.isfile(certpath):
                 # Create a Config-file that contains the wildcard-name
                 with open(f"{self.certdir.rstrip('/')}/req.conf.tmpl", "r") as f:
@@ -77,12 +80,12 @@ class CertificateCreator:
                     "{{wildcard_name}}", wildcard_name
                 )
                 config_template_name = (
-                    f"{self.certdir.rstrip('/')}/{wildcard_name}.conf"
+                    f"{self.certdir.rstrip('/')}/{wildcard_filename}.conf"
                 )
                 with open(config_template_name, "w") as f:
                     f.write(config_template)
 
-                # Create an Certificate Signing Request
+                # Create a Certificate Signing Request
                 #
                 subject = f"/CN={full_name}"[0:64]
                 commands = [
diff --git a/moto/sesv2/responses.py b/moto/sesv2/responses.py
index 124f7db71..c8c09a663 100644
--- a/moto/sesv2/responses.py
+++ b/moto/sesv2/responses.py
@@ -7,7 +7,6 @@ from urllib.parse import unquote
 
 from moto.core.responses import BaseResponse
 
-from ..ses.responses import SEND_EMAIL_RESPONSE
 from .models import SESV2Backend, sesv2_backends
 
 
@@ -57,9 +56,7 @@ class SESV2Response(BaseResponse):
         elif "Template" in content:
             raise NotImplementedError("Template functionality not ready")
 
-        # use v1 templates as response same in v1 and v2
-        template = self.response_template(SEND_EMAIL_RESPONSE)
-        return template.render(message=message)
+        return json.dumps({"MessageId": message.id})
 
     def create_contact_list(self) -> str:
         params = json.loads(self.body)


Test patch:
diff --git a/.github/workflows/tests_proxymode.yml b/.github/workflows/tests_proxymode.yml
index af7c799db..9abf533be 100644
--- a/.github/workflows/tests_proxymode.yml
+++ b/.github/workflows/tests_proxymode.yml
@@ -3,11 +3,17 @@ on: [workflow_call]
 
 jobs:
   test:
-    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
+        os: ["ubuntu-latest", "windows-latest"]
         python-version: ["3.10", "3.11", "3.12"]
+        include:
+          - os: ubuntu-latest
+            path: ~/.cache/pip
+          - os: windows-latest
+            path: ~\AppData\Local\pip\Cache
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
@@ -24,8 +30,10 @@ jobs:
     - name: pip cache
       uses: actions/cache@v4
       with:
-        path: ${{ steps.pip-cache.outputs.dir }}
-        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
+        path: ${{ matrix.path }}
+        key: pip-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
+        restore-keys: |
+          pip-${{ runner.os }}-${{ matrix.python-version }}-
     - name: Update pip
       run: |
         python -m pip install --upgrade pip
@@ -33,23 +41,38 @@ jobs:
       run: |
         pip install -r requirements-tests.txt
         pip install .[all,server]
-    - name: Start MotoProxy
+    - name: Start MotoProxy on Linux
+      if: ${{ matrix.os == 'ubuntu-latest' }}
       run: |
-        moto_proxy -h > moto_proxy.log
         moto_proxy -H 0.0.0.0 -v > moto_proxy.log &
-    - name: Test ProxyMode
+    - name: Test ProxyMode on Linux
+      if: ${{ matrix.os == 'ubuntu-latest' }}
       env:
         TEST_PROXY_MODE: ${{ true }}
       run: |
         pytest -sv tests/test_acmpca tests/test_athena tests/test_awslambda tests/test_apigateway tests/test_core/test_proxy.py tests/test_s3
+    - name: Test ProxyMode on Windows
+      if: ${{ matrix.os == 'windows-latest' }}
+      shell: pwsh
+      env:
+        TEST_PROXY_MODE: ${{ true }}
+      # We should start moto_proxy in a different step
+      # But I can't keep the process alive, across different steps
+      # It does run in the background - but it looks like the process is killed at the end of the step
+      # Starting the proxy in the same step has the same outcome though, even if it's not pretty
+      #
+      # This step does not test everything, because it takes ages on Windows for some reason
+      run: |
+        moto_proxy -v &
+        pytest -sv tests/test_acmpca tests/test_core/test_proxy.py
     - name: "Stop MotoProxy"
-      if: always()
+      if: ${{ matrix.os == 'ubuntu-latest' }}
       run: |
         kill $(lsof -t -i:5005)
     - name: Archive Proxy logs
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: motoproxy-${{ matrix.python-version }}
+        name: motoproxy-${{ matrix.os }}-${{ matrix.python-version }}
         path: |
           moto_proxy.log
\ No newline at end of file
diff --git a/tests/test_athena/__init__.py b/tests/test_athena/__init__.py
index 08a1c1568..1822c621a 100644
--- a/tests/test_athena/__init__.py
+++ b/tests/test_athena/__init__.py
@@ -1,1 +1,59 @@
-# This file is intentionally left blank.
+import os
+from functools import wraps
+from uuid import uuid4
+
+import boto3
+
+from moto import mock_aws
+from moto.s3.responses import DEFAULT_REGION_NAME
+from tests.test_s3 import empty_bucket
+
+
+def athena_aws_verified(func):
+    """
+    Function that is verified to work against AWS.
+    Can be run against AWS at any time by setting:
+      MOTO_TEST_ALLOW_AWS_REQUEST=true
+
+    If this environment variable is not set, the function runs in a `mock_aws` context.
+
+    This decorator will:
+      - Create a bucket
+      - Run the test and pass the bucket_name as an argument
+      - Delete the objects and the bucket itself
+    """
+
+    @wraps(func)
+    def pagination_wrapper():
+        bucket_name = str(uuid4())
+
+        allow_aws_request = (
+            os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
+        )
+
+        if allow_aws_request:
+            resp = create_bucket_and_test(bucket_name, is_aws=True)
+        else:
+            with mock_aws():
+                resp = create_bucket_and_test(bucket_name, is_aws=False)
+        return resp
+
+    def create_bucket_and_test(bucket_name: str, is_aws: bool):
+        client = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
+
+        client.create_bucket(Bucket=bucket_name)
+        client.put_bucket_tagging(
+            Bucket=bucket_name,
+            Tagging={"TagSet": [{"Key": "environment", "Value": "moto_tests"}]},
+        )
+        try:
+            resp = func(bucket_name, is_aws)
+        finally:
+            ### CLEANUP ###
+
+            empty_bucket(client, bucket_name)
+            client.delete_bucket(Bucket=bucket_name)
+
+        return resp
+
+    return pagination_wrapper
diff --git a/tests/test_athena/test_athena_integration.py b/tests/test_athena/test_athena_integration.py
new file mode 100644
index 000000000..6af2ffca8
--- /dev/null
+++ b/tests/test_athena/test_athena_integration.py
@@ -0,0 +1,155 @@
+import json
+from unittest import SkipTest
+from uuid import uuid4
+
+import boto3
+import pytest
+import requests
+
+from moto import settings
+from tests.test_athena import athena_aws_verified
+
+DATA = ""
+DATA += json.dumps({"k1": "r1_v1", "k2": "r1_v2", "k3": 1}) + "\n"
+DATA += json.dumps({"k1": "r2_v1", "k2": "r2_v2", "k3": 2}) + "\n"
+DATA += json.dumps({"k1": "r3_v1", "k2": "r3_v2"})
+
+ROWS_DATA = [
+    {"Data": [{"VarCharValue": "k1"}, {"VarCharValue": "k2"}, {"VarCharValue": "k3"}]},
+    {
+        "Data": [
+            {"VarCharValue": "r1_v1"},
+            {"VarCharValue": "r1_v2"},
+            {"VarCharValue": "1"},
+        ]
+    },
+    {
+        "Data": [
+            {"VarCharValue": "r2_v1"},
+            {"VarCharValue": "r2_v2"},
+            {"VarCharValue": "2"},
+        ]
+    },
+    {"Data": [{"VarCharValue": "r3_v1"}, {"VarCharValue": "r3_v2"}, {}]},
+]
+COLUMN_INFO = [
+    {
+        "CatalogName": "hive",
+        "SchemaName": "",
+        "TableName": "",
+        "Name": "k1",
+        "Label": "k1",
+        "Type": "varchar",
+        "Precision": 2147483647,
+        "Scale": 0,
+        "Nullable": "UNKNOWN",
+        "CaseSensitive": True,
+    },
+    {
+        "CatalogName": "hive",
+        "SchemaName": "",
+        "TableName": "",
+        "Name": "k2",
+        "Label": "k2",
+        "Type": "varchar",
+        "Precision": 2147483647,
+        "Scale": 0,
+        "Nullable": "UNKNOWN",
+        "CaseSensitive": True,
+    },
+    {
+        "CatalogName": "hive",
+        "SchemaName": "",
+        "TableName": "",
+        "Name": "k3",
+        "Label": "k3",
+        "Type": "integer",
+        "Precision": 10,
+        "Scale": 0,
+        "Nullable": "UNKNOWN",
+        "CaseSensitive": False,
+    },
+]
+
+
+@athena_aws_verified
+@pytest.mark.aws_verified
+def test_athena_csv_result(bucket_name=None, is_aws: bool = False):
+    if not settings.TEST_DECORATOR_MODE:
+        raise SkipTest("No point in testing this outside of decorators")
+
+    s3 = boto3.client("s3", "us-east-1")
+    s3.put_object(Bucket=bucket_name, Key="input/data1.json", Body=DATA.encode("utf-8"))
+
+    athena = boto3.client("athena", "us-east-1")
+
+    config = {"OutputLocation": f"s3://{bucket_name}/output/"}
+    db_name = f"db_{str(uuid4())[0:6]}"
+
+    athena.start_query_execution(
+        QueryString=f"create database {db_name}", ResultConfiguration=config
+    )
+
+    context = {"Database": db_name}
+    CREATE_TABLE = f"""CREATE EXTERNAL TABLE mytable (
+k1 string,
+k2 string,
+k3 int
+) 
+ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
+STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat'
+LOCATION 's3://{bucket_name}/input/'; """
+    get_query_results(CREATE_TABLE, athena, config, context)
+
+    if not is_aws:
+        athena_result = {
+            "results": [
+                {
+                    "rows": ROWS_DATA,
+                    "column_info": COLUMN_INFO,
+                }
+            ]
+        }
+        requests.post(
+            "http://motoapi.amazonaws.com/moto-api/static/athena/query-results",
+            json=athena_result,
+        )
+
+    query_exec_id, resp = get_query_results(
+        "SELECT * FROM mytable", athena, config, context
+    )
+    assert resp["ResultSet"] == {
+        "Rows": ROWS_DATA,
+        "ResultSetMetadata": {"ColumnInfo": COLUMN_INFO},
+    }
+
+    file = s3.get_object(Bucket=bucket_name, Key=f"output/{query_exec_id}.csv")
+    assert (
+        file["Body"].read()
+        == b'"k1","k2","k3"\n"r1_v1","r1_v2","1"\n"r2_v1","r2_v2","2"\n"r3_v1","r3_v2",\n'
+    )
+
+    get_query_results("DROP TABLE mytable", athena, config, context)
+
+    get_query_results(f"DROP database {db_name}", athena, config, context)
+
+
+def get_query_results(CREATE_TABLE, athena, config, context):
+    resp = athena.start_query_execution(
+        QueryString=CREATE_TABLE,
+        QueryExecutionContext=context,
+        ResultConfiguration=config,
+    )
+    query_execution_id = resp["QueryExecutionId"]
+
+    status = ""
+    while status not in ["SUCCEEDED", "FAILED"]:
+        resp = athena.get_query_execution(QueryExecutionId=query_execution_id)[
+            "QueryExecution"
+        ]
+
+        status = resp["Status"]["State"]
+
+    resp = athena.get_query_results(QueryExecutionId=query_execution_id)
+    return query_execution_id, resp
diff --git a/tests/test_dynamodb/exceptions/test_dynamodb_transactions.py b/tests/test_dynamodb/exceptions/test_dynamodb_transactions.py
index 90877bd28..722f10565 100644
--- a/tests/test_dynamodb/exceptions/test_dynamodb_transactions.py
+++ b/tests/test_dynamodb/exceptions/test_dynamodb_transactions.py
@@ -85,7 +85,7 @@ def test_transact_write_items__put_and_delete_on_same_item(table_name=None):
 
 @pytest.mark.aws_verified
 @dynamodb_aws_verified(add_range=True)
-def test_transact_write_items__update_with_multiple_sets(table_name=None):
+def test_transact_write_items__update_with_multiple_set_clauses(table_name=None):
     dynamodb = boto3.client("dynamodb", region_name="us-east-1")
 
     with pytest.raises(ClientError) as exc:
@@ -112,6 +112,28 @@ def test_transact_write_items__update_with_multiple_sets(table_name=None):
         == 'Invalid UpdateExpression: The "SET" section can only be used once in an update expression;'
     )
 
+    # Note that we can do this by simply separating the statements with a comma
+    dynamodb.transact_write_items(
+        TransactItems=[
+            {
+                "Update": {
+                    "TableName": table_name,
+                    "Key": {"pk": {"S": "s"}, "sk": {"S": "t"}},
+                    "UpdateExpression": "SET a1 = :v1, a2 = :v2",
+                    "ExpressionAttributeValues": {
+                        ":v1": {"S": "v1"},
+                        ":v2": {"S": "v1"},
+                    },
+                }
+            }
+        ]
+    )
+
+    inventory = dynamodb.scan(TableName=table_name)["Items"]
+    assert inventory == [
+        {"a1": {"S": "v1"}, "sk": {"S": "t"}, "a2": {"S": "v1"}, "pk": {"S": "s"}}
+    ]
+
 
 @mock_aws
 def test_transact_write_items__too_many_transactions():
diff --git a/tests/test_sesv2/test_sesv2.py b/tests/test_sesv2/test_sesv2.py
index 0a0a37d19..b9e951401 100644
--- a/tests/test_sesv2/test_sesv2.py
+++ b/tests/test_sesv2/test_sesv2.py
@@ -41,7 +41,7 @@ def test_send_email(ses_v1):  # pylint: disable=redefined-outer-name
     assert e.value.response["Error"]["Code"] == "MessageRejected"
 
     ses_v1.verify_domain_identity(Domain="example.com")
-    conn.send_email(**kwargs)
+    resp = conn.send_email(**kwargs)
     send_quota = ses_v1.get_send_quota()
 
     # Verify
@@ -53,6 +53,8 @@ def test_send_email(ses_v1):  # pylint: disable=redefined-outer-name
         msg: Message = backend.sent_messages[0]
         assert msg.subject == "test subject"
         assert msg.body == "test body"
+        assert "MessageId" in resp
+        assert resp["MessageId"] is not None
 
 
 @mock_aws