Repository: getmoto/moto
Pull request #8168 (2024-09-27 13:35:46)

Task - Polly: update resources from latest aws polly describe-voices
I'm updating this because I needed to make `moto` work with the `Gregory` voice which is not included 


Patch:
diff --git a/docs/docs/services/sns.rst b/docs/docs/services/sns.rst
index 82b8be949..e9fca8a2c 100644
--- a/docs/docs/services/sns.rst
+++ b/docs/docs/services/sns.rst
@@ -51,7 +51,7 @@ sns
 - [X] publish
 - [X] publish_batch
   
-        The MessageStructure and MessageDeduplicationId-parameters have not yet been implemented.
+        The MessageDeduplicationId-parameter have not yet been implemented.
         
 
 - [ ] put_data_protection_policy
diff --git a/moto/batch/models.py b/moto/batch/models.py
index e0f72540f..9352c6944 100644
--- a/moto/batch/models.py
+++ b/moto/batch/models.py
@@ -638,12 +638,17 @@ class Job(threading.Thread, BaseModel, DockerModel, ManagedState):
         if not self.parameters:
             return command
 
-        new_command = [
-            command_part.replace(f"Ref::{param}", value)
+        return [
+            next(
+                (
+                    command_part.replace(f"Ref::{param}", value)
+                    for param, value in self.parameters.items()
+                    if f"Ref::{param}" in command_part
+                ),
+                command_part,
+            )
             for command_part in command
-            for param, value in self.parameters.items()
         ]
-        return new_command
 
     def run(self) -> None:
         """
diff --git a/moto/polly/resources.py b/moto/polly/resources.py
index 8bfdf2e1a..4abca05f1 100644
--- a/moto/polly/resources.py
+++ b/moto/polly/resources.py
@@ -5,6 +5,54 @@
 #
 # aws polly describe-voices | jq .Voices --indent 4
 VOICE_DATA = [
+    {
+        "Gender": "Female",
+        "Id": "Isabelle",
+        "LanguageCode": "fr-BE",
+        "LanguageName": "Belgian French",
+        "Name": "Isabelle",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Female",
+        "Id": "Danielle",
+        "LanguageCode": "en-US",
+        "LanguageName": "US English",
+        "Name": "Danielle",
+        "SupportedEngines": ["long-form", "neural"],
+    },
+    {
+        "Gender": "Male",
+        "Id": "Gregory",
+        "LanguageCode": "en-US",
+        "LanguageName": "US English",
+        "Name": "Gregory",
+        "SupportedEngines": ["long-form", "neural"],
+    },
+    {
+        "Gender": "Female",
+        "Id": "Burcu",
+        "LanguageCode": "tr-TR",
+        "LanguageName": "Turkish",
+        "Name": "Burcu",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Female",
+        "Id": "Jitka",
+        "LanguageCode": "cs-CZ",
+        "LanguageName": "Czech",
+        "Name": "Jitka",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Female",
+        "Id": "Sabrina",
+        "LanguageCode": "de-CH",
+        "LanguageName": "Swiss Standard German",
+        "Name": "Sabrina",
+        "SupportedEngines": ["neural"],
+    },
     {
         "Gender": "Male",
         "Id": "Kevin",
@@ -141,6 +189,14 @@ VOICE_DATA = [
         "Name": "Ola",
         "SupportedEngines": ["neural"],
     },
+    {
+        "Gender": "Female",
+        "Id": "Lisa",
+        "LanguageCode": "nl-BE",
+        "LanguageName": "Belgian Dutch",
+        "Name": "Lisa",
+        "SupportedEngines": ["neural"],
+    },
     {
         "Gender": "Male",
         "Id": "Ruben",
@@ -189,6 +245,22 @@ VOICE_DATA = [
         "Name": "Seoyeon",
         "SupportedEngines": ["neural", "standard"],
     },
+    {
+        "Gender": "Female",
+        "Id": "Kazuha",
+        "LanguageCode": "ja-JP",
+        "LanguageName": "Japanese",
+        "Name": "Kazuha",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Female",
+        "Id": "Tomoko",
+        "LanguageCode": "ja-JP",
+        "LanguageName": "Japanese",
+        "Name": "Tomoko",
+        "SupportedEngines": ["neural"],
+    },
     {
         "Gender": "Male",
         "Id": "Takumi",
@@ -363,7 +435,7 @@ VOICE_DATA = [
         "LanguageCode": "en-US",
         "LanguageName": "US English",
         "Name": "Matthew",
-        "SupportedEngines": ["neural", "standard"],
+        "SupportedEngines": ["generative", "neural", "standard"],
     },
     {
         "Gender": "Female",
@@ -468,7 +540,7 @@ VOICE_DATA = [
         "LanguageCode": "en-GB",
         "LanguageName": "British English",
         "Name": "Amy",
-        "SupportedEngines": ["neural", "standard"],
+        "SupportedEngines": ["generative", "neural", "standard"],
     },
     {
         "Gender": "Male",
@@ -534,6 +606,14 @@ VOICE_DATA = [
         "Name": "Mads",
         "SupportedEngines": ["standard"],
     },
+    {
+        "Gender": "Female",
+        "Id": "Sofie",
+        "LanguageCode": "da-DK",
+        "LanguageName": "Danish",
+        "Name": "Sofie",
+        "SupportedEngines": ["neural"],
+    },
     {
         "Gender": "Female",
         "Id": "Gwyneth",
@@ -564,6 +644,7 @@ VOICE_DATA = [
         "LanguageCode": "ar-AE",
         "LanguageName": "Gulf Arabic",
         "Name": "Hala",
+        "AdditionalLanguageCodes": ["arb"],
         "SupportedEngines": ["neural"],
     },
     {
@@ -578,10 +659,26 @@ VOICE_DATA = [
         "Gender": "Female",
         "Id": "Hannah",
         "LanguageCode": "de-AT",
-        "LanguageName": "German",
+        "LanguageName": "Austrian German",
         "Name": "Hannah",
         "SupportedEngines": ["neural"],
     },
+    {
+        "Gender": "Female",
+        "Id": "Ruth",
+        "LanguageCode": "en-US",
+        "LanguageName": "US English",
+        "Name": "Ruth",
+        "SupportedEngines": ["generative", "long-form", "neural"],
+    },
+    {
+        "Gender": "Male",
+        "Id": "Stephen",
+        "LanguageCode": "en-US",
+        "LanguageName": "US English",
+        "Name": "Stephen",
+        "SupportedEngines": ["neural"],
+    },
     {
         "Gender": "Female",
         "Id": "Kajal",
@@ -607,6 +704,14 @@ VOICE_DATA = [
         "Name": "Suvi",
         "SupportedEngines": ["neural"],
     },
+    {
+        "Gender": "Female",
+        "Id": "Niamh",
+        "LanguageCode": "en-IE",
+        "LanguageName": "Irish English",
+        "Name": "Niamh",
+        "SupportedEngines": ["neural"],
+    },
     {
         "Gender": "Male",
         "Id": "Arthur",
@@ -639,6 +744,55 @@ VOICE_DATA = [
         "Name": "Pedro",
         "SupportedEngines": ["neural"],
     },
+    {
+        "Gender": "Male",
+        "Id": "Sergio",
+        "LanguageCode": "es-ES",
+        "LanguageName": "Castilian Spanish",
+        "Name": "Sergio",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Male",
+        "Id": "Andres",
+        "LanguageCode": "es-MX",
+        "LanguageName": "Mexican Spanish",
+        "Name": "Andrés",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Male",
+        "Id": "Remi",
+        "LanguageCode": "fr-FR",
+        "LanguageName": "French",
+        "Name": "Rémi",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Male",
+        "Id": "Adriano",
+        "LanguageCode": "it-IT",
+        "LanguageName": "Italian",
+        "Name": "Adriano",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Male",
+        "Id": "Thiago",
+        "LanguageCode": "pt-BR",
+        "LanguageName": "Brazilian Portuguese",
+        "Name": "Thiago",
+        "SupportedEngines": ["neural"],
+    },
+    {
+        "Gender": "Male",
+        "Id": "Zayd",
+        "LanguageCode": "ar-AE",
+        "LanguageName": "Gulf Arabic",
+        "Name": "Zayd",
+        "AdditionalLanguageCodes": ["arb"],
+        "SupportedEngines": ["neural"],
+    },
 ]
 
 LANGUAGE_CODES = {voice["LanguageCode"] for voice in VOICE_DATA}
diff --git a/moto/sns/models.py b/moto/sns/models.py
index 74321285d..e5855df0a 100644
--- a/moto/sns/models.py
+++ b/moto/sns/models.py
@@ -85,6 +85,7 @@ class Topic(CloudFormationModel):
         message_attributes: Optional[Dict[str, Any]] = None,
         group_id: Optional[str] = None,
         deduplication_id: Optional[str] = None,
+        message_structure: Optional[str] = None,
     ) -> str:
         message_id = str(mock_random.uuid4())
         subscriptions, _ = self.sns_backend.list_subscriptions_by_topic(
@@ -98,7 +99,9 @@ class Topic(CloudFormationModel):
                 message_attributes=message_attributes,
                 group_id=group_id,
                 deduplication_id=deduplication_id,
+                message_structure=message_structure,
             )
+
         self.sent_notifications.append(
             (message_id, message, subject, message_attributes, group_id)
         )
@@ -222,14 +225,19 @@ class Subscription(BaseModel):
         message_attributes: Optional[Dict[str, Any]] = None,
         group_id: Optional[str] = None,
         deduplication_id: Optional[str] = None,
+        message_structure: Optional[str] = None,
     ) -> None:
         if self._filter_policy_matcher is not None:
             if not self._filter_policy_matcher.matches(message_attributes, message):
                 return
 
+        if message_structure == "json":
+            message = self._parse_message_structure(message, self.protocol)
+
         if self.protocol == "sqs":
             queue_name = self.endpoint.split(":")[-1]
             region = self.endpoint.split(":")[3]
+
             if self.attributes.get("RawMessageDelivery") != "true":
                 sqs_backends[self.account_id][region].send_message(
                     queue_name,
@@ -379,6 +387,23 @@ Notification
     def private_key(self) -> rsa.RSAPrivateKey:
         return rsa.generate_private_key(public_exponent=65537, key_size=2028)
 
+    def _parse_message_structure(self, message: str, protocol: str) -> str:
+        try:
+            structured_message = json.loads(message)
+            message = (
+                structured_message.get(self.protocol) or structured_message["default"]
+            )
+
+            return message
+        except json.JSONDecodeError:
+            raise InvalidParameterValue(
+                "Message is not valid JSON.",
+            )
+        except KeyError:
+            raise InvalidParameterValue(
+                f"Message does not contain {protocol} or default keys.",
+            )
+
 
 class PlatformApplication(BaseModel):
     def __init__(
@@ -657,6 +682,7 @@ class SNSBackend(BaseBackend):
         message_attributes: Optional[Dict[str, Any]] = None,
         group_id: Optional[str] = None,
         deduplication_id: Optional[str] = None,
+        message_structure: Optional[str] = None,
     ) -> str:
         if subject is not None and len(subject) > 100:
             # Note that the AWS docs around length are wrong: https://github.com/getmoto/moto/issues/1503
@@ -676,6 +702,9 @@ class SNSBackend(BaseBackend):
                 "An error occurred (InvalidParameter) when calling the Publish operation: Invalid parameter: Message too long"
             )
 
+        if message_structure is not None and message_structure != "json":
+            raise InvalidParameterValue("MessageStructure must be 'json' if provided")
+
         try:
             topic = self.get_topic(arn)  # type: ignore
 
@@ -703,6 +732,7 @@ class SNSBackend(BaseBackend):
                 message_attributes=message_attributes,
                 group_id=group_id,
                 deduplication_id=deduplication_id,
+                message_structure=message_structure,
             )
         except SNSNotFoundError:
             endpoint = self.get_endpoint(arn)  # type: ignore
@@ -1107,7 +1137,7 @@ class SNSBackend(BaseBackend):
         self, topic_arn: str, publish_batch_request_entries: List[Dict[str, Any]]
     ) -> Tuple[List[Dict[str, str]], List[Dict[str, Any]]]:
         """
-        The MessageStructure and MessageDeduplicationId-parameters have not yet been implemented.
+        The MessageDeduplicationId-parameters have not yet been implemented.
         """
         topic = self.get_topic(topic_arn)
 
@@ -1139,6 +1169,7 @@ class SNSBackend(BaseBackend):
                     message_attributes=entry.get("MessageAttributes", {}),
                     group_id=entry.get("MessageGroupId"),
                     deduplication_id=entry.get("MessageDeduplicationId"),
+                    message_structure=entry.get("MessageStructure"),
                 )
                 successful.append({"MessageId": message_id, "Id": entry["Id"]})
             except Exception as e:
diff --git a/moto/sns/responses.py b/moto/sns/responses.py
index a350e1c09..847b76db9 100644
--- a/moto/sns/responses.py
+++ b/moto/sns/responses.py
@@ -346,6 +346,7 @@ class SNSResponse(BaseResponse):
         subject = self._get_param("Subject")
         message_group_id = self._get_param("MessageGroupId")
         message_deduplication_id = self._get_param("MessageDeduplicationId")
+        message_structure = self._get_param("MessageStructure")
 
         message_attributes = self._parse_message_attributes()
 
@@ -375,6 +376,7 @@ class SNSResponse(BaseResponse):
                 message_attributes=message_attributes,
                 group_id=message_group_id,
                 deduplication_id=message_deduplication_id,
+                message_structure=message_structure,
             )
         except ValueError as err:
             error_response = self._error("InvalidParameter", str(err))


Test patch:
diff --git a/tests/test_batch/test_utils.py b/tests/test_batch/test_utils.py
index 870b2bad0..2d080f9aa 100644
--- a/tests/test_batch/test_utils.py
+++ b/tests/test_batch/test_utils.py
@@ -1,6 +1,9 @@
+from unittest.mock import patch
+
 import pytest
 
 from moto.batch.exceptions import ValidationError
+from moto.batch.models import Job
 from moto.batch.utils import JobStatus
 
 
@@ -67,3 +70,14 @@ def test_JobStatus_status_transitions():
         else:
             assert before_status == JobStatus.STARTING
             assert after_status == JobStatus.RUNNING
+
+
+@patch.object(Job, "__init__", lambda self, *args, **kwargs: None)
+def test_add_parameters_to_command():
+    _parameters = {"dossier_md5": "abc", "study_id": "T123BC00001"}
+    _command = ["--dossier_md5", "Ref::dossier_md5", "--study_id", "Ref::study_id"]
+
+    job = Job()  # noqa
+    job.parameters = _parameters
+    output = job._add_parameters_to_command(command=_command)
+    assert output == ["--dossier_md5", "abc", "--study_id", "T123BC00001"]
diff --git a/tests/test_sns/test_publish_batch.py b/tests/test_sns/test_publish_batch.py
index a3da85f5e..9d9aa476a 100644
--- a/tests/test_sns/test_publish_batch.py
+++ b/tests/test_sns/test_publish_batch.py
@@ -180,3 +180,64 @@ def test_publish_batch_to_sqs_raw():
 
     assert ("foo", None) in messages
     assert ("bar", {"a": {"StringValue": "v", "DataType": "String"}}) in messages
+
+
+@mock_aws
+def test_publish_with_with_message_structure_json():
+    sns = boto3.client("sns", region_name="us-east-1")
+
+    topic = sns.create_topic(Name="some-topic")
+
+    sqs = boto3.resource("sqs", region_name="us-east-1")
+    queue = sqs.create_queue(QueueName="test-queue")
+
+    sns.subscribe(
+        TopicArn=topic["TopicArn"],
+        Protocol="sqs",
+        Endpoint=queue.attributes["QueueArn"],
+    )
+
+    entries = [
+        {
+            "Id": "no-message-structure",
+            "Message": "no-message-structure",
+        },
+        {
+            "Id": "json-message-structure",
+            "Message": json.dumps(
+                {
+                    "default": "default-message",
+                    "sqs": "queue-message",
+                }
+            ),
+            "MessageStructure": "json",
+        },
+        {
+            "Id": "json-message-structure-fallback-to-default",
+            "Message": json.dumps(
+                {
+                    "default": "default-message",
+                    "email": "email-message",
+                }
+            ),
+            "MessageStructure": "json",
+        },
+    ]
+
+    sns.publish_batch(
+        TopicArn=topic["TopicArn"],
+        PublishBatchRequestEntries=entries,
+    )
+
+    queue_messages = queue.receive_messages(MaxNumberOfMessages=10)
+
+    assert len(queue_messages) == 3
+
+    first_queue_message = json.loads(queue_messages[0].body)
+    assert first_queue_message["Message"] == "no-message-structure"
+
+    second_queue_message = json.loads(queue_messages[1].body)
+    assert second_queue_message["Message"] == "queue-message"
+
+    third_queue_message = json.loads(queue_messages[2].body)
+    assert third_queue_message["Message"] == "default-message"
diff --git a/tests/test_sns/test_publishing_boto3.py b/tests/test_sns/test_publishing_boto3.py
index f29bd1abd..f1d61071f 100644
--- a/tests/test_sns/test_publishing_boto3.py
+++ b/tests/test_sns/test_publishing_boto3.py
@@ -2500,3 +2500,114 @@ def test_filtering_message_body_nested_multiple_records_match():
     assert message_attributes == [{"match": {"Type": "String", "Value": "body"}}]
     message_bodies = [json.loads(json.loads(m.body)["Message"]) for m in messages]
     assert message_bodies == [payload]
+
+
+@mock_aws
+def test_publish_with_message_structure_json():
+    sns = boto3.resource("sns", region_name="us-east-1")
+
+    topic = sns.create_topic(Name="some-topic")
+
+    sqs = boto3.resource("sqs", region_name="us-east-1")
+    queue = sqs.create_queue(QueueName="test-queue")
+
+    topic.subscribe(
+        TopicArn=topic.arn,
+        Protocol="sqs",
+        Endpoint=queue.attributes["QueueArn"],
+    )
+
+    topic.publish(
+        Message=json.dumps(
+            {
+                "default": "message",
+                "sqs": "queue-message",
+            }
+        ),
+        MessageStructure="json",
+    )
+
+    queue_message = json.loads(queue.receive_messages()[0].body)
+    assert queue_message["Message"] == "queue-message"
+
+
+@mock_aws
+def test_publish_with_message_structure_json_fallback_to_default():
+    sns = boto3.resource("sns", region_name="us-east-1")
+
+    topic = sns.create_topic(Name="some-topic")
+
+    sqs = boto3.resource("sqs", region_name="us-east-1")
+    queue = sqs.create_queue(QueueName="test-queue")
+
+    topic.subscribe(
+        TopicArn=topic.arn,
+        Protocol="sqs",
+        Endpoint=queue.attributes["QueueArn"],
+    )
+
+    topic.publish(
+        Message=json.dumps(
+            {
+                "default": "message",
+                "email": "email-message",
+            }
+        ),
+        MessageStructure="json",
+    )
+
+    queue_message = json.loads(queue.receive_messages()[0].body)
+    assert queue_message["Message"] == "message"
+
+
+@mock_aws
+def test_publish_with_message_structure_errors():
+    sns = boto3.resource("sns", region_name="us-east-1")
+
+    topic = sns.create_topic(Name="some-topic")
+
+    sqs = boto3.resource("sqs", region_name="us-east-1")
+    queue = sqs.create_queue(QueueName="test-queue")
+
+    topic.subscribe(
+        TopicArn=topic.arn,
+        Protocol="sqs",
+        Endpoint=queue.attributes["QueueArn"],
+    )
+
+    try:
+        topic.publish(
+            Message="invalid-json",
+            MessageStructure="json",
+        )
+    except ClientError as err:
+        assert err.response["Error"]["Code"] == "InvalidParameterValue"
+        assert err.response["Error"]["Message"] == "Message is not valid JSON."
+
+    try:
+        topic.publish(
+            Message=json.dumps(
+                {
+                    "email": "email-message",
+                }
+            ),
+            MessageStructure="json",
+        )
+    except ClientError as err:
+        assert err.response["Error"]["Code"] == "InvalidParameterValue"
+        assert (
+            err.response["Error"]["Message"]
+            == "Message does not contain sqs or default keys."
+        )
+
+    try:
+        topic.publish(
+            Message="no-json-structure",
+            MessageStructure="no-json",
+        )
+    except ClientError as err:
+        assert err.response["Error"]["Code"] == "InvalidParameterValue"
+        assert (
+            err.response["Error"]["Message"]
+            == "MessageStructure must be 'json' if provided"
+        )