Repository: getmoto/moto (python)

Issue:
IoT Data Shadow Delta with complex datastructure exposes jsondiff symbol and raises TypeError
I'm using Moto 5.0.21, and jsondiff 2.2.1.

When deleting items from a second-level list in an IoT data shadow, there's a `jsondiff.deleted` symbol exposed, and breaking the subsequent JSON parsing. I suppose this is because the code only deletes those symbols from the first level of a dict: [iotdata/models.py#L81](https://github.com/bblommers/moto/blob/4f521ba49182be973b31f52791a50bdcd6802c02/moto/iotdata/models.py#L81)

Pytest showing the issue:

```Python
from moto import mock_aws
import boto3


@mock_aws
def test_list():
    iot_client = boto3.client(service_name="iot", region_name="eu-central-1")
    iot_client.create_thing(thingName="example-thing")

    iot_data_client = boto3.client(service_name="iot-data", region_name="eu-central-1")

    iot_data_client.update_thing_shadow(
        thingName="example-thing",
        payload=json.dumps(
            {
                "state": {
                    "reported": {
                        "list": [
                            ["value_1"],
                            ["value_2"],
                        ]
                    }
                }
            }
        ),
    )
    iot_data_client.update_thing_shadow(
        thingName="example-thing",
        payload=json.dumps(
            {
                "state": {
                    "desired": {
                        "list": [
                            ["value_1"],
                        ]
                    }
                }
            }
        ),
    )

    iot_data_client.get_thing_shadow(thingName="example-thing")
    # throws TypeError: keys must be str, int, float, bool or None, not Symbol
```

Traceback:
```Python
venv/python-3.11.9/lib/python3.11/site-packages/botocore/client.py:565: in _api_call                                                     
    return self._make_api_call(operation_name, kwargs)                                                                                   
venv/python-3.11.9/lib/python3.11/site-packages/botocore/client.py:1001: in _make_api_call                                               
    http, parsed_response = self._make_request(                                                                                          
venv/python-3.11.9/lib/python3.11/site-packages/botocore/client.py:1027: in _make_request                                                
    return self._endpoint.make_request(operation_model, request_dict)                                                                    
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:119: in make_request                                                
    return self._send_request(request_dict, operation_model)                                                                             
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:202: in _send_request                                               
    while self._needs_retry(                                                                                                             
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:354: in _needs_retry                                                
    responses = self._event_emitter.emit(                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:412: in emit                                                           
    return self._emitter.emit(aliased_event_name, **kwargs)                                                                              
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:256: in emit                                                           
    return self._emit(event_name, kwargs)                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:239: in _emit                                                          
    response = handler(**kwargs)                                                                                                         
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:207: in __call__                                                
    if self._checker(**checker_kwargs):                                                                                                  
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:284: in __call__                                                
    should_retry = self._should_retry(                                                                                                   
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:307: in _should_retry                                           
    return self._checker(                                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:363: in __call__                                                
    checker_response = checker(                                                                                                          
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:247: in __call__                                                
    return self._check_caught_exception(                                                                                                 
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:416: in _check_caught_exception                                 
    raise caught_exception                                                                                                               
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:278: in _do_get_response                                            
    responses = self._event_emitter.emit(event_name, request=request)                                                                    
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:412: in emit                                                           
    return self._emitter.emit(aliased_event_name, **kwargs)                                                                              
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:256: in emit                                                           
    return self._emit(event_name, kwargs)                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:239: in _emit                                                          
    response = handler(**kwargs)                                                                                                         
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/botocore_stubber.py:38: in __call__                                            
    response = self.process_request(request)                                                                                             
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/botocore_stubber.py:88: in process_request                                     
    status, headers, body = method_to_execute(                                                                                           
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/responses.py:291: in dispatch                                                  
    return cls()._dispatch(*args, **kwargs)                                                                                              
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/responses.py:503: in _dispatch                                                 
    return self.call_action()                                                                                                            
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/responses.py:591: in call_action
    response = method()
venv/python-3.11.9/lib/python3.11/site-packages/moto/iotdata/responses.py:45: in get_thing_shadow
    return json.dumps(payload.to_dict())
/usr/local/lib/python3.11/json/__init__.py:231: in dumps
    return _default_encoder.encode(obj)
/usr/local/lib/python3.11/json/encoder.py:200: in encode
    chunks = self.iterencode(o, _one_shot=True)
```

Also, I'm not sure if AWS would even detect such an item deletion, or if it would just see the list as completely new. I don't have the resources accessible right now to test that.

Pull request:
IOTData: update_thing_shadow() now properly handles complex nested deltas
Fixes #8331 

If there are other scenarios that I've missed, and the diff computation becomes even more complex, it might make sense to move away from `jsondiff` altogether, and add our own implementation. The whole point of `jsondiff` is to not have to deal with complexity like this. But for now, this will do.

Patch:
diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py
index 03341a69d..c50516152 100644
--- a/moto/ec2/utils.py
+++ b/moto/ec2/utils.py
@@ -90,7 +90,7 @@ def random_resource_id(size: int = 8) -> str:
     return "".join(random.choice(HEX_CHARS) for _ in range(size))
 
 
-def random_id(prefix: str = "", size: int = 8) -> str:
+def random_id(prefix: str = "", size: int = 17) -> str:
     return f"{prefix}-{random_resource_id(size)}"
 
 
@@ -99,7 +99,7 @@ def random_ami_id() -> str:
 
 
 def random_instance_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["instance"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["instance"])
 
 
 def random_reservation_id() -> str:
@@ -107,11 +107,11 @@ def random_reservation_id() -> str:
 
 
 def random_security_group_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["security-group"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["security-group"])
 
 
 def random_security_group_rule_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["security-group-rule"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["security-group-rule"])
 
 
 def random_fleet_id() -> str:
@@ -181,7 +181,7 @@ def random_vpc_id() -> str:
 
 
 def random_vpc_ep_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["vpc-endpoint"], size=8)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["vpc-endpoint"])
 
 
 def random_vpc_cidr_association_id() -> str:
@@ -201,9 +201,7 @@ def random_internet_gateway_id() -> str:
 
 
 def random_egress_only_internet_gateway_id() -> str:
-    return random_id(
-        prefix=EC2_RESOURCE_TO_PREFIX["egress-only-internet-gateway"], size=17
-    )
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["egress-only-internet-gateway"])
 
 
 def random_route_table_id() -> str:
@@ -227,27 +225,27 @@ def random_eni_attach_id() -> str:
 
 
 def random_nat_gateway_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["nat-gateway"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["nat-gateway"])
 
 
 def random_transit_gateway_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["transit-gateway"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["transit-gateway"])
 
 
 def random_transit_gateway_route_table_id() -> str:
-    return random_id(
-        prefix=EC2_RESOURCE_TO_PREFIX["transit-gateway-route-table"], size=17
-    )
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["transit-gateway-route-table"])
 
 
 def random_transit_gateway_attachment_id() -> str:
-    return random_id(
-        prefix=EC2_RESOURCE_TO_PREFIX["transit-gateway-attachment"], size=17
-    )
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["transit-gateway-attachment"])
+
+
+def random_managed_prefix_list_id() -> str:
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["managed-prefix-list"])
 
 
 def random_launch_template_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["launch-template"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["launch-template"])
 
 
 def random_launch_template_name() -> str:
@@ -259,7 +257,7 @@ def random_iam_instance_profile_association_id() -> str:
 
 
 def random_carrier_gateway_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["carrier-gateway"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["carrier-gateway"])
 
 
 def random_public_ip() -> str:
@@ -267,7 +265,7 @@ def random_public_ip() -> str:
 
 
 def random_dedicated_host_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["dedicated_host"], size=17)
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["dedicated_host"])
 
 
 def random_private_ip(cidr: Optional[str] = None, ipv6: bool = False) -> str:
@@ -319,10 +317,6 @@ def generate_route_id(
     return f"{route_table_id}~{cidr_block}"
 
 
-def random_managed_prefix_list_id() -> str:
-    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["managed-prefix-list"], size=8)
-
-
 def create_dns_entries(service_name: str, vpc_endpoint_id: str) -> Dict[str, str]:
     return {
         "dns_name": f"{vpc_endpoint_id}-{random_resource_id(8)}.{service_name}",
diff --git a/moto/iotdata/models.py b/moto/iotdata/models.py
index b8f3e78fa..eecf3e37e 100644
--- a/moto/iotdata/models.py
+++ b/moto/iotdata/models.py
@@ -71,20 +71,58 @@ class FakeShadow(BaseModel):
         return FakeShadow(desired, reported, payload, version)
 
     @classmethod
-    def parse_payload(cls, desired: Optional[str], reported: Optional[str]) -> Any:  # type: ignore[misc]
+    def parse_payload(cls, desired: Any, reported: Any) -> Any:  # type: ignore[misc]
         if not desired and not reported:
             delta = None
         elif reported is None and desired:
             delta = desired
         elif desired and reported:
-            delta = jsondiff.diff(reported, desired)
-            delta.pop(jsondiff.add, None)  # type: ignore
-            delta.pop(jsondiff.delete, None)  # type: ignore
-            delta.pop(jsondiff.replace, None)  # type: ignore
+            delta = jsondiff.diff(reported, desired, syntax="symmetric")
+            delta.pop(jsondiff.add, None)
+            delta.pop(jsondiff.delete, None)
+            delta.pop(jsondiff.replace, None)
+            cls._resolve_nested_deltas(desired, reported, delta)
         else:
             delta = None
         return delta
 
+    @classmethod
+    def _resolve_nested_deltas(cls, desired: Any, reported: Any, delta: Any) -> None:  # type: ignore[misc]
+        for key, value in delta.items():
+            if isinstance(value, dict):
+                # delta = {insert: [(0, val1),, ], delete: [(0, val2),..] }
+                if isinstance(reported.get(key), list):
+                    # Delta is a dict, but were supposed to have a list
+                    # Explicitly insert/delete from the original list
+                    list_delta = reported[key].copy()
+                    if jsondiff.delete in value:
+                        for idx, _ in sorted(value[jsondiff.delete], reverse=True):
+                            del list_delta[idx]
+                    if jsondiff.insert in value:
+                        for new_val in sorted(value[jsondiff.insert], reverse=True):
+                            list_delta.insert(*new_val)
+                    delta[key] = list_delta
+                if isinstance(reported.get(key), dict):
+                    # Delta is a dict, exactly what we're expecting
+                    # Just delete any unknown symbols
+                    value.pop(jsondiff.add, None)
+                    value.pop(jsondiff.delete, None)
+                    value.pop(jsondiff.replace, None)
+            elif isinstance(value, list):
+                # delta = [v1, v2]
+                # If the actual value is a string/bool/int and our delta is a list,
+                # that means that the delta reports both values - reported and desired
+                # But we only want to show the desired value here
+                # (Note that bool is a type of int, so we don't have to explicitly mention it)
+                if isinstance(desired.get(key), (str, int, float)):
+                    delta[key] = desired[key]
+
+            # Resolve nested deltas
+            if isinstance(delta[key], dict):
+                cls._resolve_nested_deltas(
+                    desired.get(key), reported.get(key), delta[key]
+                )
+
     def _create_metadata_from_state(self, state: Any, ts: Any) -> Any:
         """
         state must be desired or reported stype dict object


Test patch:
diff --git a/tests/test_iotdata/test_iotdata.py b/tests/test_iotdata/test_iotdata.py
index b2185859e..234c2cc36 100644
--- a/tests/test_iotdata/test_iotdata.py
+++ b/tests/test_iotdata/test_iotdata.py
@@ -223,6 +223,61 @@ def test_delete_field_from_device_shadow(name: Optional[str] = None) -> None:
             {"reported": {"online": False}},
             {"reported": {"online": False}},
         ),
+        # Remove from list
+        (
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"desired": {"list": ["value_1"]}},
+            {
+                "desired": {"list": ["value_1"]},
+                "reported": {"list": ["value_1", "value_2"]},
+                "delta": {"list": ["value_1"]},
+            },
+        ),
+        # Remove And Update from list
+        (
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"desired": {"list": ["value_1", "value_3"]}},
+            {
+                "desired": {"list": ["value_1", "value_3"]},
+                "reported": {"list": ["value_1", "value_2"]},
+                "delta": {"list": ["value_1", "value_3"]},
+            },
+        ),
+        # Remove from nested lists
+        (
+            {"reported": {"list": [["value_1"], ["value_2"]]}},
+            {"reported": {"list": [["value_1"], ["value_2"]]}},
+            {"desired": {"list": [["value_1"]]}},
+            {
+                "desired": {"list": [["value_1"]]},
+                "reported": {"list": [["value_1"], ["value_2"]]},
+                "delta": {"list": [["value_1"]]},
+            },
+        ),
+        # Append to nested list
+        (
+            {"reported": {"a": {"b": ["d"]}}},
+            {"reported": {"a": {"b": ["d"]}}},
+            {"desired": {"a": {"b": ["c", "d"]}}},
+            {
+                "delta": {"a": {"b": ["c", "d"]}},
+                "desired": {"a": {"b": ["c", "d"]}},
+                "reported": {"a": {"b": ["d"]}},
+            },
+        ),
+        # Update nested dict
+        (
+            {"reported": {"a": {"b": {"c": "d", "e": "f"}}}},
+            {"reported": {"a": {"b": {"c": "d", "e": "f"}}}},
+            {"desired": {"a": {"b": {"c": "d2"}}}},
+            {
+                "delta": {"a": {"b": {"c": "d2"}}},
+                "desired": {"a": {"b": {"c": "d2"}}},
+                "reported": {"a": {"b": {"c": "d", "e": "f"}}},
+            },
+        ),
     ],
 )
 def test_delta_calculation(
diff --git a/tests/test_moto_api/seeder/test_seeder.py b/tests/test_moto_api/seeder/test_seeder.py
index 970c30524..e85ae78b3 100644
--- a/tests/test_moto_api/seeder/test_seeder.py
+++ b/tests/test_moto_api/seeder/test_seeder.py
@@ -41,7 +41,7 @@ class TestDifferentAccountsDoesNotBreakSeeding:
         instances = self.ec2_client.run_instances(MaxCount=1, MinCount=1)["Instances"]
 
         instance_ids = [instance["InstanceId"] for instance in instances]
-        assert instance_ids == ["i-73bd4755d05ad7853"]
+        assert instance_ids == ["i-6e8b5359309cc6273"]
 
     def test_1(self) -> None:
         # Create some data in a different account (111111111111)