Repository: getmoto/moto
Issue category: Bug Report

Issue:
IoT Data Shadow Delta with complex datastructure exposes jsondiff symbol and raises TypeError
I'm using Moto 5.0.21, and jsondiff 2.2.1.

When deleting items from a second-level list in an IoT data shadow, there's a `jsondiff.deleted` symbol exposed, and breaking the subsequent JSON parsing. I suppose this is because the code only deletes those symbols from the first level of a dict: [iotdata/models.py#L81](https://github.com/bblommers/moto/blob/4f521ba49182be973b31f52791a50bdcd6802c02/moto/iotdata/models.py#L81)

Pytest showing the issue:

```Python
from moto import mock_aws
import boto3


@mock_aws
def test_list():
    iot_client = boto3.client(service_name="iot", region_name="eu-central-1")
    iot_client.create_thing(thingName="example-thing")

    iot_data_client = boto3.client(service_name="iot-data", region_name="eu-central-1")

    iot_data_client.update_thing_shadow(
        thingName="example-thing",
        payload=json.dumps(
            {
                "state": {
                    "reported": {
                        "list": [
                            ["value_1"],
                            ["value_2"],
                        ]
                    }
                }
            }
        ),
    )
    iot_data_client.update_thing_shadow(
        thingName="example-thing",
        payload=json.dumps(
            {
                "state": {
                    "desired": {
                        "list": [
                            ["value_1"],
                        ]
                    }
                }
            }
        ),
    )

    iot_data_client.get_thing_shadow(thingName="example-thing")
    # throws TypeError: keys must be str, int, float, bool or None, not Symbol
```

Traceback:
```Python
venv/python-3.11.9/lib/python3.11/site-packages/botocore/client.py:565: in _api_call                                                     
    return self._make_api_call(operation_name, kwargs)                                                                                   
venv/python-3.11.9/lib/python3.11/site-packages/botocore/client.py:1001: in _make_api_call                                               
    http, parsed_response = self._make_request(                                                                                          
venv/python-3.11.9/lib/python3.11/site-packages/botocore/client.py:1027: in _make_request                                                
    return self._endpoint.make_request(operation_model, request_dict)                                                                    
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:119: in make_request                                                
    return self._send_request(request_dict, operation_model)                                                                             
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:202: in _send_request                                               
    while self._needs_retry(                                                                                                             
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:354: in _needs_retry                                                
    responses = self._event_emitter.emit(                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:412: in emit                                                           
    return self._emitter.emit(aliased_event_name, **kwargs)                                                                              
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:256: in emit                                                           
    return self._emit(event_name, kwargs)                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:239: in _emit                                                          
    response = handler(**kwargs)                                                                                                         
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:207: in __call__                                                
    if self._checker(**checker_kwargs):                                                                                                  
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:284: in __call__                                                
    should_retry = self._should_retry(                                                                                                   
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:307: in _should_retry                                           
    return self._checker(                                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:363: in __call__                                                
    checker_response = checker(                                                                                                          
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:247: in __call__                                                
    return self._check_caught_exception(                                                                                                 
venv/python-3.11.9/lib/python3.11/site-packages/botocore/retryhandler.py:416: in _check_caught_exception                                 
    raise caught_exception                                                                                                               
venv/python-3.11.9/lib/python3.11/site-packages/botocore/endpoint.py:278: in _do_get_response                                            
    responses = self._event_emitter.emit(event_name, request=request)                                                                    
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:412: in emit                                                           
    return self._emitter.emit(aliased_event_name, **kwargs)                                                                              
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:256: in emit                                                           
    return self._emit(event_name, kwargs)                                                                                                
venv/python-3.11.9/lib/python3.11/site-packages/botocore/hooks.py:239: in _emit                                                          
    response = handler(**kwargs)                                                                                                         
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/botocore_stubber.py:38: in __call__                                            
    response = self.process_request(request)                                                                                             
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/botocore_stubber.py:88: in process_request                                     
    status, headers, body = method_to_execute(                                                                                           
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/responses.py:291: in dispatch                                                  
    return cls()._dispatch(*args, **kwargs)                                                                                              
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/responses.py:503: in _dispatch                                                 
    return self.call_action()                                                                                                            
venv/python-3.11.9/lib/python3.11/site-packages/moto/core/responses.py:591: in call_action
    response = method()
venv/python-3.11.9/lib/python3.11/site-packages/moto/iotdata/responses.py:45: in get_thing_shadow
    return json.dumps(payload.to_dict())
/usr/local/lib/python3.11/json/__init__.py:231: in dumps
    return _default_encoder.encode(obj)
/usr/local/lib/python3.11/json/encoder.py:200: in encode
    chunks = self.iterencode(o, _one_shot=True)
```

Also, I'm not sure if AWS would even detect such an item deletion, or if it would just see the list as completely new. I don't have the resources accessible right now to test that.


Hints:

Patch:
diff --git a/moto/iotdata/models.py b/moto/iotdata/models.py
index b8f3e78faac3..eecf3e37e3b4 100644
--- a/moto/iotdata/models.py
+++ b/moto/iotdata/models.py
@@ -71,20 +71,58 @@ def create_from_previous_version(  # type: ignore[misc]
         return FakeShadow(desired, reported, payload, version)
 
     @classmethod
-    def parse_payload(cls, desired: Optional[str], reported: Optional[str]) -> Any:  # type: ignore[misc]
+    def parse_payload(cls, desired: Any, reported: Any) -> Any:  # type: ignore[misc]
         if not desired and not reported:
             delta = None
         elif reported is None and desired:
             delta = desired
         elif desired and reported:
-            delta = jsondiff.diff(reported, desired)
-            delta.pop(jsondiff.add, None)  # type: ignore
-            delta.pop(jsondiff.delete, None)  # type: ignore
-            delta.pop(jsondiff.replace, None)  # type: ignore
+            delta = jsondiff.diff(reported, desired, syntax="symmetric")
+            delta.pop(jsondiff.add, None)
+            delta.pop(jsondiff.delete, None)
+            delta.pop(jsondiff.replace, None)
+            cls._resolve_nested_deltas(desired, reported, delta)
         else:
             delta = None
         return delta
 
+    @classmethod
+    def _resolve_nested_deltas(cls, desired: Any, reported: Any, delta: Any) -> None:  # type: ignore[misc]
+        for key, value in delta.items():
+            if isinstance(value, dict):
+                # delta = {insert: [(0, val1),, ], delete: [(0, val2),..] }
+                if isinstance(reported.get(key), list):
+                    # Delta is a dict, but were supposed to have a list
+                    # Explicitly insert/delete from the original list
+                    list_delta = reported[key].copy()
+                    if jsondiff.delete in value:
+                        for idx, _ in sorted(value[jsondiff.delete], reverse=True):
+                            del list_delta[idx]
+                    if jsondiff.insert in value:
+                        for new_val in sorted(value[jsondiff.insert], reverse=True):
+                            list_delta.insert(*new_val)
+                    delta[key] = list_delta
+                if isinstance(reported.get(key), dict):
+                    # Delta is a dict, exactly what we're expecting
+                    # Just delete any unknown symbols
+                    value.pop(jsondiff.add, None)
+                    value.pop(jsondiff.delete, None)
+                    value.pop(jsondiff.replace, None)
+            elif isinstance(value, list):
+                # delta = [v1, v2]
+                # If the actual value is a string/bool/int and our delta is a list,
+                # that means that the delta reports both values - reported and desired
+                # But we only want to show the desired value here
+                # (Note that bool is a type of int, so we don't have to explicitly mention it)
+                if isinstance(desired.get(key), (str, int, float)):
+                    delta[key] = desired[key]
+
+            # Resolve nested deltas
+            if isinstance(delta[key], dict):
+                cls._resolve_nested_deltas(
+                    desired.get(key), reported.get(key), delta[key]
+                )
+
     def _create_metadata_from_state(self, state: Any, ts: Any) -> Any:
         """
         state must be desired or reported stype dict object


Test patch:
diff --git a/tests/test_iotdata/test_iotdata.py b/tests/test_iotdata/test_iotdata.py
index b2185859ee42..234c2cc361cc 100644
--- a/tests/test_iotdata/test_iotdata.py
+++ b/tests/test_iotdata/test_iotdata.py
@@ -223,6 +223,61 @@ def test_delete_field_from_device_shadow(name: Optional[str] = None) -> None:
             {"reported": {"online": False}},
             {"reported": {"online": False}},
         ),
+        # Remove from list
+        (
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"desired": {"list": ["value_1"]}},
+            {
+                "desired": {"list": ["value_1"]},
+                "reported": {"list": ["value_1", "value_2"]},
+                "delta": {"list": ["value_1"]},
+            },
+        ),
+        # Remove And Update from list
+        (
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"reported": {"list": ["value_1", "value_2"]}},
+            {"desired": {"list": ["value_1", "value_3"]}},
+            {
+                "desired": {"list": ["value_1", "value_3"]},
+                "reported": {"list": ["value_1", "value_2"]},
+                "delta": {"list": ["value_1", "value_3"]},
+            },
+        ),
+        # Remove from nested lists
+        (
+            {"reported": {"list": [["value_1"], ["value_2"]]}},
+            {"reported": {"list": [["value_1"], ["value_2"]]}},
+            {"desired": {"list": [["value_1"]]}},
+            {
+                "desired": {"list": [["value_1"]]},
+                "reported": {"list": [["value_1"], ["value_2"]]},
+                "delta": {"list": [["value_1"]]},
+            },
+        ),
+        # Append to nested list
+        (
+            {"reported": {"a": {"b": ["d"]}}},
+            {"reported": {"a": {"b": ["d"]}}},
+            {"desired": {"a": {"b": ["c", "d"]}}},
+            {
+                "delta": {"a": {"b": ["c", "d"]}},
+                "desired": {"a": {"b": ["c", "d"]}},
+                "reported": {"a": {"b": ["d"]}},
+            },
+        ),
+        # Update nested dict
+        (
+            {"reported": {"a": {"b": {"c": "d", "e": "f"}}}},
+            {"reported": {"a": {"b": {"c": "d", "e": "f"}}}},
+            {"desired": {"a": {"b": {"c": "d2"}}}},
+            {
+                "delta": {"a": {"b": {"c": "d2"}}},
+                "desired": {"a": {"b": {"c": "d2"}}},
+                "reported": {"a": {"b": {"c": "d", "e": "f"}}},
+            },
+        ),
     ],
 )
 def test_delta_calculation(