Repository: getmoto/moto

Issue:
iot_data: `None` as value is not deleting key but is saved as `key: None`
When deleting keys in the shadow state with `key: None` moto is saving the None as value instead of deleting the key.

This test succeeds if running agains AWS but is failing with Moto (second last assert):
(moto v2.0.1 via pip on python3.8)

```
from boto3 import client
from moto import mock_iot, mock_iotdata
from json import loads, dumps


@mock_iot
@mock_iotdata
def test_delete_key():
    test_thing_name = "TestThing"

    iot_raw_client = client("iot", region_name="eu-central-1")
    iot_raw_client.create_thing(thingName=test_thing_name)
    iot = client("iot-data", region_name="eu-central-1")

    iot.update_thing_shadow(thingName=test_thing_name, payload=dumps({"state": {"desired": {"state1": 1, "state2": 2}}}))
    response = loads(iot.get_thing_shadow(thingName=test_thing_name)["payload"].read())
    assert len(response["state"]["desired"]) == 2

    iot.update_thing_shadow(thingName=test_thing_name, payload=dumps({"state": {"desired": {"state1": None}}}))
    response = loads(iot.get_thing_shadow(thingName=test_thing_name)["payload"].read())
    assert len(response["state"]["desired"]) == 1
    assert "state2" in response["state"]["desired"]

    iot.update_thing_shadow(thingName=test_thing_name, payload=dumps({"state": {"desired": {"state2": None}}}))
    response = loads(iot.get_thing_shadow(thingName=test_thing_name)["payload"].read())
    assert "desired" not in response["state"]
```


Discussion:

Resolving patch:
diff --git a/moto/core/utils.py b/moto/core/utils.py
--- a/moto/core/utils.py
+++ b/moto/core/utils.py
@@ -408,3 +408,26 @@ def remap_nested_keys(root, key_transform):
             for k, v in six.iteritems(root)
         }
     return root
+
+
+def merge_dicts(dict1, dict2, remove_nulls=False):
+    """Given two arbitrarily nested dictionaries, merge the second dict into the first.
+
+    :param dict dict1: the dictionary to be updated.
+    :param dict dict2: a dictionary of keys/values to be merged into dict1.
+
+    :param bool remove_nulls: If true, updated values equal to None or an empty dictionary
+        will be removed from dict1.
+    """
+    for key in dict2:
+        if isinstance(dict2[key], dict):
+            if key in dict1 and key in dict2:
+                merge_dicts(dict1[key], dict2[key], remove_nulls)
+            else:
+                dict1[key] = dict2[key]
+            if dict1[key] == {} and remove_nulls:
+                dict1.pop(key)
+        else:
+            dict1[key] = dict2[key]
+            if dict1[key] is None and remove_nulls:
+                dict1.pop(key)
diff --git a/moto/iotdata/models.py b/moto/iotdata/models.py
--- a/moto/iotdata/models.py
+++ b/moto/iotdata/models.py
@@ -5,6 +5,7 @@
 from boto3 import Session
 
 from moto.core import BaseBackend, BaseModel
+from moto.core.utils import merge_dicts
 from moto.iot import iot_backends
 from .exceptions import (
     ConflictException,
@@ -50,13 +51,12 @@ def create_from_previous_version(cls, previous_shadow, payload):
             shadow = FakeShadow(None, None, None, version, deleted=True)
             return shadow
 
-        # we can make sure that payload has 'state' key
-        desired = payload["state"].get(
-            "desired", previous_payload.get("state", {}).get("desired", None)
-        )
-        reported = payload["state"].get(
-            "reported", previous_payload.get("state", {}).get("reported", None)
-        )
+        # Updates affect only the fields specified in the request state document.
+        # Any field with a value of None is removed from the device's shadow.
+        state_document = previous_payload.copy()
+        merge_dicts(state_document, payload, remove_nulls=True)
+        desired = state_document.get("state", {}).get("desired")
+        reported = state_document.get("state", {}).get("reported")
         shadow = FakeShadow(desired, reported, payload, version)
         return shadow
 


Test patch:
diff --git a/tests/test_iotdata/test_iotdata.py b/tests/test_iotdata/test_iotdata.py
--- a/tests/test_iotdata/test_iotdata.py
+++ b/tests/test_iotdata/test_iotdata.py
@@ -109,3 +109,41 @@ def test_update():
 def test_publish():
     client = boto3.client("iot-data", region_name="ap-northeast-1")
     client.publish(topic="test/topic", qos=1, payload=b"")
+
+
+@mock_iot
+@mock_iotdata
+def test_delete_field_from_device_shadow():
+    test_thing_name = "TestThing"
+
+    iot_raw_client = boto3.client("iot", region_name="eu-central-1")
+    iot_raw_client.create_thing(thingName=test_thing_name)
+    iot = boto3.client("iot-data", region_name="eu-central-1")
+
+    iot.update_thing_shadow(
+        thingName=test_thing_name,
+        payload=json.dumps({"state": {"desired": {"state1": 1, "state2": 2}}}),
+    )
+    response = json.loads(
+        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
+    )
+    assert len(response["state"]["desired"]) == 2
+
+    iot.update_thing_shadow(
+        thingName=test_thing_name,
+        payload=json.dumps({"state": {"desired": {"state1": None}}}),
+    )
+    response = json.loads(
+        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
+    )
+    assert len(response["state"]["desired"]) == 1
+    assert "state2" in response["state"]["desired"]
+
+    iot.update_thing_shadow(
+        thingName=test_thing_name,
+        payload=json.dumps({"state": {"desired": {"state2": None}}}),
+    )
+    response = json.loads(
+        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
+    )
+    assert "desired" not in response["state"]