Repository: getmoto/moto
Issue: [Bug][dynamodb2] list_append doesn't construct DynamoType objects

The `list_append` support in `dynamodb2` does not construct `DynamoType` for new appended values. It results in type inconsistency and runtime errors. Details are as follows:

* Reproduce code:

```python
import boto3
import moto

@moto.mock_dynamodb2
def test_dynamodb2():
    client = boto3.client('dynamodb', region_name='us-west-1')
    client.create_table(
        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}, {"AttributeName": "rid", "AttributeType": "S"}],
        TableName="TestTable",
        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}, {"AttributeName": "rid", "KeyType": "RANGE"}],
        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
    )
    client.put_item(
        TableName="TestTable",
        Item={
            "id": {"S": "nested_list_append"},
            "rid": {"S": "range_key"},
            "a": {"L": [{"M": {"b": {"S": "bar1"}}}]},
        },
    )

    # Update item using list_append expression
    client.update_item(
        TableName="TestTable",
        Key={"id": {"S": "nested_list_append"}, "rid": {"S": "range_key"}},
        UpdateExpression="SET a = list_append(a, :i)",
        ExpressionAttributeValues={":i": {"L": [{"M": {"b": {"S": "bar2"}}}]}}
    )

    # Verify item is appended to the existing list
    result = client.query(
        TableName="TestTable",
        KeyConditionExpression="id = :i AND begins_with(rid, :r)",
        ExpressionAttributeValues={":i": {"S": "nested_list_append"}, ":r": {"S": "range_key"}}
    )["Items"]
```

* Error message
```
        # Verify item is appended to the existing list
        result = client.query(
            TableName="TestTable",
            KeyConditionExpression="id = :i AND begins_with(rid, :r)",
>           ExpressionAttributeValues={":i": {"S": "nested_list_append"}, ":r": {"S": "range_key"}}
        )["Items"]

moto_bug.py:34: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../.local/lib/python3.6/site-packages/botocore/client.py:276: in _api_call
    return self._make_api_call(operation_name, kwargs)
../.local/lib/python3.6/site-packages/botocore/client.py:573: in _make_api_call
    operation_model, request_dict, request_context)
../.local/lib/python3.6/site-packages/botocore/client.py:592: in _make_request
    return self._endpoint.make_request(operation_model, request_dict)
../.local/lib/python3.6/site-packages/botocore/endpoint.py:102: in make_request
    return self._send_request(request_dict, operation_model)
../.local/lib/python3.6/site-packages/botocore/endpoint.py:137: in _send_request
    success_response, exception):
../.local/lib/python3.6/site-packages/botocore/endpoint.py:231: in _needs_retry
    caught_exception=caught_exception, request_dict=request_dict)
../.local/lib/python3.6/site-packages/botocore/hooks.py:356: in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
../.local/lib/python3.6/site-packages/botocore/hooks.py:228: in emit
    return self._emit(event_name, kwargs)
../.local/lib/python3.6/site-packages/botocore/hooks.py:211: in _emit
    response = handler(**kwargs)
../.local/lib/python3.6/site-packages/botocore/retryhandler.py:183: in __call__
    if self._checker(attempts, response, caught_exception):
../.local/lib/python3.6/site-packages/botocore/retryhandler.py:251: in __call__
    caught_exception)
../.local/lib/python3.6/site-packages/botocore/retryhandler.py:269: in _should_retry
    return self._checker(attempt_number, response, caught_exception)
../.local/lib/python3.6/site-packages/botocore/retryhandler.py:317: in __call__
    caught_exception)
../.local/lib/python3.6/site-packages/botocore/retryhandler.py:223: in __call__
    attempt_number, caught_exception)
../.local/lib/python3.6/site-packages/botocore/retryhandler.py:359: in _check_caught_exception
    raise caught_exception
../.local/lib/python3.6/site-packages/botocore/endpoint.py:197: in _do_get_response
    responses = self._event_emitter.emit(event_name, request=request)
../.local/lib/python3.6/site-packages/botocore/hooks.py:356: in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
../.local/lib/python3.6/site-packages/botocore/hooks.py:228: in emit
    return self._emit(event_name, kwargs)
../.local/lib/python3.6/site-packages/botocore/hooks.py:211: in _emit
    response = handler(**kwargs)
/usr/local/lib/python3.6/dist-packages/moto/core/models.py:278: in __call__
    request, request.url, request.headers
/usr/local/lib/python3.6/dist-packages/moto/core/responses.py:197: in dispatch
    return cls()._dispatch(*args, **kwargs)
/usr/local/lib/python3.6/dist-packages/moto/core/responses.py:295: in _dispatch
    return self.call_action()
/usr/local/lib/python3.6/dist-packages/moto/core/utils.py:271: in _wrapper
    response = f(*args, **kwargs)
/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/responses.py:75: in call_action
    response = getattr(self, endpoint)()
/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/responses.py:559: in query
    **filter_kwargs
/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py:1333: in query
    **filter_kwargs
/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py:991: in query
    results = copy.deepcopy(results)
/usr/lib/python3.6/copy.py:150: in deepcopy
    y = copier(x, memo)
/usr/lib/python3.6/copy.py:215: in _deepcopy_list
    append(deepcopy(a, memo))
/usr/lib/python3.6/copy.py:180: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/lib/python3.6/copy.py:280: in _reconstruct
    state = deepcopy(state, memo)
/usr/lib/python3.6/copy.py:150: in deepcopy
    y = copier(x, memo)
/usr/lib/python3.6/copy.py:240: in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
/usr/lib/python3.6/copy.py:180: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/lib/python3.6/copy.py:307: in _reconstruct
    y[key] = value
/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py:261: in __setitem__
    value.size() if type(value) == DynamoType else bytesize(str(value))
/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py:209: in size
    value_size = sum([v.size() for v in self.value])
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

.0 = <list_iterator object at 0x7f4baaede710>

>   value_size = sum([v.size() for v in self.value])
E   AttributeError: 'dict' object has no attribute 'size'

/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py:209: AttributeError
```

* Analysis

The error object was `{"M": {"b": {"S": "bar1"}}}`. It was stored to the table by the following line without constructing a DynamoType object:

`[moto/dynamodb2/models.py:L460] old_list.value.extend(new_value["L"])`

Make the following change can pass the test case:

`old_list.value.extend([DynamoType(v) for v in new_value["L"]])`

@bblommers since you filed PR #2598 , could you help verify this issue?

Thanks!


Changed files:
moto/dynamodb2/models.py, tests/test_dynamodb2/test_dynamodb.py

diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py
index d4907cba5..2313a6e41 100644
--- a/moto/dynamodb2/models.py
+++ b/moto/dynamodb2/models.py
@@ -457,7 +457,7 @@ class Item(BaseModel):
                     )
                 if not old_list.is_list():
                     raise ParamValidationError
-                old_list.value.extend(new_value["L"])
+                old_list.value.extend([DynamoType(v) for v in new_value["L"]])
                 value = old_list
         return value
 
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
index 831538054..2d961b406 100644
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -3489,6 +3489,58 @@ def test_update_supports_nested_list_append_onto_another_list():
     )
 
 
+@mock_dynamodb2
+def test_update_supports_list_append_maps():
+    client = boto3.client("dynamodb", region_name="us-west-1")
+    client.create_table(
+        AttributeDefinitions=[
+            {"AttributeName": "id", "AttributeType": "S"},
+            {"AttributeName": "rid", "AttributeType": "S"},
+        ],
+        TableName="TestTable",
+        KeySchema=[
+            {"AttributeName": "id", "KeyType": "HASH"},
+            {"AttributeName": "rid", "KeyType": "RANGE"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    client.put_item(
+        TableName="TestTable",
+        Item={
+            "id": {"S": "nested_list_append"},
+            "rid": {"S": "range_key"},
+            "a": {"L": [{"M": {"b": {"S": "bar1"}}}]},
+        },
+    )
+
+    # Update item using list_append expression
+    client.update_item(
+        TableName="TestTable",
+        Key={"id": {"S": "nested_list_append"}, "rid": {"S": "range_key"}},
+        UpdateExpression="SET a = list_append(a, :i)",
+        ExpressionAttributeValues={":i": {"L": [{"M": {"b": {"S": "bar2"}}}]}},
+    )
+
+    # Verify item is appended to the existing list
+    result = client.query(
+        TableName="TestTable",
+        KeyConditionExpression="id = :i AND begins_with(rid, :r)",
+        ExpressionAttributeValues={
+            ":i": {"S": "nested_list_append"},
+            ":r": {"S": "range_key"},
+        },
+    )["Items"]
+    result.should.equal(
+        [
+            {
+                "a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]},
+                "rid": {"S": "range_key"},
+                "id": {"S": "nested_list_append"},
+            }
+        ]
+    )
+
+
 @mock_dynamodb2
 def test_update_catches_invalid_list_append_operation():
     client = boto3.client("dynamodb", region_name="us-east-1")