Repository: getmoto/moto

Issue:
DynamoDB: Combining `list_append` and `if_not_exists` does not work
### Description

Our product sets a nested attribute to a combination of the two functions above to create or append to a history log in DynamoDB.

Expected: the list is created at the specified key if it does not exist with the new item in it.
Actual: the client crashes with a `KeyError`.

This is reproducible with the following query:

``` python
from moto import mock_dynamodb2
import boto3

with mock_dynamodb2():
    dynamo = boto3.resource("dynamodb")

    dynamo.create_table(
        TableName="test",
        AttributeDefinitions=[{"AttributeName": "Id", "AttributeType": "S"}],
        KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}],
        ProvisionedThroughput={"ReadCapacityUnits": 20, "WriteCapacityUnits": 20},
    )

    table = dynamo.Table("test")

    table.put_item(Item={"Id": "item-id", "Nest1": {"nest2": {}}})
    table.update_item(
        Key={"Id": "item-id"},
        UpdateExpression="SET Nest1.nest2.event_history = list_append(if_not_exists(Nest1.nest2.event_history, :empty_list), :new_value)",
        ExpressionAttributeValues={":empty_list": [], ":new_value": "some_value"},
    )
```

Stack trace:

```
Traceback (most recent call last):
  File "moto_list_append_bug.py", line 20, in <module>
    ExpressionAttributeValues={":empty_list": [], ":new_value": "some_value"},
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/boto3/resources/factory.py", line 520, in do_action
    response = action(self, *args, **kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/boto3/resources/action.py", line 83, in __call__
    response = getattr(parent.meta.client, operation_name)(**params)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/client.py", line 357, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/client.py", line 648, in _make_api_call
    operation_model, request_dict, request_context)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/client.py", line 667, in _make_request
    return self._endpoint.make_request(operation_model, request_dict)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/endpoint.py", line 102, in make_request
    return self._send_request(request_dict, operation_model)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/endpoint.py", line 137, in _send_request
    success_response, exception):
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/endpoint.py", line 231, in _needs_retry
    caught_exception=caught_exception, request_dict=request_dict)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/hooks.py", line 356, in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/hooks.py", line 228, in emit
    return self._emit(event_name, kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/hooks.py", line 211, in _emit
    response = handler(**kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/retryhandler.py", line 183, in __call__
    if self._checker(attempts, response, caught_exception):
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/retryhandler.py", line 251, in __call__
    caught_exception)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/retryhandler.py", line 269, in _should_retry
    return self._checker(attempt_number, response, caught_exception)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/retryhandler.py", line 317, in __call__
    caught_exception)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/retryhandler.py", line 223, in __call__
    attempt_number, caught_exception)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
    raise caught_exception
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/endpoint.py", line 197, in _do_get_response
    responses = self._event_emitter.emit(event_name, request=request)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/hooks.py", line 356, in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/hooks.py", line 228, in emit
    return self._emit(event_name, kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/botocore/hooks.py", line 211, in _emit
    response = handler(**kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/core/models.py", line 272, in __call__
    request, request.url, request.headers
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/core/responses.py", line 197, in dispatch
    return cls()._dispatch(*args, **kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/core/responses.py", line 295, in _dispatch
    return self.call_action()
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/core/utils.py", line 267, in _wrapper
    response = f(*args, **kwargs)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/dynamodb2/responses.py", line 75, in call_action
    response = getattr(self, endpoint)()
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/dynamodb2/responses.py", line 745, in update_item
    condition_expression,
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/dynamodb2/models.py", line 1394, in update_item
    expression_attribute_values,
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/dynamodb2/models.py", line 338, in update
    value = self._get_appended_list(value, expression_attribute_values)
  File "/Users/jcasavant/tmp/.venv/lib/python3.7/site-packages/moto/dynamodb2/models.py", line 420, in _get_appended_list
    old_list = self.attrs[list_append_re.group(1)]
KeyError: 'if_not_exists(Nest1.nest2.event_history, :empty_list)'
```

### System info

Moto 1.3.14 installed via `pip` in a `pyvenv` virtual environment.

`python --version`:
`Python 3.7.5`

`uname -a`:
`Darwin jcasavant-MacBook-Pro 19.0.0 Darwin Kernel Version 19.0.0: Thu Oct 17 16:17:15 PDT 2019; root:xnu-6153.41.3~29/RELEASE_X86_64 x86_64`

`pip freeze`:
```
attrs==19.3.0
aws-sam-translator==1.15.1
aws-xray-sdk==2.4.2
boto==2.49.0
boto3==1.10.19
botocore==1.13.19
certifi==2019.9.11
cffi==1.13.2
cfn-lint==0.25.1
chardet==3.0.4
cryptography==2.8
docker==4.1.0
docutils==0.15.2
ecdsa==0.14.1
future==0.18.2
idna==2.8
importlib-metadata==0.23
Jinja2==2.10.3
jmespath==0.9.4
jsondiff==1.1.2
jsonpatch==1.24
jsonpickle==1.2
jsonpointer==2.0
jsonschema==3.2.0
MarkupSafe==1.1.1
mock==3.0.5
more-itertools==7.2.0
moto==1.3.14
pyasn1==0.4.8
pycparser==2.19
pyrsistent==0.15.5
python-dateutil==2.8.1
python-jose==3.0.1
pytz==2019.3
PyYAML==5.1.2
requests==2.22.0
responses==0.10.6
rsa==4.0
s3transfer==0.2.1
six==1.13.0
sshpubkeys==3.1.0
urllib3==1.25.7
websocket-client==0.56.0
Werkzeug==0.16.0
wrapt==1.11.2
xmltodict==0.12.0
zipp==0.6.0
```


Discussion:
Same issue with the query:
`SET teamFines = list_append(if_not_exists(teamFines, :emptyList), :fine)` on 1.3.14
Same issue even without `if_not_exists`:

```python
update_expr = 'SET BestConfigs = list_append(BestConfigs, :config)'
update_expr_vals = {':config': {'L': [...]}}
```

My case is that the appended item is not stored as `DynamoType` and it results in the following error when querying. I printed out all `v` in `self.value` and found that the first element (the one committed with `put_item`) is `DynamoType`, but the second element (the one committed with `update_item`) is the Python native `dict`.
 
```
  File "/usr/local/lib/python3.6/dist-packages/botocore/client.py", line 276, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.6/dist-packages/botocore/client.py", line 573, in _make_api_call
    operation_model, request_dict, request_context)
  File "/usr/local/lib/python3.6/dist-packages/botocore/client.py", line 592, in _make_request
    return self._endpoint.make_request(operation_model, request_dict)
  File "/usr/local/lib/python3.6/dist-packages/botocore/endpoint.py", line 102, in make_request
    return self._send_request(request_dict, operation_model)
  File "/usr/local/lib/python3.6/dist-packages/botocore/endpoint.py", line 137, in _send_request
    success_response, exception):
  File "/usr/local/lib/python3.6/dist-packages/botocore/endpoint.py", line 231, in _needs_retry
    caught_exception=caught_exception, request_dict=request_dict)
  File "/usr/local/lib/python3.6/dist-packages/botocore/hooks.py", line 356, in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/botocore/hooks.py", line 228, in emit
    return self._emit(event_name, kwargs)
  File "/usr/local/lib/python3.6/dist-packages/botocore/hooks.py", line 211, in _emit
    response = handler(**kwargs)
  File "/usr/local/lib/python3.6/dist-packages/botocore/retryhandler.py", line 183, in __call__
    if self._checker(attempts, response, caught_exception):
  File "/usr/local/lib/python3.6/dist-packages/botocore/retryhandler.py", line 251, in __call__
    caught_exception)
  File "/usr/local/lib/python3.6/dist-packages/botocore/retryhandler.py", line 269, in _should_retry
    return self._checker(attempt_number, response, caught_exception)
  File "/usr/local/lib/python3.6/dist-packages/botocore/retryhandler.py", line 317, in __call__
    caught_exception)
  File "/usr/local/lib/python3.6/dist-packages/botocore/retryhandler.py", line 223, in __call__
    attempt_number, caught_exception)
  File "/usr/local/lib/python3.6/dist-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
    raise caught_exception
  File "/usr/local/lib/python3.6/dist-packages/botocore/endpoint.py", line 197, in _do_get_response
    responses = self._event_emitter.emit(event_name, request=request)
  File "/usr/local/lib/python3.6/dist-packages/botocore/hooks.py", line 356, in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/botocore/hooks.py", line 228, in emit
    return self._emit(event_name, kwargs)
  File "/usr/local/lib/python3.6/dist-packages/botocore/hooks.py", line 211, in _emit
    response = handler(**kwargs)
  File "/usr/local/lib/python3.6/dist-packages/moto/core/models.py", line 272, in __call__
    request, request.url, request.headers
  File "/usr/local/lib/python3.6/dist-packages/moto/core/responses.py", line 197, in dispatch
    return cls()._dispatch(*args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/moto/core/responses.py", line 295, in _dispatch
    return self.call_action()
  File "/usr/local/lib/python3.6/dist-packages/moto/core/utils.py", line 267, in _wrapper
    response = f(*args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/responses.py", line 75, in call_action
    response = getattr(self, endpoint)()
  File "/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/responses.py", line 554, in query
    **filter_kwargs
  File "/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py", line 1295, in query
    **filter_kwargs
  File "/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py", line 953, in query
    results = copy.deepcopy(results)
  File "/usr/lib/python3.6/copy.py", line 150, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.6/copy.py", line 215, in _deepcopy_list
    append(deepcopy(a, memo))
  File "/usr/lib/python3.6/copy.py", line 180, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/usr/lib/python3.6/copy.py", line 280, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python3.6/copy.py", line 150, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.6/copy.py", line 240, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python3.6/copy.py", line 180, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/usr/lib/python3.6/copy.py", line 307, in _reconstruct
    y[key] = value
  File "/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py", line 247, in __setitem__
    value.size() if type(value) == DynamoType else bytesize(str(value))
  File "/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py", line 195, in size
    value_size = sum([v.size() for v in self.value])
  File "/usr/local/lib/python3.6/dist-packages/moto/dynamodb2/models.py", line 195, in <listcomp>
    value_size = sum([v.size() for v in self.value])
AttributeError: 'dict' object has no attribute 'size'
```

Resolving patch:
diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py
--- a/moto/dynamodb2/models.py
+++ b/moto/dynamodb2/models.py
@@ -448,13 +448,18 @@ def _get_appended_list(self, value, expression_attribute_values):
             if list_append_re:
                 new_value = expression_attribute_values[list_append_re.group(2).strip()]
                 old_list_key = list_append_re.group(1)
-                # Get the existing value
-                old_list = self.attrs[old_list_key.split(".")[0]]
-                if "." in old_list_key:
-                    # Value is nested inside a map - find the appropriate child attr
-                    old_list = old_list.child_attr(
-                        ".".join(old_list_key.split(".")[1:])
+                # old_key could be a function itself (if_not_exists)
+                if old_list_key.startswith("if_not_exists"):
+                    old_list = DynamoType(
+                        expression_attribute_values[self._get_default(old_list_key)]
                     )
+                else:
+                    old_list = self.attrs[old_list_key.split(".")[0]]
+                    if "." in old_list_key:
+                        # Value is nested inside a map - find the appropriate child attr
+                        old_list = old_list.child_attr(
+                            ".".join(old_list_key.split(".")[1:])
+                        )
                 if not old_list.is_list():
                     raise ParamValidationError
                 old_list.value.extend([DynamoType(v) for v in new_value["L"]])


Test patch:
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -3609,6 +3609,31 @@ def test_update_supports_list_append_maps():
     )
 
 
+@mock_dynamodb2
+def test_update_supports_list_append_with_nested_if_not_exists_operation():
+    dynamo = boto3.resource("dynamodb", region_name="us-west-1")
+    table_name = "test"
+
+    dynamo.create_table(
+        TableName=table_name,
+        AttributeDefinitions=[{"AttributeName": "Id", "AttributeType": "S"}],
+        KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 20, "WriteCapacityUnits": 20},
+    )
+
+    table = dynamo.Table(table_name)
+
+    table.put_item(Item={"Id": "item-id", "nest1": {"nest2": {}}})
+    table.update_item(
+        Key={"Id": "item-id"},
+        UpdateExpression="SET nest1.nest2.event_history = list_append(if_not_exists(nest1.nest2.event_history, :empty_list), :new_value)",
+        ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]},
+    )
+    table.get_item(Key={"Id": "item-id"})["Item"].should.equal(
+        {"Id": "item-id", "nest1": {"nest2": {"event_history": ["some_value"]}}}
+    )
+
+
 @mock_dynamodb2
 def test_update_catches_invalid_list_append_operation():
     client = boto3.client("dynamodb", region_name="us-east-1")