Repository: getmoto/moto

Issue:
DynamoDB2 update does not work with multiple/complex expression attribute values
DynamoDB updates do not work with all possible expression attribute values.

## Code Snippet
This code snippet works exactly as written against the real DynamoDB (as long as you sleep a few seconds between table creation and item insertion). However, it breaks in moto:

```python
import boto3
import moto

@moto.mock_dynamodb2()
def update_does_not_work():
    dynamo_client = boto3.client('dynamodb')

    # Create table - works fine.
    dynamo_client.create_table(
        AttributeDefinitions=[{'AttributeName': 'SHA256', 'AttributeType': 'S'}],
        TableName='TestTable',
        KeySchema=[{'AttributeName': 'SHA256', 'KeyType': 'HASH'}],
        ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5}
    )

    # Update item - does not work. "ValueError: too many values to unpack"
    dynamo_client.update_item(
        TableName='TestTable',
        Key={'SHA256': {'S': 'sha-of-file'}},
        UpdateExpression=(
            'SET MD5 = :md5,'
            'MyStringSet = :string_set,'
            'MyMap = :map'  # Maps string to StringSet of strings.
        ),
        ExpressionAttributeValues={
            ':md5': {'S': 'md5-of-file'},
            ':string_set': {'SS': ['string1', 'string2']},
            ':map': {'M': {'EntryKey': {'SS': ['thing1', 'thing2']}}}
        }
    )
```

I suspect that there is an error in the update parsing logic. Dynamo item updates can be really difficult to wrangle.

## Stack Trace
```
Traceback (most recent call last):
  File "poc.py", line 35, in <module>
    update_does_not_work()
  File "/usr/local/lib/python2.7/site-packages/moto/core/models.py", line 71, in wrapper
    result = func(*args, **kwargs)
  File "poc.py", line 30, in update_does_not_work
    ':map': {'M': {'EntryKey': {'SS': ['thing1', 'thing2']}}}
  File "/usr/local/lib/python2.7/site-packages/botocore/client.py", line 253, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python2.7/site-packages/botocore/client.py", line 530, in _make_api_call
    operation_model, request_dict)
  File "/usr/local/lib/python2.7/site-packages/botocore/endpoint.py", line 141, in make_request
    return self._send_request(request_dict, operation_model)
  File "/usr/local/lib/python2.7/site-packages/botocore/endpoint.py", line 170, in _send_request
    success_response, exception):
  File "/usr/local/lib/python2.7/site-packages/botocore/endpoint.py", line 249, in _needs_retry
    caught_exception=caught_exception, request_dict=request_dict)
  File "/usr/local/lib/python2.7/site-packages/botocore/hooks.py", line 227, in emit
    return self._emit(event_name, kwargs)
  File "/usr/local/lib/python2.7/site-packages/botocore/hooks.py", line 210, in _emit
    response = handler(**kwargs)
  File "/usr/local/lib/python2.7/site-packages/botocore/retryhandler.py", line 183, in __call__
    if self._checker(attempts, response, caught_exception):
  File "/usr/local/lib/python2.7/site-packages/botocore/retryhandler.py", line 251, in __call__
    caught_exception)
  File "/usr/local/lib/python2.7/site-packages/botocore/retryhandler.py", line 269, in _should_retry
    return self._checker(attempt_number, response, caught_exception)
  File "/usr/local/lib/python2.7/site-packages/botocore/retryhandler.py", line 317, in __call__
    caught_exception)
  File "/usr/local/lib/python2.7/site-packages/botocore/retryhandler.py", line 223, in __call__
    attempt_number, caught_exception)
  File "/usr/local/lib/python2.7/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
    raise caught_exception
ValueError: too many values to unpack
```

## Environment
I'm running Python 2.7 on Mac OS X. Moto is installed with `pip`:
```
$ pip show moto
Version: 0.4.31
Location: /usr/local/lib/python2.7/site-packages
```


Discussion:
Same issue FWIW. Only expression i'm using is:

    UpdateExpression="SET crontab = list_append(crontab, :i)",
            ExpressionAttributeValues={
                ':i': crontab,
            },
I had quite an awkward one: 

```
table.update_item(
          Key={'entity': 'cycle', 'id': 'cycle-id-1'},
          UpdateExpression="SET nominations = :nominationIds",
          ExpressionAttributeValues={':nominationIds': nominationIds},
          ReturnValues="UPDATED_NEW"
        )
```
... and I see the response from DynamoDB which looks OK:

```
{
    "Attributes": {
        "nominations": {
            "SS": [
                "nomination-id-1",
                "nomination-id-2"
            ]
        }
    },
    "ResponseMetadata": {
        "RetryAttempts": 0,
        "HTTPStatusCode": 200,
        "RequestId": "3db8cae9-47e8-447b-89e4-7e35426e1a68",
        "HTTPHeaders": {
            "server": "Jetty(8.1.12.v20130726)",
            "content-length": "681",
            "x-amz-crc32": "2586038436",
            "x-amzn-requestid": "3db8cae9-47e8-447b-89e4-7e35426e1a68",
            "content-type": "application/x-amz-json-1.0"
        }
    }
}
```

but then when I query the table the type is completely different: 
```
"nominations": {
                "M": {
                    "SS": {
                        "L": [
                            {
                                "S": "nomination-id-1"
                            }, 
                            {
                                "S": "nomination-id-2"
                            }
                        ]
                    }
                }
            }
```

`print(nominations) # nominations ['nomination-id-1', 'nomination-id-2']`


Resolving patch:
diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py
--- a/moto/dynamodb2/models.py
+++ b/moto/dynamodb2/models.py
@@ -9,6 +9,7 @@
 import six
 
 import boto3
+from botocore.exceptions import ParamValidationError
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import unix_time
@@ -302,6 +303,8 @@ def update(self, update_expression, expression_attribute_names, expression_attri
                     attr, list_index = attribute_is_list(key.split('.')[0])
                     # If value not exists, changes value to a default if needed, else its the same as it was
                     value = self._get_default(value)
+                    # If operation == list_append, get the original value and append it
+                    value = self._get_appended_list(value, expression_attribute_values)
 
                     if type(value) != DynamoType:
                         if value in expression_attribute_values:
@@ -370,6 +373,18 @@ def update(self, update_expression, expression_attribute_names, expression_attri
                 else:
                     raise NotImplementedError('{} update action not yet supported'.format(action))
 
+    def _get_appended_list(self, value, expression_attribute_values):
+        if type(value) != DynamoType:
+            list_append_re = re.match('list_append\\((.+),(.+)\\)', value)
+            if list_append_re:
+                new_value = expression_attribute_values[list_append_re.group(2).strip()]
+                old_list = self.attrs[list_append_re.group(1)]
+                if not old_list.is_list():
+                    raise ParamValidationError
+                old_list.value.extend(new_value['L'])
+                value = old_list
+        return value
+
     def _get_default(self, value):
         if value.startswith('if_not_exists'):
             # Function signature


Test patch:
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -11,7 +11,7 @@
 from moto import mock_dynamodb2, mock_dynamodb2_deprecated
 from moto.dynamodb2 import dynamodb_backend2
 from boto.exception import JSONResponseError
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, ParamValidationError
 from tests.helpers import requires_boto_gte
 import tests.backport_assert_raises
 
@@ -1323,7 +1323,7 @@ def test_bad_scan_filter():
     except ClientError as err:
         err.response['Error']['Code'].should.equal('ValidationError')
     else:
-        raise RuntimeError('Should of raised ResourceInUseException')
+        raise RuntimeError('Should have raised ResourceInUseException')
 
 
 @mock_dynamodb2
@@ -1375,7 +1375,7 @@ def test_duplicate_create():
     except ClientError as err:
         err.response['Error']['Code'].should.equal('ResourceInUseException')
     else:
-        raise RuntimeError('Should of raised ResourceInUseException')
+        raise RuntimeError('Should have raised ResourceInUseException')
 
 
 @mock_dynamodb2
@@ -1400,7 +1400,7 @@ def test_delete_table():
     except ClientError as err:
         err.response['Error']['Code'].should.equal('ResourceNotFoundException')
     else:
-        raise RuntimeError('Should of raised ResourceNotFoundException')
+        raise RuntimeError('Should have raised ResourceNotFoundException')
 
 
 @mock_dynamodb2
@@ -2734,6 +2734,13 @@ def test_item_size_is_under_400KB():
                                     Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'M': {'item1': {'S': large_item}}}]}})
 
 
+def assert_failure_due_to_item_size(func, **kwargs):
+    with assert_raises(ClientError) as ex:
+        func(**kwargs)
+    ex.exception.response['Error']['Code'].should.equal('ValidationException')
+    ex.exception.response['Error']['Message'].should.equal('Item size has exceeded the maximum allowed size')
+
+
 @mock_dynamodb2
 # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html#DDB-Query-request-KeyConditionExpression
 def test_hash_key_cannot_use_begins_with_operations():
@@ -2759,11 +2766,74 @@ def test_hash_key_cannot_use_begins_with_operations():
     ex.exception.response['Error']['Message'].should.equal('Query key condition not supported')
 
 
-def assert_failure_due_to_item_size(func, **kwargs):
-    with assert_raises(ClientError) as ex:
-        func(**kwargs)
-    ex.exception.response['Error']['Code'].should.equal('ValidationException')
-    ex.exception.response['Error']['Message'].should.equal('Item size has exceeded the maximum allowed size')
+@mock_dynamodb2
+def test_update_supports_complex_expression_attribute_values():
+    client = boto3.client('dynamodb')
+
+    client.create_table(AttributeDefinitions=[{'AttributeName': 'SHA256', 'AttributeType': 'S'}],
+                        TableName='TestTable',
+                        KeySchema=[{'AttributeName': 'SHA256', 'KeyType': 'HASH'}],
+                        ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5})
+
+    client.update_item(TableName='TestTable',
+                       Key={'SHA256': {'S': 'sha-of-file'}},
+                       UpdateExpression=('SET MD5 = :md5,'
+                                         'MyStringSet = :string_set,'
+                                         'MyMap = :map' ),
+                       ExpressionAttributeValues={':md5': {'S': 'md5-of-file'},
+                                                  ':string_set': {'SS': ['string1', 'string2']},
+                                                  ':map': {'M': {'EntryKey': {'SS': ['thing1', 'thing2']}}}})
+    result = client.get_item(TableName='TestTable', Key={'SHA256': {'S': 'sha-of-file'}})['Item']
+    result.should.equal({u'MyStringSet': {u'SS': [u'string1', u'string2']},
+                         'MyMap': {u'M': {u'EntryKey': {u'SS': [u'thing1', u'thing2']}}},
+                         'SHA256': {u'S': u'sha-of-file'},
+                         'MD5': {u'S': u'md5-of-file'}})
+
+
+@mock_dynamodb2
+def test_update_supports_list_append():
+    client = boto3.client('dynamodb')
+
+    client.create_table(AttributeDefinitions=[{'AttributeName': 'SHA256', 'AttributeType': 'S'}],
+                        TableName='TestTable',
+                        KeySchema=[{'AttributeName': 'SHA256', 'KeyType': 'HASH'}],
+                        ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5})
+    client.put_item(TableName='TestTable',
+                    Item={'SHA256': {'S': 'sha-of-file'}, 'crontab': {'L': [{'S': 'bar1'}]}})
+
+    # Update item using list_append expression
+    client.update_item(TableName='TestTable',
+                       Key={'SHA256': {'S': 'sha-of-file'}},
+                       UpdateExpression="SET crontab = list_append(crontab, :i)",
+                       ExpressionAttributeValues={':i': {'L': [{'S': 'bar2'}]}})
+
+    # Verify item is appended to the existing list
+    result = client.get_item(TableName='TestTable', Key={'SHA256': {'S': 'sha-of-file'}})['Item']
+    result.should.equal({'SHA256': {'S': 'sha-of-file'},
+                         'crontab': {'L': [{'S': 'bar1'}, {'S': 'bar2'}]}})
+
+
+@mock_dynamodb2
+def test_update_catches_invalid_list_append_operation():
+    client = boto3.client('dynamodb')
+
+    client.create_table(AttributeDefinitions=[{'AttributeName': 'SHA256', 'AttributeType': 'S'}],
+                        TableName='TestTable',
+                        KeySchema=[{'AttributeName': 'SHA256', 'KeyType': 'HASH'}],
+                        ProvisionedThroughput={'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5})
+    client.put_item(TableName='TestTable',
+                    Item={'SHA256': {'S': 'sha-of-file'}, 'crontab': {'L': [{'S': 'bar1'}]}})
+
+    # Update item using invalid list_append expression
+    with assert_raises(ParamValidationError) as ex:
+        client.update_item(TableName='TestTable',
+                           Key={'SHA256': {'S': 'sha-of-file'}},
+                           UpdateExpression="SET crontab = list_append(crontab, :i)",
+                           ExpressionAttributeValues={':i': [{'S': 'bar2'}]})
+
+    # Verify correct error is returned
+    str(ex.exception).should.match("Parameter validation failed:")
+    str(ex.exception).should.match("Invalid type for parameter ExpressionAttributeValues.")
 
 
 def _create_user_table():