Repository: getmoto/moto

Issue:
@mock_dynamodb2 update_item with UpdateExpression doesn't work for list items
I tried to use something like `UpdateExpression='SET testList[1]  = 10'` to update the elements of a list, which works if I use Boto3 to interact directly with an actual dynamoDB table on AWS. However, with mock_dynamodb2, I got a new item in the table named "testList[1]". An example code is attached below: 
```python
from __future__ import unicode_literals

import boto3
from moto import mock_dynamodb2
from boto3.dynamodb.conditions import Key
from botocore.exceptions import ClientError
import sure


def _create_table_with_range_key():
    dynamodb = boto3.resource('dynamodb', region_name='us-east-1')

    # Create the DynamoDB table.
    table = dynamodb.create_table(
        TableName='users',
        KeySchema=[
            {
                'AttributeName': 'forum_name',
                'KeyType': 'HASH'
            },
            {
                'AttributeName': 'subject',
                'KeyType': 'RANGE'
            },
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'forum_name',
                'AttributeType': 'S'
            },
            {
                'AttributeName': 'subject',
                'AttributeType': 'S'
            }
        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 5,
            'WriteCapacityUnits': 5
        }
    )
    return dynamodb.Table('users')


@mock_dynamodb2
def test_update_item_with_expression():
    table = _create_table_with_range_key()

    table.put_item(Item={
        'forum_name': 'the-key',
        'subject': '123',
        'field': '1',
        'testList': [1, 2, 3]
    })

    item_key = {'forum_name': 'the-key', 'subject': '123'}

    table.update_item(
        Key=item_key,
        UpdateExpression='SET testList[1]  = 10',
    )
    dict(table.get_item(Key=item_key)['Item']).should.equal({
        'field': '1',
        'forum_name': 'the-key',
        'subject': '123',
        'testList': [1, 10, 3]
    })

if __name__ == '__main__':
    test_update_item_with_expression()
```

Here is the Traceback:
```
Traceback (most recent call last):
  File "/Users/admin/anaconda3/lib/python3.6/site-packages/sure/__init__.py", line 385, in wrapper
    value = func(self, *args, **kw)
  File "/Users/admin/anaconda3/lib/python3.6/site-packages/sure/__init__.py", line 673, in equal
    raise error
AssertionError: given
X = {'field': '1', 'forum_name': 'the-key', 'subject': '123', 'testList': [Decimal('1'), Decimal('2'), Decimal('3')], 'testList[1]': '10'}
    and
Y = {'field': '1', 'forum_name': 'the-key', 'subject': '123', 'testList': [1, 10, 3]}
X has the key "'testList[1]'" whereas Y does not

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "tests.py", line 69, in <module>
    test_update_item_with_expression()
  File "/Users/admin/anaconda3/lib/python3.6/site-packages/moto/core/models.py", line 71, in wrapper
    result = func(*args, **kwargs)
  File "tests.py", line 65, in test_update_item_with_expression
    'testList': [1, 10, 3]
  File "/Users/admin/anaconda3/lib/python3.6/site-packages/sure/__init__.py", line 387, in wrapper
    raise AssertionError(e)
AssertionError: given
X = {'field': '1', 'forum_name': 'the-key', 'subject': '123', 'testList': [Decimal('1'), Decimal('2'), Decimal('3')], 'testList[1]': '10'}
    and
Y = {'field': '1', 'forum_name': 'the-key', 'subject': '123', 'testList': [1, 10, 3]}
X has the key "'testList[1]'" whereas Y does not
```



Discussion:
Thanks; setting as a feature request.

Resolving patch:
diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb2/exceptions.py
--- a/moto/dynamodb2/exceptions.py
+++ b/moto/dynamodb2/exceptions.py
@@ -2,6 +2,9 @@ class InvalidIndexNameError(ValueError):
     pass
 
 
+class InvalidUpdateExpression(ValueError):
+    pass
+
+
 class ItemSizeTooLarge(Exception):
     message = 'Item size has exceeded the maximum allowed size'
-    pass
diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py
--- a/moto/dynamodb2/models.py
+++ b/moto/dynamodb2/models.py
@@ -16,7 +16,7 @@
 from .comparisons import get_comparison_func
 from .comparisons import get_filter_expression
 from .comparisons import get_expected
-from .exceptions import InvalidIndexNameError, ItemSizeTooLarge
+from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
 
 
 class DynamoJsonEncoder(json.JSONEncoder):
@@ -237,7 +237,16 @@ def update(self, update_expression, expression_attribute_names, expression_attri
                 if action == "REMOVE":
                     key = value
                     if '.' not in key:
-                        self.attrs.pop(value, None)
+                        list_index_update = re.match('(.+)\\[([0-9]+)\\]', key)
+                        if list_index_update:
+                            # We need to remove an item from a list (REMOVE listattr[0])
+                            key_attr = self.attrs[list_index_update.group(1)]
+                            list_index = int(list_index_update.group(2))
+                            if key_attr.is_list():
+                                if len(key_attr.value) > list_index:
+                                    del key_attr.value[list_index]
+                        else:
+                            self.attrs.pop(value, None)
                     else:
                         # Handle nested dict updates
                         key_parts = key.split('.')
@@ -247,6 +256,9 @@ def update(self, update_expression, expression_attribute_names, expression_attri
 
                         last_val = self.attrs[attr].value
                         for key_part in key_parts[:-1]:
+                            list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part)
+                            if list_index_update:
+                                key_part = list_index_update.group(1)  # listattr[1] ==> listattr
                             # Hack but it'll do, traverses into a dict
                             last_val_type = list(last_val.keys())
                             if last_val_type and last_val_type[0] == 'M':
@@ -256,12 +268,23 @@ def update(self, update_expression, expression_attribute_names, expression_attri
                                 last_val[key_part] = {'M': {}}
 
                             last_val = last_val[key_part]
+                            if list_index_update:
+                                last_val = last_val['L'][int(list_index_update.group(2))]
 
                         last_val_type = list(last_val.keys())
-                        if last_val_type and last_val_type[0] == 'M':
-                            last_val['M'].pop(key_parts[-1], None)
+                        list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_parts[-1])
+                        if list_index_update:
+                            # We need to remove an item from a list (REMOVE attr.listattr[0])
+                            key_part = list_index_update.group(1)  # listattr[1] ==> listattr
+                            list_to_update = last_val[key_part]['L']
+                            index_to_remove = int(list_index_update.group(2))
+                            if index_to_remove < len(list_to_update):
+                                del list_to_update[index_to_remove]
                         else:
-                            last_val.pop(key_parts[-1], None)
+                            if last_val_type and last_val_type[0] == 'M':
+                                last_val['M'].pop(key_parts[-1], None)
+                            else:
+                                last_val.pop(key_parts[-1], None)
                 elif action == 'SET':
                     key, value = value.split("=", 1)
                     key = key.strip()
@@ -282,38 +305,61 @@ def update(self, update_expression, expression_attribute_names, expression_attri
 
                     if type(value) != DynamoType:
                         if value in expression_attribute_values:
-                            value = DynamoType(expression_attribute_values[value])
+                            dyn_value = DynamoType(expression_attribute_values[value])
                         else:
-                            value = DynamoType({"S": value})
+                            dyn_value = DynamoType({"S": value})
+                    else:
+                        dyn_value = value
 
                     if '.' not in key:
-                        self.attrs[key] = value
+                        list_index_update = re.match('(.+)\\[([0-9]+)\\]', key)
+                        if list_index_update:
+                            key_attr = self.attrs[list_index_update.group(1)]
+                            list_index = int(list_index_update.group(2))
+                            if key_attr.is_list():
+                                if len(key_attr.value) > list_index:
+                                    key_attr.value[list_index] = expression_attribute_values[value]
+                                else:
+                                    key_attr.value.append(expression_attribute_values[value])
+                            else:
+                                raise InvalidUpdateExpression
+                        else:
+                            self.attrs[key] = dyn_value
                     else:
                         # Handle nested dict updates
                         key_parts = key.split('.')
                         attr = key_parts.pop(0)
                         if attr not in self.attrs:
                             raise ValueError
-
                         last_val = self.attrs[attr].value
                         for key_part in key_parts:
+                            list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part)
+                            if list_index_update:
+                                key_part = list_index_update.group(1)  # listattr[1] ==> listattr
                             # Hack but it'll do, traverses into a dict
                             last_val_type = list(last_val.keys())
                             if last_val_type and last_val_type[0] == 'M':
-                                    last_val = last_val['M']
+                                last_val = last_val['M']
 
                             if key_part not in last_val:
                                 last_val[key_part] = {'M': {}}
-
                             last_val = last_val[key_part]
 
-                        # We have reference to a nested object but we cant just assign to it
                         current_type = list(last_val.keys())[0]
-                        if current_type == value.type:
-                            last_val[current_type] = value.value
+                        if list_index_update:
+                            # We need to add an item to a list
+                            list_index = int(list_index_update.group(2))
+                            if len(last_val['L']) > list_index:
+                                last_val['L'][list_index] = expression_attribute_values[value]
+                            else:
+                                last_val['L'].append(expression_attribute_values[value])
                         else:
-                            last_val[value.type] = value.value
-                            del last_val[current_type]
+                            # We have reference to a nested object but we cant just assign to it
+                            if current_type == dyn_value.type:
+                                last_val[current_type] = dyn_value.value
+                            else:
+                                last_val[dyn_value.type] = dyn_value.value
+                                del last_val[current_type]
 
                 elif action == 'ADD':
                     key, value = value.split(" ", 1)
diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py
--- a/moto/dynamodb2/responses.py
+++ b/moto/dynamodb2/responses.py
@@ -6,7 +6,7 @@
 
 from moto.core.responses import BaseResponse
 from moto.core.utils import camelcase_to_underscores, amzn_request_id
-from .exceptions import InvalidIndexNameError, ItemSizeTooLarge
+from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
 from .models import dynamodb_backends, dynamo_json_dump
 
 
@@ -683,6 +683,9 @@ def update_item(self):
                 name, key, update_expression, attribute_updates, expression_attribute_names,
                 expression_attribute_values, expected, condition_expression
             )
+        except InvalidUpdateExpression:
+            er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
+            return self.error(er, 'The document path provided in the update expression is invalid for update')
         except ItemSizeTooLarge:
             er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
             return self.error(er, ItemSizeTooLarge.message)


Test patch:
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -2506,6 +2506,144 @@ def test_index_with_unknown_attributes_should_fail():
     ex.exception.response['Error']['Message'].should.contain(expected_exception)
 
 
+@mock_dynamodb2
+def test_update_list_index__set_existing_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}},
+                       UpdateExpression='set itemlist[1]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar2_update'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}]}
+
+
+@mock_dynamodb2
+def test_update_list_index__set_existing_nested_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                       UpdateExpression='set itemmap.itemlist[1]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar2_update'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}]
+
+
+@mock_dynamodb2
+def test_update_list_index__set_index_out_of_range():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}},
+                       UpdateExpression='set itemlist[10]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar10'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}]}
+
+
+@mock_dynamodb2
+def test_update_list_index__set_nested_index_out_of_range():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                       UpdateExpression='set itemmap.itemlist[10]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar10'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}]
+
+
+@mock_dynamodb2
+def test_update_list_index__set_index_of_a_string():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name, Item={'id': {'S': 'foo2'}, 'itemstr': {'S': 'somestring'}})
+    with assert_raises(ClientError) as ex:
+        client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                           UpdateExpression='set itemstr[1]=:Item',
+                           ExpressionAttributeValues={':Item': {'S': 'string_update'}})
+        result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+
+    ex.exception.response['Error']['Code'].should.equal('ValidationException')
+    ex.exception.response['Error']['Message'].should.equal(
+        'The document path provided in the update expression is invalid for update')
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_existing_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[1]')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar3'}]}
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_existing_nested_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, UpdateExpression='REMOVE itemmap.itemlist[1]')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}]
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_existing_double_nested_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'},
+                          'itemmap': {'M': {'itemlist': {'L': [{'M': {'foo00': {'S': 'bar1'},
+                                                                      'foo01': {'S': 'bar2'}}},
+                                                               {'M': {'foo10': {'S': 'bar1'},
+                                                                      'foo11': {'S': 'bar2'}}}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                       UpdateExpression='REMOVE itemmap.itemlist[1].foo10')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'][0]['M'].should.equal({'foo00': {'S': 'bar1'},
+                                                                         'foo01': {'S': 'bar2'}})  # untouched
+    assert result['itemmap']['M']['itemlist']['L'][1]['M'].should.equal({'foo11': {'S': 'bar2'}})  # changed
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_index_out_of_range():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[10]')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}
+
+
+def create_table_with_list(table_name):
+    client = boto3.client('dynamodb', region_name='us-east-1')
+    client.create_table(TableName=table_name,
+                        KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}],
+                        AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}],
+                        BillingMode='PAY_PER_REQUEST')
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}})
+    return client
+
+
 @mock_dynamodb2
 def test_sorted_query_with_numerical_sort_key():
     dynamodb = boto3.resource('dynamodb', region_name='us-east-1')