Repository: getmoto/moto

Issue:
dynamodb: "IndexError: list index out of range" with hash_key_expression
I'm seeing an `IndexError` with dynamodb2.responses.query:

```
key_condition_expression: {'format': '({0} {operator} {1})', 'operator': 'AND', 'values': (<boto3.dynamodb.conditions.Equals object at 0x7f698d034a20>, <boto3.dynamodb.conditions.Between object at 0x7f698d027d30>)}
QUERY: {'ExpressionAttributeNames': {'#date': 'date'},
 'KeyConditionExpression': {'format': '({0} {operator} {1})',
                            'operator': 'AND',
                            'values': ({'format': '{0} {operator} {1}',
                                        'operator': '=',
                                        'values': ('Key: organization',
                                                   'XXX')},
                                       {'format': '{0} {operator} {1} AND {2}',
                                        'operator': 'BETWEEN',
                                        'values': ('Key: date',
                                                   20160123,
                                                   20160124)})},
 'ProjectionExpression': '#date,foobar'}
Calling dynamodb:query with {'KeyConditionExpression': <boto3.dynamodb.conditions.And object at 0x7f698d034518>, 'TableName': 'foobar-baz-test', 'ProjectionExpression': '#date,foobar', 'ExpressionAttributeNames': {'#date': 'date'}}
Resetting dropped connection: dynamodb.XXX.amazonaws.com
Internal Server Error: /api/metrics/foobar
Traceback (most recent call last):
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py", line 372, in _make_request
    httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "…/pyenv/project/lib/python3.5/site-packages/django/core/handlers/base.py", line 149, in get_response
    response = self.process_exception_by_middleware(e, request)
  File "…/pyenv/project/lib/python3.5/site-packages/django/core/handlers/base.py", line 147, in get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/lib64/python3.5/contextlib.py", line 30, in inner
    return func(*args, **kwds)
  File "…/pyenv/project/lib/python3.5/site-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/django/views/generic/base.py", line 68, in view
    return self.dispatch(request, *args, **kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/rest_framework/views.py", line 466, in dispatch
    response = self.handle_exception(exc)
  File "…/pyenv/project/lib/python3.5/site-packages/rest_framework/views.py", line 463, in dispatch
    response = handler(request, *args, **kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/rest_framework/decorators.py", line 53, in handler
    return func(*args, **kwargs)
  File "…/app/views.py", line 1455, in metrics
    org, start, end)
  File "…/app/metrics.py", line 134, in get_metrics_from_dynamodb
    response = table.query(**query_kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/boto3/resources/factory.py", line 455, in do_action
    response = action(self, *args, **kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/boto3/resources/action.py", line 79, in __call__
    response = getattr(parent.meta.client, operation_name)(**params)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/client.py", line 310, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/client.py", line 396, in _make_api_call
    operation_model, request_dict)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/endpoint.py", line 111, in make_request
    return self._send_request(request_dict, operation_model)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/endpoint.py", line 140, in _send_request
    success_response, exception):
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/endpoint.py", line 213, in _needs_retry
    caught_exception=caught_exception)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/hooks.py", line 226, in emit
    return self._emit(event_name, kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/hooks.py", line 209, in _emit
    response = handler(**kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/retryhandler.py", line 183, in __call__
    if self._checker(attempts, response, caught_exception):
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/retryhandler.py", line 250, in __call__
    caught_exception)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/retryhandler.py", line 265, in _should_retry
    return self._checker(attempt_number, response, caught_exception)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/retryhandler.py", line 313, in __call__
    caught_exception)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/retryhandler.py", line 222, in __call__
    return self._check_caught_exception(attempt_number, caught_exception)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/retryhandler.py", line 355, in _check_caught_exception
    raise caught_exception
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/endpoint.py", line 168, in _get_response
    proxies=self.proxies, timeout=self.timeout)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/vendored/requests/sessions.py", line 573, in send
    r = adapter.send(request, **kwargs)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/vendored/requests/adapters.py", line 370, in send
    timeout=timeout
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py", line 544, in urlopen
    body=body, headers=headers)
  File "…/pyenv/project/lib/python3.5/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py", line 374, in _make_request
    httplib_response = conn.getresponse()
  File "/usr/lib64/python3.5/http/client.py", line 1170, in getresponse
    response = self.response_class(self.sock, method=self._method)
  File "/usr/lib64/python3.5/http/client.py", line 220, in __init__
    self.fp = sock.makefile("rb")
  File "…/pyenv/project/lib/python3.5/site-packages/httpretty/core.py", line 332, in makefile
    self._entry.fill_filekind(self.fd)
  File "…/pyenv/project/lib/python3.5/site-packages/httpretty/core.py", line 591, in fill_filekind
    status, headers, self.body = self.callable_body(self.request, self.info.full_url(), headers)
  File "…/moto/moto/core/responses.py", line 95, in dispatch
    return cls()._dispatch(*args, **kwargs)
  File "…/moto/moto/core/responses.py", line 136, in _dispatch
    return self.call_action()
  File "…/moto/moto/dynamodb2/responses.py", line 62, in call_action
    response = getattr(self, endpoint)()
  File "…/moto/moto/dynamodb2/responses.py", line 294, in query
    hash_key_value_alias = hash_key_expression.split("=")[1].strip()
IndexError: list index out of range
```

There is another issue with that code (above), where the order of "AND" and "BETWEEN" (which also has an "AND" in it) matters.

If I remember correctly there's a TODO in this regard in the code already.



Discussion:
`hash_key_expression` is `'#n1 BETWEEN :v1 AND :v2'` there.

```
(Pdb++) hash_key_expression
'#n1 BETWEEN :v1 AND :v2'
(Pdb++) key_condition_expression
'(#n0 = :v0 AND #n1 BETWEEN :v1 AND :v2)'
(Pdb++) expressions
['(#n0 = :v0']
(Pdb++) index_hash_key
{'KeyType': 'HASH', 'AttributeName': 'organization'}
(Pdb++) value_alias_map
{':v1': {'N': '20160101'}, ':v2': {'N': '20161231'}, ':v0': {'S': 'dd28753f-6e33-4a97-8b2f-687b1f8dde55'}}
(Pdb++) hash_key_expression
'#n1 BETWEEN :v1 AND :v2'
```

It should probably get both `:v1` and `:v2` out of it?
But then `dynamodb_backend2.query` only supports a single key anyway?!

Can you give some of the code for the test? I'm having trouble reproducing.

@spulec 
No, sorry.

I've decided to not use `moto` anymore, but a real DynamoDB instance instead.

You should be able to build it from the `QUERY` in the original comment though.

Below is a `pytest` test that reproduces this issue. Seems to only be an issue when I query using `begins_with` in the line `KeyConditionExpression=Key('key').begins_with('prefix-'))`.

```python
@mock_dynamodb2
def test_dynamodb():
    dynamodb = boto3.resource(
        'dynamodb',
        region_name='us-east-1',
        aws_access_key_id='MOCK_ACCESS_KEY',
        aws_secret_access_key='MOCK_SECRET_KEY',
    )
    table = dynamodb.create_table(
        TableName='test-table',
        KeySchema=[
            {
                'AttributeName': 'key',
                'KeyType': 'HASH'
            },
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'key',
                'AttributeType': 'S'
            },
            {
                'AttributeName': 'value',
                'AttributeType': 'S'
            },
        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 1,
            'WriteCapacityUnits': 1
        })

    items = [
        {
            'key': 'prefix-$LATEST',
            'value': '$LATEST',
        },
        {
            'key': 'prefix-DEV',
            'value': 'DEV',
        },
        {
            'key': 'prefix-PROD',
            'value': 'PROD',
        },
    ]

    with table.batch_writer() as batch:
        for item in items:
            batch.put_item(Item=item)

    table = dynamodb.Table('test-table')
    response = table.query(
        KeyConditionExpression=Key('key').begins_with('prefix-'))
    print(response)
    assert False
```

Resolving patch:
diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py
--- a/moto/dynamodb2/responses.py
+++ b/moto/dynamodb2/responses.py
@@ -457,6 +457,9 @@ def query(self):
                 range_comparison = None
                 range_values = []
 
+            if '=' not in hash_key_expression:
+                return self.error('com.amazonaws.dynamodb.v20111205#ValidationException',
+                                  'Query key condition not supported')
             hash_key_value_alias = hash_key_expression.split("=")[1].strip()
             # Temporary fix until we get proper KeyConditionExpression function
             hash_key = value_alias_map.get(hash_key_value_alias, {'S': hash_key_value_alias})


Test patch:
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -2705,6 +2705,31 @@ def test_item_size_is_under_400KB():
                                     Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'M': {'item1': {'S': large_item}}}]}})
 
 
+@mock_dynamodb2
+# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html#DDB-Query-request-KeyConditionExpression
+def test_hash_key_cannot_use_begins_with_operations():
+    dynamodb = boto3.resource('dynamodb')
+    table = dynamodb.create_table(
+        TableName='test-table',
+        KeySchema=[{'AttributeName': 'key', 'KeyType': 'HASH'}],
+        AttributeDefinitions=[{'AttributeName': 'key', 'AttributeType': 'S'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1})
+
+    items = [{'key': 'prefix-$LATEST', 'value': '$LATEST'},
+             {'key': 'prefix-DEV', 'value': 'DEV'},
+             {'key': 'prefix-PROD', 'value': 'PROD'}]
+
+    with table.batch_writer() as batch:
+        for item in items:
+            batch.put_item(Item=item)
+
+    table = dynamodb.Table('test-table')
+    with assert_raises(ClientError) as ex:
+        table.query(KeyConditionExpression=Key('key').begins_with('prefix-'))
+    ex.exception.response['Error']['Code'].should.equal('ValidationException')
+    ex.exception.response['Error']['Message'].should.equal('Query key condition not supported')
+
+
 def assert_failure_due_to_item_size(func, **kwargs):
     with assert_raises(ClientError) as ex:
         func(**kwargs)