Repository: getmoto/moto

Issue:
pynamodb startswith works with moto but not under pytest
environment:
windows 10 64 bit python 3.6

see zip for code to reproduce issue.
do a virtualenv and install requirements. model, test_model and test_dynamodb contain the same code, except that the 2nd and 3rd are wrapped in order for pytest to work. The 3rd, doesn't use moto, but a dynamodb local listening on localhost 8000.
If you execute
 python model.py,
 everything works correctly.
if, in the test dir, you execute
pytest
the 'query' call using moto mock which uses the startswith, fails with
============================= test session starts =============================
platform win32 -- Python 3.6.4, pytest-4.0.2, py-1.7.0, pluggy-0.8.0
rootdir: C:\Users\FIM514, inifile: pytest.ini
collected 1 item

test_model.py F                                                          [100%]

================================== FAILURES ===================================
______________________________ MyTest.test_main _______________________________

self = <test.test_model.MyTest testMethod=test_main>

    def test_main(self):
        model.create_table(wait=True)
        model(f1='1', f2='2').save()
        print(next(model.query('1', model.f2=='2')))
>       print(next(model.query('1', model.f2.startswith('2'))))

test_model.py:24:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
..\envs\temp\lib\site-packages\pynamodb\pagination.py:173: in __next__
    self._get_next_page()
..\envs\temp\lib\site-packages\pynamodb\pagination.py:158: in _get_next_page
    page = next(self.page_iter)
..\envs\temp\lib\site-packages\pynamodb\pagination.py:103: in __next__
    page = self._operation(*self._args, **self._kwargs)
..\envs\temp\lib\site-packages\pynamodb\connection\table.py:257: in query
    select=select)
..\envs\temp\lib\site-packages\pynamodb\connection\base.py:1409: in query
    return self.dispatch(QUERY, operation_kwargs)
..\envs\temp\lib\site-packages\pynamodb\connection\base.py:313: in dispatch
    data = self._make_api_call(operation_name, operation_kwargs)
..\envs\temp\lib\site-packages\pynamodb\connection\base.py:362: in _make_api_call
    proxies=proxies,
..\envs\temp\lib\site-packages\requests\sessions.py:646: in send
    r = adapter.send(request, **kwargs)
..\envs\temp\lib\site-packages\responses.py:622: in unbound_on_send
    return self._on_request(adapter, request, *a, **kwargs)
..\envs\temp\lib\site-packages\responses.py:600: in _on_request
    response = adapter.build_response(request, match.get_response(request))
..\envs\temp\lib\site-packages\moto\core\models.py:151: in get_response
    result = self.callback(request)
..\envs\temp\lib\site-packages\moto\core\utils.py:173: in __call__
    result = self.callback(request, request.url, request.headers)
..\envs\temp\lib\site-packages\moto\core\responses.py:117: in dispatch
    return cls()._dispatch(*args, **kwargs)
..\envs\temp\lib\site-packages\moto\core\responses.py:200: in _dispatch
    return self.call_action()
..\envs\temp\lib\site-packages\moto\core\utils.py:264: in _wrapper
    response = f(*args, **kwargs)
..\envs\temp\lib\site-packages\moto\dynamodb2\responses.py:64: in call_action
    response = getattr(self, endpoint)()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <moto.dynamodb2.responses.DynamoHandler object at 0x000002DC6554C6D8>

    def query(self):
        name = self.body['TableName']
        # {u'KeyConditionExpression': u'#n0 = :v0', u'ExpressionAttributeValues': {u':v0': {u'S': u'johndoe'}}, u'ExpressionAttributeNames': {u'#n0': u'username'}}
        key_condition_expression = self.body.get('KeyConditionExpression')
        projection_expression = self.body.get('ProjectionExpression')
        expression_attribute_names = self.body.get('ExpressionAttributeNames', {})
        filter_expression = self.body.get('FilterExpression')
        expression_attribute_values = self.body.get('ExpressionAttributeValues', {})

        if projection_expression and expression_attribute_names:
            expressions = [x.strip() for x in projection_expression.split(',')]
            for expression in expressions:
                if expression in expression_attribute_names:
                    projection_expression = projection_expression.replace(expression, expression_attribute_names[expression])

        filter_kwargs = {}

        if key_condition_expression:
            value_alias_map = self.body.get('ExpressionAttributeValues', {})

            table = self.dynamodb_backend.get_table(name)

            # If table does not exist
            if table is None:
                return self.error('com.amazonaws.dynamodb.v20120810#ResourceNotFoundException',
                                  'Requested resource not found')

            index_name = self.body.get('IndexName')
            if index_name:
                all_indexes = (table.global_indexes or []) + \
                    (table.indexes or [])
                indexes_by_name = dict((i['IndexName'], i)
                                       for i in all_indexes)
                if index_name not in indexes_by_name:
                    raise ValueError('Invalid index: %s for table: %s. Available indexes are: %s' % (
                        index_name, name, ', '.join(indexes_by_name.keys())
                    ))

                index = indexes_by_name[index_name]['KeySchema']
            else:
                index = table.schema

            reverse_attribute_lookup = dict((v, k) for k, v in
                                            six.iteritems(self.body.get('ExpressionAttributeNames', {})))

            if " AND " in key_condition_expression:
                expressions = key_condition_expression.split(" AND ", 1)

                index_hash_key = [key for key in index if key['KeyType'] == 'HASH'][0]
                hash_key_var = reverse_attribute_lookup.get(index_hash_key['AttributeName'],
                                                            index_hash_key['AttributeName'])
                hash_key_regex = r'(^|[\s(]){0}\b'.format(hash_key_var)
                i, hash_key_expression = next((i, e) for i, e in enumerate(expressions)
                                              if re.search(hash_key_regex, e))
                hash_key_expression = hash_key_expression.strip('()')
                expressions.pop(i)

                # TODO implement more than one range expression and OR operators
                range_key_expression = expressions[0].strip('()')
                range_key_expression_components = range_key_expression.split()
                range_comparison = range_key_expression_components[1]

                if 'AND' in range_key_expression:
                    range_comparison = 'BETWEEN'
                    range_values = [
                        value_alias_map[range_key_expression_components[2]],
                        value_alias_map[range_key_expression_components[4]],
                    ]
                elif 'begins_with' in range_key_expression:
                    range_comparison = 'BEGINS_WITH'
                    range_values = [
>                       value_alias_map[range_key_expression_components[1]],
                    ]
E                   KeyError: '(#1,'

..\envs\temp\lib\site-packages\moto\dynamodb2\responses.py:382: KeyError
---------------------------- Captured stdout call -----------------------------
table<1, 2>


the query call in test_dynamodb, once I start dynamodb-local so I don't use moto mock, works correctly.
I expect the 2nd case to be working correctly too.
[test.zip](https://github.com/spulec/moto/files/2679965/test.zip)



Discussion:
Can you please add the code needed to reproduce this to the issue text? Thanks
Also running into this issue. Here's a small example (using pynamodb for table declaration)

```
import unittest
from pynamodb.attributes import UnicodeAttribute
from pynamodb.attributes import NumberAttribute
from pynamodb.models import Model
from moto import mock_dynamodb2


class TestTable(Model):

    class Meta:
        table_name = 'TestTable'
        read_capacity_units = 25
        write_capacity_units = 25

    user_id = NumberAttribute(hash_key=True)
    table_id = UnicodeAttribute(range_key=True)


@mock_dynamodb2
class TestTestTable(unittest.TestCase):

    def setUp(self):
        TestTable.create_table(wait=True)

    def tearDown(self):
        TestTable.delete_table()

    def test_beginswith(self):
        try:
            return TestTable.query(
                1234,
                range_key_condition=(TestTable.table_id.startswith('something')),
                scan_index_forward=False,
                limit=1,
            ).next()
        except StopIteration:
            return None
```

Python 3.7.3
boto==2.49.0
boto3==1.9.133
botocore==1.12.133
moto==1.3.7
pynamodb==4.0.0b1
I think that might be the same as https://github.com/spulec/moto/issues/416

Can you try activating the mock in your setUp with a [manual mock](https://github.com/spulec/moto#raw-use).
Like this? (I get the keyerror with the startswith, but doing a plain query works fine)

```
import unittest
from pynamodb.attributes import UnicodeAttribute
from pynamodb.attributes import NumberAttribute
from pynamodb.models import Model
from moto import mock_dynamodb2


class TestTable(Model):

    class Meta:
        table_name = 'TestTable'
        read_capacity_units = 25
        write_capacity_units = 25

    user_id = NumberAttribute(hash_key=True)
    table_id = UnicodeAttribute(range_key=True)


class TestTestTable(unittest.TestCase):

    def test_beginswith(self):  # Keyerror
        mock = mock_dynamodb2()
        mock.start()
        TestTable.create_table(wait=True)

        try:
            return TestTable.query(
                1234,
                range_key_condition=(TestTable.table_id.startswith('something')),
                scan_index_forward=False,
                limit=1,
            ).next()
        except StopIteration:
            return None

        TestTable.delete_table()
        mock.stop()

    def test_query(self):  # Works fine
        mock = mock_dynamodb2()
        mock.start()
        TestTable.create_table(wait=True)

        try:
            return TestTable.query(
                1234,
            ).next()
        except StopIteration:
            return None

        TestTable.delete_table()
        mock.stop()
```
Can you print the traceback for the KeyError? It sounds like it might be a different issue, but at least the mock is being activated.
```
self = <test.test.TestTestTable testMethod=test_beginswith>

    def test_beginswith(self):  # Keyerror
        mock = mock_dynamodb2()
        mock.start()
        TestTable.create_table(wait=True)
    
        try:
            return TestTable.query(
                1234,
                range_key_condition=(TestTable.table_id.startswith('something')),
                scan_index_forward=False,
>               limit=1,
            ).next()

test/test.py:31: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
venv/lib/python3.7/site-packages/pynamodb/pagination.py:187: in next
    return self.__next__()
venv/lib/python3.7/site-packages/pynamodb/pagination.py:173: in __next__
    self._get_next_page()
venv/lib/python3.7/site-packages/pynamodb/pagination.py:158: in _get_next_page
    page = next(self.page_iter)
venv/lib/python3.7/site-packages/pynamodb/pagination.py:103: in __next__
    page = self._operation(*self._args, **self._kwargs)
venv/lib/python3.7/site-packages/pynamodb/connection/table.py:261: in query
    select=select)
venv/lib/python3.7/site-packages/pynamodb/connection/base.py:1424: in query
    return self.dispatch(QUERY, operation_kwargs)
venv/lib/python3.7/site-packages/pynamodb/connection/base.py:316: in dispatch
    data = self._make_api_call(operation_name, operation_kwargs)
venv/lib/python3.7/site-packages/pynamodb/connection/base.py:369: in _make_api_call
    event_responses = self.client._endpoint._event_emitter.emit(event_name, request=prepared_request)
venv/lib/python3.7/site-packages/botocore/hooks.py:356: in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
venv/lib/python3.7/site-packages/botocore/hooks.py:228: in emit
    return self._emit(event_name, kwargs)
venv/lib/python3.7/site-packages/botocore/hooks.py:211: in _emit
    response = handler(**kwargs)
venv/lib/python3.7/site-packages/moto/core/models.py:292: in __call__
    status, headers, body = response_callback(request, request.url, request.headers)
venv/lib/python3.7/site-packages/moto/core/responses.py:117: in dispatch
    return cls()._dispatch(*args, **kwargs)
venv/lib/python3.7/site-packages/moto/core/responses.py:200: in _dispatch
    return self.call_action()
venv/lib/python3.7/site-packages/moto/core/utils.py:264: in _wrapper
    response = f(*args, **kwargs)
venv/lib/python3.7/site-packages/moto/dynamodb2/responses.py:64: in call_action
    response = getattr(self, endpoint)()
```
(sorry, I've not been getting notifications of this thread, so my responses have been a bit delayed)
Does the traceback go deeper than that? That particular line in responses.py seems unusual to throw a KeyError.
Nope, sorry.
```
venv/lib/python3.7/site-packages/moto/dynamodb2/responses.py:64: in call_action
    response = getattr(self, endpoint)()
```
is the last thing returned.
Can you put a pdb inside of dynamodb2/responses.py:query and step through it until the exception happens? I'm trying to isolate where this is coming from and I'm struggling to reproduce it.
```
test/test.py > python3.7/site-packages/moto/dynamodb2/responses.py(315)query()
    314         # {u'KeyConditionExpression': u'#n0 = :v0', u'ExpressionAttributeValues': {u':v0': {u'S': u'johndoe'}}, u'ExpressionAttributeNames': {u'#n0': u'username'}}
--> 315         key_condition_expression = self.body.get('KeyConditionExpression')
    316         projection_expression = self.body.get('ProjectionExpression')

ipdb> self.body                                                                                                   
{'TableName': 'TestTable', 'KeyConditionExpression': '(#0 = :0 AND begins_with (#1, :1))', 'Limit': 1, 'ScanIndexForward': False, 'ExpressionAttributeNames': {'#0': 'user_id', '#1': 'table_id'}, 'ExpressionAttributeValues': {':0': {'N': '1234'}, ':1': {'S': 'something'}}, 'ReturnConsumedCapacity': 'TOTAL'}
```
...
```
ipdb> key_condition_expression                                                                                    
'(#0 = :0 AND begins_with (#1, :1))'
ipdb> projection_expression                                                                                       
ipdb> expression_attribute_names                                                                                  
{'#0': 'user_id', '#1': 'table_id'}
ipdb> filter_expression                                                                                           
ipdb> expression_attribute_values                                                                                 
{':0': {'N': '1234'}, ':1': {'S': 'something'}}
ipdb>     
```
...
```
> python3.7/site-packages/moto/dynamodb2/responses.py(381)query()
    380                 elif 'begins_with' in range_key_expression:
--> 381                     range_comparison = 'BEGINS_WITH'
    382                     range_values = [

ipdb> n                                                                                                           
> python3.7/site-packages/moto/dynamodb2/responses.py(383)query()
    382                     range_values = [
--> 383                         value_alias_map[range_key_expression_components[1]],
    384                     ]

ipdb> n                                                                                                           
KeyError: '(#1,'
> python3.7/site-packages/moto/dynamodb2/responses.py(383)query()
    382                     range_values = [
--> 383                         value_alias_map[range_key_expression_components[1]],
    384                     ]

ipdb> range_key_expression_components                                                                             
['begins_with', '(#1,', ':1']
ipdb> value_alias_map                                                                                             
{':0': {'N': '1234'}, ':1': {'S': 'something'}}
```
Looks like it was expecting the `':1'` to be the 1st index and not `'(#1'`

Doing some more digging (eg comparing to the tests in test_dynamodb2/test_dynamodb_table_with_range_key.py), and in my test case, there ends up being a space after "begins_with".... whereas 

```
# test_dynamodb_table_with_range_key.py
ipdb> range_key_expression                                                     
'begins_with(#n1, :v1'
```
vs
```
# test.py
begins_with (#1, :1))'
```
Ah, got it. So it sounds like we are being too strict in what we are parsing. I'm going to set this as a feature to make the parsing be looser.
I have this problem too, any news here?

Resolving patch:
diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py
--- a/moto/dynamodb2/responses.py
+++ b/moto/dynamodb2/responses.py
@@ -438,9 +438,12 @@ def query(self):
                 all_indexes = (table.global_indexes or []) + (table.indexes or [])
                 indexes_by_name = dict((i["IndexName"], i) for i in all_indexes)
                 if index_name not in indexes_by_name:
-                    raise ValueError(
-                        "Invalid index: %s for table: %s. Available indexes are: %s"
-                        % (index_name, name, ", ".join(indexes_by_name.keys()))
+                    er = "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException"
+                    return self.error(
+                        er,
+                        "Invalid index: {} for table: {}. Available indexes are: {}".format(
+                            index_name, name, ", ".join(indexes_by_name.keys())
+                        ),
                     )
 
                 index = indexes_by_name[index_name]["KeySchema"]
@@ -481,7 +484,9 @@ def query(self):
                     ]
                 elif "begins_with" in range_key_expression:
                     range_comparison = "BEGINS_WITH"
-                    range_values = [value_alias_map[range_key_expression_components[1]]]
+                    range_values = [
+                        value_alias_map[range_key_expression_components[-1]]
+                    ]
                 else:
                     range_values = [value_alias_map[range_key_expression_components[2]]]
             else:


Test patch:
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -2630,6 +2630,44 @@ def test_scan_by_non_exists_index():
     )
 
 
+@mock_dynamodb2
+def test_query_by_non_exists_index():
+    dynamodb = boto3.client("dynamodb", region_name="us-east-1")
+
+    dynamodb.create_table(
+        TableName="test",
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[
+            {"AttributeName": "id", "AttributeType": "S"},
+            {"AttributeName": "gsi_col", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
+        GlobalSecondaryIndexes=[
+            {
+                "IndexName": "test_gsi",
+                "KeySchema": [{"AttributeName": "gsi_col", "KeyType": "HASH"}],
+                "Projection": {"ProjectionType": "ALL"},
+                "ProvisionedThroughput": {
+                    "ReadCapacityUnits": 1,
+                    "WriteCapacityUnits": 1,
+                },
+            }
+        ],
+    )
+
+    with assert_raises(ClientError) as ex:
+        dynamodb.query(
+            TableName="test",
+            IndexName="non_exists_index",
+            KeyConditionExpression="CarModel=M",
+        )
+
+    ex.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+    ex.exception.response["Error"]["Message"].should.equal(
+        "Invalid index: non_exists_index for table: test. Available indexes are: test_gsi"
+    )
+
+
 @mock_dynamodb2
 def test_batch_items_returns_all():
     dynamodb = _create_user_table()