Repository: getmoto/moto

Issue:
moto 1.1.22 breaks dynamodb scan 
Hello, I have an issue that started with moto 1.1.22. Reproduce with:

test_moto_failure.py
```from moto import mock_dynamodb2
import boto3

@mock_dynamodb2
def test_moto_problem():

    client = boto3.client('dynamodb',
                          aws_access_key_id="FakeKey",
                          aws_secret_access_key='FakeSecretKey',
                          aws_session_token='FakeSessionToken')

    table = client.create_table(
        TableName='table_name',
        KeySchema=[
            {
                'AttributeName': 'id',
                'KeyType': 'HASH'
            }
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'time-bucket',
                'AttributeType': 'N'
            }
        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 5,
            'WriteCapacityUnits': 5
        }
    )

    resp = client.scan(TableName='table_name',
                       Select='ALL_ATTRIBUTES',
                       FilterExpression='#tb >= :dt',
                       ExpressionAttributeNames={"#tb": "time-bucket"},
                       ExpressionAttributeValues={":dt": {"N": str(100)}
                                                  }
                       )
```

Environment:

```
$ pip freeze
asn1crypto==0.23.0
awscli==1.11.170
boto==2.48.0
boto3==1.4.7
botocore==1.7.28
certifi==2017.7.27.1
cffi==1.11.2
chardet==3.0.4
colorama==0.3.7
cookies==2.2.1
cryptography==2.1.1
daiquiri==1.3.0
dicttoxml==1.7.4
docker==2.5.1
docker-pycreds==0.2.1
docutils==0.14
idna==2.6
Jinja2==2.9.6
jmespath==0.9.3
MarkupSafe==1.0
mock==2.0.0
moto==1.1.22
numpy==1.13.1
pandas==0.20.3
pbr==3.1.1
py==1.4.34
pyaml==17.10.0
pyasn1==0.3.7
pycparser==2.18
pytest==3.2.1
python-dateutil==2.6.1
pytz==2017.2
PyYAML==3.12
requests==2.18.4
rsa==3.4.2
s3transfer==0.1.11
scikit-learn==0.19.0
scipy==0.19.1
six==1.10.0
urllib3==1.22
websocket-client==0.44.0
Werkzeug==0.12.2
xmltodict==0.11.0
```

`$ python -m pytest -k "test_moto_problem"`

Error:

```
======================================================================================================= test session starts =======================================================================================================
platform darwin -- Python 3.6.2, pytest-3.2.1, py-1.4.34, pluggy-0.4.0
rootdir: /Users/william/Documents/bde-deckard/score_server, inifile:
collected 9 items                                                                                                                                                                                                                  

tests/test_moto_failure.py F

============================================================================================================ FAILURES =============================================================================================================
________________________________________________________________________________________________________ test_moto_problem ________________________________________________________________________________________________________

    @mock_dynamodb2
    def test_moto_problem():
    
        client = boto3.client('dynamodb',
                              aws_access_key_id="FakeKey",
                              aws_secret_access_key='FakeSecretKey',
                              aws_session_token='FakeSessionToken')
    
        table = client.create_table(
            TableName='table_name',
            KeySchema=[
                {
                    'AttributeName': 'id',
                    'KeyType': 'HASH'
                }
            ],
            AttributeDefinitions=[
                {
                    'AttributeName': 'time-bucket',
                    'AttributeType': 'N'
                }
            ],
            ProvisionedThroughput={
                'ReadCapacityUnits': 5,
                'WriteCapacityUnits': 5
            }
        )
        epoch = "1000"
    
        resp = client.scan(TableName='table_name',
                           Select='ALL_ATTRIBUTES',
                           FilterExpression='#tb >= :dt',
                           ExpressionAttributeNames={"#tb": "time-bucket"},
>                          ExpressionAttributeValues={":dt": {"N": str(100)}
                                                      }
                           )

tests/test_moto_failure.py:37: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../anaconda/envs/deckard/lib/python3.6/site-packages/botocore/client.py:312: in _api_call
    return self._make_api_call(operation_name, kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <botocore.client.DynamoDB object at 0x107b044a8>, operation_name = 'Scan'
api_params = {'ExpressionAttributeNames': {'#tb': 'time-bucket'}, 'ExpressionAttributeValues': {':dt': {'N': '100'}}, 'FilterExpression': '#tb >= :dt', 'Select': 'ALL_ATTRIBUTES', ...}

    def _make_api_call(self, operation_name, api_params):
        operation_model = self._service_model.operation_model(operation_name)
        request_context = {
            'client_region': self.meta.region_name,
            'client_config': self.meta.config,
            'has_streaming_input': operation_model.has_streaming_input,
            'auth_type': operation_model.auth_type,
        }
        request_dict = self._convert_to_request_dict(
            api_params, operation_model, context=request_context)
    
        handler, event_response = self.meta.events.emit_until_response(
            'before-call.{endpoint_prefix}.{operation_name}'.format(
                endpoint_prefix=self._service_model.endpoint_prefix,
                operation_name=operation_name),
            model=operation_model, params=request_dict,
            request_signer=self._request_signer, context=request_context)
    
        if event_response is not None:
            http, parsed_response = event_response
        else:
            http, parsed_response = self._endpoint.make_request(
                operation_model, request_dict)
    
        self.meta.events.emit(
            'after-call.{endpoint_prefix}.{operation_name}'.format(
                endpoint_prefix=self._service_model.endpoint_prefix,
                operation_name=operation_name),
            http_response=http, parsed=parsed_response,
            model=operation_model, context=request_context
        )
    
        if http.status_code >= 300:
            error_code = parsed_response.get("Error", {}).get("Code")
            error_class = self.exceptions.from_code(error_code)
>           raise error_class(parsed_response, operation_name)
E           botocore.exceptions.ClientError: An error occurred (InternalFailure) when calling the Scan operation: Internal error. replace() argument 2 must be str, not float

../../../anaconda/envs/deckard/lib/python3.6/site-packages/botocore/client.py:601: ClientError
```

The error `replace() argument 2 must be str, not float` is very odd because clearly I'm sending `str(100)` as my `ExpressionAttributeValues`.

When using `pip install moto==1.1.21` the test passes.
[moto (1.1.22)] Scan operation on dynamodb2 mock with boolean filter expression raises exceptions
I have a mocked table with some seed data. 
I was able to verify that table is being mocked, sample data inserts with correct attribute types.

What I cannot figure out is why exception is raised from scan operations with BOOL filter expression `Attr('active').eq(True)` while scan operations with S filter expressions are working as expected.


```
@pytest.fixture(scope="function")
def sub_table():
    mock_dynamodb2().start()

    dynamodb = boto3.resource('dynamodb', 'us-east-1')
    dynamodb.create_table(TableName= 'devsubscriptions',
                                            KeySchema=[{'AttributeName': 'site_key', 'KeyType': 'HASH'}],
                                            AttributeDefinitions=[{'AttributeType': 'BOOL', 'AttributeName': 'active'},
                                                                               {'AttributeType': 'N', 'AttributeName': 'site_id'},
                                                                               {'AttributeType': 'S', 'AttributeName': 'frequency'},
                                                                               {'AttributeType': 'S', 'AttributeName': 'endpoint'}],                                                           
                                            ProvisionedThroughput={'ReadCapacityUnits': 5,   'WriteCapacityUnits': 5})

    table = dynamodb.Table(table_name)

        'seed_data': [
            {
                'site_key': 'random-hash',
                'active': True,
                'site_id': 1187,
                'frequency': 'daily',
                'endpoint': 'eddy+test@adlightning.com'
            }
        ]

        for data in seed_data:
            table.put_item(Item=data)

    yield table

    mock_dynamodb2().stop()
```

Scan Function
```
from boto3.dynamodb.conditions import Key, Attr
from botocore.exceptions import ClientError

    fe = Attr('active').eq(True) & Attr('frequency').eq(freq)  ##Problem filter 'active' == True
table_name = 'devsubscriptions'

def scan_table(table_name, fe):
    try:
        dynamo = boto3.resource('dynamodb', region_name='us-east-1')
        table = dynamo.Table(table_name)

        result = table.scan(FilterExpression=fe)

        return result['Items']
    except Exception as e:
        logger.error(e.response['Error']['Message'])
        raise e

```

-Table is created and seeded with data correctly.
-Attribute types look to be correct as well.
-I can scan on any attributes defined as `S` and returns correct results.
- If I scan on filter expression `Attr('frequency').eq(freq)` returns as expected

```
WITH Attr('frequency').eq('daily) expression alone:

-> result = table.scan(FilterExpression=fe)
(Pdb) table.scan()
{'Items': [{'site_key': 'random-hash', 'active': True, 'site_id': Decimal('1187'), 'frequency': 'daily', 'endpoint': 'eddy+test@adlightning.com'}], 'Count': 1, 'ScannedCount': 1, 'ConsumedCapacity': {'TableName': 'devsubscriptions', 'CapacityUnits': 1}, 'ResponseMetadata': {'HTTPStatusCode': 200, 'HTTPHeaders': {'Content-Type': 'text/plain', 'server': 'amazon.com'}, 'RetryAttempts': 0}}
```


However, the exception seems to be raised when including BOOL filter. `Attr('active').eq(True)`.
Here's the stack trace:

```
2017-10-14 00:47:02,057 botocore.hooks DEBUG    Event before-parameter-build.dynamodb.Scan: calling handler <function generate_idempotent_uuid at 0x103bb8488>
2017-10-14 00:47:02,057 botocore.endpoint DEBUG    Making request for OperationModel(name=Scan) (verify_ssl=True) with params: {'url_path': '/', 'query_string': '', 'method': 'POST', 'headers': {'X-Amz-Target': 'DynamoDB_20120810.Scan', 'Content-Type': 'application/x-amz-json-1.0', 'User-Agent': 'Boto3/1.4.7 Python/3.6.2 Darwin/16.7.0 Botocore/1.7.19 Resource'}, 'body': b'{"TableName": "devsubscriptions", "FilterExpression": "(#n0 = :v0 AND #n1 = :v1)", "ExpressionAttributeNames": {"#n0": "active", "#n1": "frequency"}, "ExpressionAttributeValues": {":v0": {"BOOL": true}, ":v1": {"S": "daily"}}}', 'url': 'https://dynamodb.us-east-1.amazonaws.com/', 'context': {'client_region': 'us-east-1', 'client_config': <botocore.config.Config object at 0x1036b8a58>, 'has_streaming_input': False, 'auth_type': None}}
2017-10-14 00:47:02,057 botocore.hooks DEBUG    Event request-created.dynamodb.Scan: calling handler <bound method RequestSigner.handler of <botocore.signers.RequestSigner object at 0x1056b2d30>>
2017-10-14 00:47:02,057 botocore.hooks DEBUG    Event choose-signer.dynamodb.Scan: calling handler <function set_operation_specific_signer at 0x103bb8378>
2017-10-14 00:47:02,058 botocore.auth DEBUG    Calculating signature using v4 auth.
2017-10-14 00:47:02,058 botocore.auth DEBUG    CanonicalRequest:
POST
/

content-type:application/x-amz-json-1.0
host:dynamodb.us-east-1.amazonaws.com
x-amz-date:20171014T074702Z
x-amz-target:DynamoDB_20120810.Scan

content-type;host;x-amz-date;x-amz-target
48ff26dd3290d08b75b3716a1a770265e809e31c5b0f1b110a1aa2c5ea70f409
2017-10-14 00:47:02,058 botocore.auth DEBUG    StringToSign:
AWS4-HMAC-SHA256
20171014T074702Z
20171014/us-east-1/dynamodb/aws4_request
f84e0264162122d66fdffa6fc69af2fbfe1159c92ac1e694162ff76884a17bee
2017-10-14 00:47:02,058 botocore.auth DEBUG    Signature:
266b42bfe2d3eaa8664bfb48e64c153088eaa90b1618411cd534cc9058725ef2
2017-10-14 00:47:02,059 botocore.endpoint DEBUG    Sending http request: <PreparedRequest [POST]>
2017-10-14 00:47:02,059 botocore.parsers DEBUG    Response headers: {'Content-Type': 'text/plain', 'server': 'amazon.com'}
2017-10-14 00:47:02,059 botocore.parsers DEBUG    Response body:
b'{"__type": "com.amazonaws.dynamodb.v20111205#InternalFailure", "message": "Internal error. \'S\'"}'
2017-10-14 00:47:02,059 botocore.hooks DEBUG    Event needs-retry.dynamodb.Scan: calling handler <botocore.retryhandler.RetryHandler object at 0x10591f5c0>
2017-10-14 00:47:02,060 botocore.retryhandler DEBUG    crc32 check skipped, the x-amz-crc32 header is not in the http response.
2017-10-14 00:47:02,060 botocore.retryhandler DEBUG    No retry needed.
2017-10-14 00:47:02,060 botocore.hooks DEBUG    Event after-call.dynamodb.Scan: calling handler <bound method TransformationInjector.inject_attribute_value_output of <boto3.dynamodb.transform.TransformationInjector object at 0x10591fda0>>
2017-10-14 00:47:02,061 root         ERROR    Internal error. 'S'```




Discussion:
Ahh I think i know whats going on


Resolving patch:
diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py
--- a/moto/dynamodb2/comparisons.py
+++ b/moto/dynamodb2/comparisons.py
@@ -61,15 +61,27 @@ def get_filter_expression(expr, names, values):
     # Do substitutions
     for key, value in names.items():
         expr = expr.replace(key, value)
+
+    # Store correct types of values for use later
+    values_map = {}
     for key, value in values.items():
         if 'N' in value:
-            expr.replace(key, float(value['N']))
+            values_map[key] = float(value['N'])
+        elif 'BOOL' in value:
+            values_map[key] = value['BOOL']
+        elif 'S' in value:
+            values_map[key] = value['S']
+        elif 'NS' in value:
+            values_map[key] = tuple(value['NS'])
+        elif 'SS' in value:
+            values_map[key] = tuple(value['SS'])
+        elif 'L' in value:
+            values_map[key] = tuple(value['L'])
         else:
-            expr = expr.replace(key, value['S'])
+            raise NotImplementedError()
 
     # Remove all spaces, tbf we could just skip them in the next step.
     # The number of known options is really small so we can do a fair bit of cheating
-    #expr = list(re.sub('\s', '', expr))  # 'Id>5ANDattribute_exists(test)ORNOTlength<6'
     expr = list(expr)
 
     # DodgyTokenisation stage 1
@@ -130,13 +142,9 @@ def is_op(val):
 
             next_token = six.next(token_iterator)
             while next_token != ')':
-                try:
-                    next_token = int(next_token)
-                except ValueError:
-                    try:
-                        next_token = float(next_token)
-                    except ValueError:
-                        pass
+                if next_token in values_map:
+                    next_token = values_map[next_token]
+
                 tuple_list.append(next_token)
                 next_token = six.next(token_iterator)
 
@@ -149,10 +157,14 @@ def is_op(val):
                 tokens2.append(tuple(tuple_list))
         elif token == 'BETWEEN':
             field = tokens2.pop()
-            op1 = int(six.next(token_iterator))
+            # if values map contains a number, it would be a float
+            # so we need to int() it anyway
+            op1 = six.next(token_iterator)
+            op1 = int(values_map.get(op1, op1))
             and_op = six.next(token_iterator)
             assert and_op == 'AND'
-            op2 = int(six.next(token_iterator))
+            op2 = six.next(token_iterator)
+            op2 = int(values_map.get(op2, op2))
             tokens2.append(['between', field, op1, op2])
 
         elif is_function(token):
@@ -169,14 +181,15 @@ def is_op(val):
             tokens2.append(function_list)
 
         else:
-            try:
-                token = int(token)
-            except ValueError:
-                try:
-                    token = float(token)
-                except ValueError:
-                    pass
-            tokens2.append(token)
+            # Convert tokens back to real types
+            if token in values_map:
+                token = values_map[token]
+
+            # Need to join >= <= <>
+            if len(tokens2) > 0 and ((tokens2[-1] == '>' and token == '=') or (tokens2[-1] == '<' and token == '=') or (tokens2[-1] == '<' and token == '>')):
+                tokens2.append(tokens2.pop() + token)
+            else:
+                tokens2.append(token)
 
     # Start of the Shunting-Yard algorithm. <-- Proper beast algorithm!
     def is_number(val):


Test patch:
diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py
--- a/tests/test_dynamodb2/test_dynamodb.py
+++ b/tests/test_dynamodb2/test_dynamodb.py
@@ -581,24 +581,24 @@ def test_filter_expression():
     row2 = moto.dynamodb2.models.Item(None, None, None, None, {'Id': {'N': '8'}, 'Subs': {'N': '10'}, 'Desc': {'S': 'A description'}, 'KV': {'SS': ['test3', 'test4']}})
 
     # AND test
-    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id > 5 AND Subs < 7', {}, {})
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id > :v0 AND Subs < :v1', {}, {':v0': {'N': 5}, ':v1': {'N': 7}})
     filter_expr.expr(row1).should.be(True)
     filter_expr.expr(row2).should.be(False)
 
     # OR test
-    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id = 5 OR Id=8', {}, {})
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id = :v0 OR Id=:v1', {}, {':v0': {'N': 5}, ':v1': {'N': 8}})
     filter_expr.expr(row1).should.be(True)
 
     # BETWEEN test
-    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id BETWEEN 5 AND 10', {}, {})
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id BETWEEN :v0 AND :v1', {}, {':v0': {'N': 5}, ':v1': {'N': 10}})
     filter_expr.expr(row1).should.be(True)
 
     # PAREN test
-    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id = 8 AND (Subs = 8 OR Subs = 5)', {}, {})
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id = :v0 AND (Subs = :v0 OR Subs = :v1)', {}, {':v0': {'N': 8}, ':v1': {'N': 5}})
     filter_expr.expr(row1).should.be(True)
 
     # IN test
-    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id IN (7,8, 9)', {}, {})
+    filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id IN :v0', {}, {':v0': {'NS': [7, 8, 9]}})
     filter_expr.expr(row1).should.be(True)
 
     # attribute function tests
@@ -655,6 +655,63 @@ def test_scan_filter():
     assert response['Count'] == 1
 
 
+@mock_dynamodb2
+def test_scan_filter2():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'N'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+    client.put_item(
+        TableName='test1',
+        Item={
+            'client': {'S': 'client1'},
+            'app': {'N': '1'}
+        }
+    )
+
+    response = client.scan(
+        TableName='test1',
+        Select='ALL_ATTRIBUTES',
+        FilterExpression='#tb >= :dt',
+        ExpressionAttributeNames={"#tb": "app"},
+        ExpressionAttributeValues={":dt": {"N": str(1)}}
+    )
+    assert response['Count'] == 1
+
+
+@mock_dynamodb2
+def test_scan_filter3():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+    dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'N'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+    client.put_item(
+        TableName='test1',
+        Item={
+            'client': {'S': 'client1'},
+            'app': {'N': '1'},
+            'active': {'BOOL': True}
+        }
+    )
+
+    table = dynamodb.Table('test1')
+    response = table.scan(
+        FilterExpression=Attr('active').eq(True)
+    )
+    assert response['Count'] == 1
+
+
 @mock_dynamodb2
 def test_bad_scan_filter():
     client = boto3.client('dynamodb', region_name='us-east-1')
@@ -680,7 +737,6 @@ def test_bad_scan_filter():
         raise RuntimeError('Should of raised ResourceInUseException')
 
 
-
 @mock_dynamodb2
 def test_duplicate_create():
     client = boto3.client('dynamodb', region_name='us-east-1')