Repository: Kinto/kinto

Issue:
Exclude on nested fields breaks the PostgreSQL backend.
https://kinto-ota.dev.mozaws.net/v1/buckets/tweet_blockchain_anchoring/collections/MarCharlott/records?exclude_receipt.header.chainpoint_version=1.0
Exclude on nested fields breaks the PostgreSQL backend.
https://kinto-ota.dev.mozaws.net/v1/buckets/tweet_blockchain_anchoring/collections/MarCharlott/records?exclude_receipt.header.chainpoint_version=1.0


Discussion:
```
BackendError: ProgrammingError: (psycopg2.ProgrammingError) operator does not exist: text <> numeric\nLINE 7: ...->\'receipt\'->\'header\'->>\'chainpoint_version\', \'\') NOT IN (1....\n                                                             ^\nHINT:  No operator matches the given name and argument type(s). You might need to add explicit type casts.\n [SQL: "\\n        WITH total_filtered AS (\\n            SELECT COUNT(id) AS count\\n              FROM records\\n             WHERE parent_id = %(parent_id)s\\n               AND collection_id = %(collection_id)s\\n               AND coalesce(data->%(filters_field_0_0)s->%(filters_field_0_1)s->>%(filters_field_0_2)s, \'\') NOT IN %(filters_value_0)s\\n        ),\\n        collection_filtered AS (\\n            SELECT id, last_modified, data\\n              FROM records\\n             WHERE parent_id = %(parent_id)s\\n               AND collection_id = %(collection_id)s\\n               AND coalesce(data->%(filters_field_0_0)s->%(filters_field_0_1)s->>%(filters_field_0_2)s, \'\') NOT IN %(filters_value_0)s\\n             LIMIT 10000\\n        ),\\n        fake_deleted AS (\\n            SELECT (%(deleted_field)s)::JSONB AS data\\n        ),\\n        filtered_deleted AS (\\n            SELECT id, last_modified, fake_deleted.data AS data\\n              FROM deleted, fake_deleted\\n             WHERE parent_id = %(parent_id)s\\n               AND collection_id = %(collection_id)s\\n               AND coalesce(data->%(filters_field_0_0)s->%(filters_field_0_1)s->>%(filters_field_0_2)s, \'\') NOT IN %(filters_value_0)s\\n               LIMIT 0\\n        ),\\n        all_records AS (\\n            SELECT * FROM filtered_deleted\\n             UNION ALL\\n            SELECT * FROM collection_filtered\\n        ),\\n        paginated_records AS (\\n            SELECT DISTINCT id\\n              FROM all_records\\n              \\n        )\\n        SELECT total_filtered.count AS count_total,\\n               a.id, as_epoch(a.last_modified) AS last_modified, a.data\\n          FROM paginated_records AS p JOIN all_records AS a ON (a.id = p.id),\\n               total_filtered\\n          ORDER BY last_modified DESC\\n          ;\\n        "] [parameters: {\'filters_field_0_1\': u\'header\', \'filters_field_0_0\': u\'receipt\', \'filters_field_0_2\': u\'chainpoint_version\', \'filters_value_0\': (1.0,), \'parent_id\': u\'/buckets/tweet_blockchain_anchoring/collections/MarCharlott\', \'collection_id\': \'record\', \'deleted_field\': \'{"deleted":true}\'}]\x1b[0m \x1b[36mlang\x1b[0m=\x1b[35mpt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3\x1b[0m \x1b[36muid\x1b[0m=\x1b[35mNone\x1b[0m']
```
```
BackendError: ProgrammingError: (psycopg2.ProgrammingError) operator does not exist: text <> numeric\nLINE 7: ...->\'receipt\'->\'header\'->>\'chainpoint_version\', \'\') NOT IN (1....\n                                                             ^\nHINT:  No operator matches the given name and argument type(s). You might need to add explicit type casts.\n [SQL: "\\n        WITH total_filtered AS (\\n            SELECT COUNT(id) AS count\\n              FROM records\\n             WHERE parent_id = %(parent_id)s\\n               AND collection_id = %(collection_id)s\\n               AND coalesce(data->%(filters_field_0_0)s->%(filters_field_0_1)s->>%(filters_field_0_2)s, \'\') NOT IN %(filters_value_0)s\\n        ),\\n        collection_filtered AS (\\n            SELECT id, last_modified, data\\n              FROM records\\n             WHERE parent_id = %(parent_id)s\\n               AND collection_id = %(collection_id)s\\n               AND coalesce(data->%(filters_field_0_0)s->%(filters_field_0_1)s->>%(filters_field_0_2)s, \'\') NOT IN %(filters_value_0)s\\n             LIMIT 10000\\n        ),\\n        fake_deleted AS (\\n            SELECT (%(deleted_field)s)::JSONB AS data\\n        ),\\n        filtered_deleted AS (\\n            SELECT id, last_modified, fake_deleted.data AS data\\n              FROM deleted, fake_deleted\\n             WHERE parent_id = %(parent_id)s\\n               AND collection_id = %(collection_id)s\\n               AND coalesce(data->%(filters_field_0_0)s->%(filters_field_0_1)s->>%(filters_field_0_2)s, \'\') NOT IN %(filters_value_0)s\\n               LIMIT 0\\n        ),\\n        all_records AS (\\n            SELECT * FROM filtered_deleted\\n             UNION ALL\\n            SELECT * FROM collection_filtered\\n        ),\\n        paginated_records AS (\\n            SELECT DISTINCT id\\n              FROM all_records\\n              \\n        )\\n        SELECT total_filtered.count AS count_total,\\n               a.id, as_epoch(a.last_modified) AS last_modified, a.data\\n          FROM paginated_records AS p JOIN all_records AS a ON (a.id = p.id),\\n               total_filtered\\n          ORDER BY last_modified DESC\\n          ;\\n        "] [parameters: {\'filters_field_0_1\': u\'header\', \'filters_field_0_0\': u\'receipt\', \'filters_field_0_2\': u\'chainpoint_version\', \'filters_value_0\': (1.0,), \'parent_id\': u\'/buckets/tweet_blockchain_anchoring/collections/MarCharlott\', \'collection_id\': \'record\', \'deleted_field\': \'{"deleted":true}\'}]\x1b[0m \x1b[36mlang\x1b[0m=\x1b[35mpt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3\x1b[0m \x1b[36muid\x1b[0m=\x1b[35mNone\x1b[0m']
```

Resolving patch:
diff --git a/kinto/core/storage/postgresql/__init__.py b/kinto/core/storage/postgresql/__init__.py
--- a/kinto/core/storage/postgresql/__init__.py
+++ b/kinto/core/storage/postgresql/__init__.py
@@ -7,7 +7,7 @@
     StorageBase, exceptions,
     DEFAULT_ID_FIELD, DEFAULT_MODIFIED_FIELD, DEFAULT_DELETED_FIELD)
 from kinto.core.storage.postgresql.client import create_from_config
-from kinto.core.utils import COMPARISON, json
+from kinto.core.utils import COMPARISON, json, is_numeric
 
 
 class Storage(StorageBase):
@@ -697,16 +697,21 @@ def _format_conditions(self, filters, id_field, modified_field,
                 # If field is missing, we default to ''.
                 sql_field = "coalesce({}, '')".format(column_name)
                 # Cast when comparing to number (eg. '4' < '12')
-                if isinstance(value, (int, float)) and \
-                   value not in (True, False):
-                    sql_field = "({})::numeric".format(column_name)
+                try:
+                    if is_numeric(value) or all([is_numeric(v) for v in value]):
+                        sql_field = "({})::numeric".format(column_name)
+                except TypeError:  # not iterable
+                    pass
 
             if filtr.operator not in (COMPARISON.IN, COMPARISON.EXCLUDE):
                 # For the IN operator, let psycopg escape the values list.
                 # Otherwise JSON-ify the native value (e.g. True -> 'true')
-                if not isinstance(filtr.value, str):
-                    value = json.dumps(filtr.value).strip('"')
+                if not isinstance(value, str):
+                    value = json.dumps(value).strip('"')
             else:
+                # If not all numeric, fallback to string comparison.
+                if not all([is_numeric(v) for v in value]):
+                    value = [str(v) for v in value]
                 value = tuple(value)
                 # WHERE field IN ();  -- Fails with syntax error.
                 if len(value) == 0:
diff --git a/kinto/core/utils.py b/kinto/core/utils.py
--- a/kinto/core/utils.py
+++ b/kinto/core/utils.py
@@ -58,6 +58,14 @@ def classname(obj):
     return obj.__class__.__name__.lower()
 
 
+def is_numeric(value):
+    """Check if the provided value is a numeric value.
+
+    :rtype: bool
+    """
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
 def merge_dicts(a, b):
     """Merge b into a recursively, without overwriting values.
 


Test patch:
diff --git a/kinto/core/storage/testing.py b/kinto/core/storage/testing.py
--- a/kinto/core/storage/testing.py
+++ b/kinto/core/storage/testing.py
@@ -450,6 +450,30 @@ def test_get_all_can_filter_with_list_of_excluded_values(self):
                                           **self.storage_kw)
         self.assertEqual(len(records), 1)
 
+    def test_get_all_can_filter_a_list_of_integer_values(self):
+        for l in [1, 2, 3]:
+            self.create_record({'code': l})
+        filters = [Filter('code', (1, 2), utils.COMPARISON.EXCLUDE)]
+        records, _ = self.storage.get_all(filters=filters,
+                                          **self.storage_kw)
+        self.assertEqual(len(records), 1)
+
+    def test_get_all_can_filter_a_list_of_mixed_typed_values(self):
+        for l in [1, 2, 3]:
+            self.create_record({'code': l})
+        filters = [Filter('code', (1, "b"), utils.COMPARISON.EXCLUDE)]
+        records, _ = self.storage.get_all(filters=filters,
+                                          **self.storage_kw)
+        self.assertEqual(len(records), 2)
+
+    def test_get_all_can_filter_a_list_of_integer_values_on_subobjects(self):
+        for l in [1, 2, 3]:
+            self.create_record({'code': {'city': l}})
+        filters = [Filter('code.city', (1, 2), utils.COMPARISON.EXCLUDE)]
+        records, _ = self.storage.get_all(filters=filters,
+                                          **self.storage_kw)
+        self.assertEqual(len(records), 1)
+
     def test_get_all_can_filter_by_subobjects_values(self):
         for l in ['a', 'b', 'c']:
             self.create_record({'code': {'sub': l}})