Repository: proteanhq/protean

Issue:
Add support for `where` query method to accept direct database queries

Discussion:

Patch:
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index ba7b4d42..12bf3f16 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -118,6 +118,13 @@ To set up `protean` for local development:
 
 6. Submit a pull request through the GitHub website.
 
+Testing
+^^^^^^^
+
+Protean tests have been written to work with **Pytest**, and you can run tests with the command ``pytest``.
+
+By default, tests that are marked as ``slow`` are skipped. If you want to run the slow tests, use ``pytest --slow``.
+
 Pull Request Guidelines
 ^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/src/protean/core/provider/base.py b/src/protean/core/provider/base.py
index f079586d..7bb6f774 100644
--- a/src/protean/core/provider/base.py
+++ b/src/protean/core/provider/base.py
@@ -3,6 +3,7 @@
 import uuid
 from abc import ABCMeta
 from abc import abstractmethod
+from typing import Any
 
 from protean.utils.query import RegisterLookupMixin
 
@@ -46,16 +47,26 @@ class BaseProvider(RegisterLookupMixin, metaclass=ABCMeta):
 
     @abstractmethod
     def get_connection(self):
-        """ Get the connection object for the repository"""
+        """Get the connection object for the repository"""
 
     @abstractmethod
     def close_connection(self, conn):
-        """ Close the connection object for the repository"""
+        """Close the connection object for the repository"""
 
     @abstractmethod
     def get_repository(self, model_cls):
-        """ Return a repository object configured with a live connection"""
+        """Return a repository object configured with a live connection"""
 
     @abstractmethod
     def get_model(self, entity_cls):
-        """ Return associated Model Class"""
+        """Return associated Model Class"""
+
+    @abstractmethod
+    def raw(self, query: Any, data: Any = None):
+        """Run raw query directly on the database
+
+        Query should be executed immediately on the database as a separate unit of work
+            (in a different transaction context). The results should be returned as returned by
+            the database without any intervention. It is left to the consumer to interpret and
+            organize the results correctly.
+        """
diff --git a/src/protean/core/queryset.py b/src/protean/core/queryset.py
index 8d9dbc7f..d9423b35 100644
--- a/src/protean/core/queryset.py
+++ b/src/protean/core/queryset.py
@@ -1,7 +1,7 @@
 """QuerySet Implementation"""
 
 import logging
-from typing import Union
+from typing import Union, Any
 
 from protean.core.repository import repo_factory
 from protean.utils.query import Q
@@ -169,18 +169,18 @@ class QuerySet:
 
         return updated_item_count
 
-    def raw(self, query_string: str):
+    def raw(self, query: Any, data: Any = None):
         """Runs raw query directly on the database and returns Entity objects
 
         Note that this method will raise an exception if the returned objects
             are not of the Entity type.
 
-        `query_string` is not checked for correctness or validity, and any errors thrown
-            by the plugin or database are passed as-is.
+        `query` is not checked for correctness or validity, and any errors thrown by the plugin or
+            database are passed as-is. Data passed will be transferred as-is to the plugin.
 
         All other query options like `order_by`, `page` and `per_page` are ignored for this action.
         """
-        logger.debug(f'Query `{self.__class__.__name__}` objects with raw query {query_string}')
+        logger.debug(f'Query `{self.__class__.__name__}` objects with raw query {query}')
 
         # Destroy any cached results
         self._result_cache = None
@@ -190,9 +190,8 @@ class QuerySet:
         repository = repo_factory.get_repository(self._entity_cls)
 
         try:
-            # Call the read method of the repository
-            # Ensures that the string contains double quotes around keys and values
-            results = repository.raw(query_string.replace("'", "\""))
+            # Call the raw method of the repository
+            results = repository.raw(query, data)
 
             # Convert the returned results to entity and return it
             entity_items = []
diff --git a/src/protean/core/repository/base.py b/src/protean/core/repository/base.py
index fbf34606..d46410ac 100644
--- a/src/protean/core/repository/base.py
+++ b/src/protean/core/repository/base.py
@@ -54,5 +54,9 @@ class BaseRepository(metaclass=ABCMeta):
         """Delete a Record from the Repository"""
 
     @abstractmethod
-    def raw(self, query_string: str):
-        """Run raw query on Repository"""
+    def raw(self, query: Any, data: Any = None):
+        """Run raw query on Data source.
+
+        Running a raw query on the repository should always returns entity instance objects. If
+        the results were not synthesizable back into entity objects, an exception should be thrown.
+        """
diff --git a/src/protean/impl/repository/dict_repo.py b/src/protean/impl/repository/dict_repo.py
index a682e827..a2eaac7f 100644
--- a/src/protean/impl/repository/dict_repo.py
+++ b/src/protean/impl/repository/dict_repo.py
@@ -1,10 +1,11 @@
-""" Implementation of a dictionary based repository """
+"""Implementation of a dictionary based repository """
 
 import json
 from collections import defaultdict
 from itertools import count
 from operator import itemgetter
 from threading import Lock
+from typing import Any
 
 from protean.core.entity import Entity
 from protean.core.exceptions import ObjectNotFoundError
@@ -23,11 +24,11 @@ _counters = defaultdict(count)
 
 
 class DictModel(BaseModel):
-    """ A model for the dictionary repository"""
+    """A model for the dictionary repository"""
 
     @classmethod
     def from_entity(cls, entity: Entity) -> 'DictModel':
-        """ Convert the entity to a dictionary record """
+        """Convert the entity to a dictionary record """
         dict_obj = {}
         for field_name in entity.meta_.attributes:
             dict_obj[field_name] = getattr(entity, field_name)
@@ -35,7 +36,7 @@ class DictModel(BaseModel):
 
     @classmethod
     def to_entity(cls, item: 'DictModel') -> Entity:
-        """ Convert the dictionary record to an entity """
+        """Convert the dictionary record to an entity """
         return cls.entity_cls(item)
 
 
@@ -51,7 +52,7 @@ class DictProvider(BaseProvider):
         pass
 
     def get_connection(self):
-        """ Return the dictionary database object """
+        """Return the dictionary database object """
         database = {
             'data': _databases.setdefault(self.identifier, defaultdict(dict)),
             'lock': _locks.setdefault(self.identifier, Lock()),
@@ -60,27 +61,78 @@ class DictProvider(BaseProvider):
         return database
 
     def close_connection(self, conn):
-        """ Close connection does nothing on the repo """
+        """Close connection does nothing on the repo """
         pass
 
     def get_model(self, entity_cls):
-        """ Return associated, fully-baked Model class"""
+        """Return associated, fully-baked Model class"""
         cls = DictModel
         cls.entity_cls = entity_cls
 
         return cls
 
     def get_repository(self, entity_cls):
-        """ Return a repository object configured with a live connection"""
+        """Return a repository object configured with a live connection"""
         model_cls = self.get_model(entity_cls)
         return DictRepository(self, entity_cls, model_cls)
 
+    def _evaluate_lookup(self, key, value, negated, db):
+        """Extract values from DB that match the given criteria"""
+        results = {}
+        for record_key, record_value in db.items():
+            match = True
+
+            stripped_key, lookup_class = self._extract_lookup(key)
+            lookup = lookup_class(record_value[stripped_key], value)
+
+            if negated:
+                match &= not eval(lookup.as_expression())
+            else:
+                match &= eval(lookup.as_expression())
+
+            if match:
+                results[record_key] = record_value
+
+        return results
+
+    def raw(self, query: Any, data: Any = None):
+        """Run raw queries on the database
+
+        As an example of running ``raw`` queries on a Dict repository, we will run the query
+        on all possible schemas, and return all results.
+        """
+        assert isinstance(query, str)
+
+        conn = self.get_connection()
+        items = []
+
+        for schema_name in conn['data']:
+            input_db = conn['data'][schema_name]
+            try:
+                # Ensures that the string contains double quotes around keys and values
+                query = query.replace("'", "\"")
+                criteria = json.loads(query)
+
+                for key, value in criteria.items():
+                    input_db = self._evaluate_lookup(key, value, False, input_db)
+
+                items.extend(list(input_db.values()))
+
+            except json.JSONDecodeError:
+                # FIXME Log Exception
+                raise Exception("Query Malformed")
+            except KeyError:
+                # We encountered a repository where the key was not found
+                pass
+
+        return items
+
 
 class DictRepository(BaseRepository):
-    """ A repository for storing data in a dictionary """
+    """A repository for storing data in a dictionary """
 
     def _set_auto_fields(self, model_obj):
-        """ Set the values of the auto field using counter"""
+        """Set the values of the auto field using counter"""
         for field_name, field_obj in \
                 self.entity_cls.meta_.auto_fields:
             counter_key = f'{self.schema_name}_{field_name}'
@@ -94,7 +146,7 @@ class DictRepository(BaseRepository):
         return model_obj
 
     def create(self, model_obj):
-        """ Write a record to the dict repository"""
+        """Write a record to the dict repository"""
         # Update the value of the counters
         model_obj = self._set_auto_fields(model_obj)
 
@@ -105,25 +157,6 @@ class DictRepository(BaseRepository):
 
         return model_obj
 
-    def _evaluate_lookup(self, key, value, negated, db):
-        """Extract values from DB that match the given criteria"""
-        results = {}
-        for record_key, record_value in db.items():
-            match = True
-
-            stripped_key, lookup_class = self.provider._extract_lookup(key)
-            lookup = lookup_class(record_value[stripped_key], value)
-
-            if negated:
-                match &= not eval(lookup.as_expression())
-            else:
-                match &= eval(lookup.as_expression())
-
-            if match:
-                results[record_key] = record_value
-
-        return results
-
     def _filter(self, criteria: Q, db):
         """Recursive function to filter items from dictionary"""
         # Filter the dictionary objects based on the filters
@@ -138,7 +171,8 @@ class DictRepository(BaseRepository):
                 if isinstance(child, Q):
                     input_db = self._filter(child, input_db)
                 else:
-                    input_db = self._evaluate_lookup(child[0], child[1], negated, input_db)
+                    input_db = self.provider._evaluate_lookup(child[0], child[1],
+                                                              negated, input_db)
         else:
             # Grow database records over successive iterations
             #   Whatever is left at the end satisfy any criteria (OR)
@@ -147,14 +181,14 @@ class DictRepository(BaseRepository):
                 if isinstance(child, Q):
                     results = self._filter(child, db)
                 else:
-                    results = self._evaluate_lookup(child[0], child[1], negated, db)
+                    results = self.provider._evaluate_lookup(child[0], child[1], negated, db)
 
                 input_db = {**input_db, **results}
 
         return input_db
 
     def filter(self, criteria: Q, page: int = 1, per_page: int = 10, order_by: list = ()):
-        """ Read the repository and return results as per the filer"""
+        """Read the repository and return results as per the filer"""
 
         if criteria.children:
             items = list(self._filter(criteria, self.conn['data'][self.schema_name]).values())
@@ -183,7 +217,7 @@ class DictRepository(BaseRepository):
         return result
 
     def update(self, model_obj):
-        """ Update the entity record in the dictionary """
+        """Update the entity record in the dictionary """
         identifier = model_obj[self.entity_cls.meta_.id_field.field_name]
         with self.conn['lock']:
             # Check if object is present
@@ -196,7 +230,7 @@ class DictRepository(BaseRepository):
         return model_obj
 
     def update_all(self, criteria: Q, *args, **kwargs):
-        """ Update all objects satisfying the criteria """
+        """Update all objects satisfying the criteria """
         items = self._filter(criteria, self.conn['data'][self.schema_name])
 
         update_count = 0
@@ -211,7 +245,7 @@ class DictRepository(BaseRepository):
         return update_count
 
     def delete(self, model_obj):
-        """ Delete the entity record in the dictionary """
+        """Delete the entity record in the dictionary """
         identifier = model_obj[self.entity_cls.meta_.id_field.field_name]
         with self.conn['lock']:
             # Check if object is present
@@ -224,7 +258,7 @@ class DictRepository(BaseRepository):
         return model_obj
 
     def delete_all(self, criteria: Q = None):
-        """ Delete the dictionary object by its criteria"""
+        """Delete the dictionary object by its criteria"""
         if criteria:
             # Delete the object from the dictionary and return the deletion count
             items = self._filter(criteria, self.conn['data'][self.schema_name])
@@ -240,21 +274,28 @@ class DictRepository(BaseRepository):
                 if self.schema_name in self.conn['data']:
                     del self.conn['data'][self.schema_name]
 
-    def raw(self, query_string: str):
+    def raw(self, query: Any, data: Any = None):
         """Run raw query on Repository.
 
         For this stand-in repository, the query string is a json string that contains kwargs
         criteria with straigh-forward equality checks. Individual criteria are always ANDed
         and the result is always a subset of the full repository.
+
+        We will ignore the `data` parameter for this kind of repository.
         """
+        # Ensure that we are dealing with a string, for this repository
+        assert isinstance(query, str)
+
         input_db = self.conn['data'][self.schema_name]
         result = None
 
         try:
-            criteria = json.loads(query_string)
+            # Ensures that the string contains double quotes around keys and values
+            query = query.replace("'", "\"")
+            criteria = json.loads(query)
 
             for key, value in criteria.items():
-                input_db = self._evaluate_lookup(key, value, False, input_db)
+                input_db = self.provider._evaluate_lookup(key, value, False, input_db)
 
             items = list(input_db.values())
             result = Pagination(


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index f1e935f0..a3a732f0 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,12 +7,14 @@ os.environ['PROTEAN_CONFIG'] = 'tests.support.sample_config'
 
 
 def pytest_addoption(parser):
+    """Additional options for running tests with pytest"""
     parser.addoption(
         "--slow", action="store_true", default=False, help="run slow tests"
     )
 
 
 def pytest_collection_modifyitems(config, items):
+    """Configure special markers on tests, so as to control execution"""
     if config.getoption("--slow"):
         # --slow given in cli: do not skip slow tests
         return
diff --git a/tests/core/test_providers.py b/tests/core/test_providers.py
index 2d4db3ea..e22c0bf9 100644
--- a/tests/core/test_providers.py
+++ b/tests/core/test_providers.py
@@ -2,6 +2,9 @@
 from protean.core.provider import providers
 from protean.impl.repository.dict_repo import DictProvider
 
+from tests.support.dog import Dog, RelatedDog
+from tests.support.human import Human
+
 
 class TestProviders:
     """This class holds tests for Providers Singleton"""
@@ -21,3 +24,37 @@ class TestProviders:
 
         conn = providers.get_provider('default').get_connection()
         assert all(key in conn for key in ['data', 'lock', 'counters'])
+
+    def test_provider_raw(self):
+        """Test raw queries"""
+        Dog.create(name='Murdock', age=7, owner='John')
+        Dog.create(name='Jean', age=3, owner='John')
+        Dog.create(name='Bart', age=6, owner='Carrie')
+
+        # This is a raw query, so should bring results from all repositories
+        human = Human.create(first_name='John', last_name='Doe', email='john.doe@gmail.com')
+        RelatedDog.create(name='RelMurdock', age=2, owner=human)
+
+        provider = providers.get_provider('default')
+
+        # Filter by Dog attributes
+        results = provider.raw('{"owner":"John"}')
+        assert isinstance(results, list)
+        assert len(results) == 2
+
+        # Try with single quotes in JSON String
+        results = provider.raw("{'owner':'John'}")
+        assert len(results) == 2
+
+        results = provider.raw('{"owner":"John", "age":3}')
+        assert len(results) == 1
+
+        # This query brings results from multiple repositories
+        results = provider.raw('{"age__in":[2, 3]}')
+        assert len(results) == 2
+
+        results = provider.raw('{"owner":"John", "age__in":[6,7]}')
+        assert len(results) == 1
+
+        results = provider.raw('{"owner":"John", "age__in":[2, 3,7]}')
+        assert len(results) == 2