Repository: ckan/ckan

Issue:
Datastore depends on english localisation 
There are many checks where we rely on english error messages in the datastore.

See https://github.com/okfn/ckan/blob/master/ckanext/datastore/db.py

It is probably a good idea to use the [error codes](http://www.postgresql.org/docs/9.1/static/errcodes-appendix.html) instead of error message even though one error code seems to subsume a number of possible errors.



Discussion:

Resolving patch:
diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -11,7 +11,7 @@
 import logging
 import pprint
 import sqlalchemy
-from sqlalchemy.exc import ProgrammingError, IntegrityError
+from sqlalchemy.exc import ProgrammingError, IntegrityError, DBAPIError
 import psycopg2.extras
 
 log = logging.getLogger(__name__)
@@ -31,19 +31,27 @@ def __init__(self, error_dict):
 _type_names = set()
 _engines = {}
 
+# See http://www.postgresql.org/docs/9.2/static/errcodes-appendix.html
+_PG_ERR_CODE = {
+    'unique_violation': 23505,
+    'query_canceled': 57014,
+    'undefined_object': 42704,
+    'syntax_error': 42601
+}
+
 _date_formats = ['%Y-%m-%d',
-                '%Y-%m-%d %H:%M:%S',
-                '%Y-%m-%dT%H:%M:%S',
-                '%Y-%m-%dT%H:%M:%SZ',
-                '%d/%m/%Y',
-                '%m/%d/%Y',
-                '%d-%m-%Y',
-                '%m-%d-%Y',
-                ]
-INSERT = 'insert'
-UPSERT = 'upsert'
-UPDATE = 'update'
-_methods = [INSERT, UPSERT, UPDATE]
+                 '%Y-%m-%d %H:%M:%S',
+                 '%Y-%m-%dT%H:%M:%S',
+                 '%Y-%m-%dT%H:%M:%SZ',
+                 '%d/%m/%Y',
+                 '%m/%d/%Y',
+                 '%d-%m-%Y',
+                 '%m-%d-%Y',
+                 ]
+
+_INSERT = 'insert'
+_UPSERT = 'upsert'
+_UPDATE = 'update'
 
 
 def _strip(input):
@@ -57,7 +65,7 @@ def _pluck(field, arr):
 
 
 def _get_list(input, strip=True):
-    """Transforms a string or list to a list"""
+    '''Transforms a string or list to a list'''
     if input is None:
         return
     if input == '':
@@ -106,7 +114,7 @@ def _validate_int(i, field_name, non_negative=False):
 
 
 def _get_engine(context, data_dict):
-    'Get either read or write engine.'
+    '''Get either read or write engine.'''
     connection_url = data_dict['connection_url']
     engine = _engines.get(connection_url)
 
@@ -159,10 +167,10 @@ def _is_valid_pg_type(context, type_name):
         try:
             connection.execute('SELECT %s::regtype', type_name)
         except ProgrammingError, e:
-            if 'invalid type name' in str(e) or 'does not exist' in str(e):
+            if int(e.orig.pgcode) in [_PG_ERR_CODE['undefined_object'],
+                                      _PG_ERR_CODE['syntax_error']]:
                 return False
-            else:
-                raise
+            raise
         else:
             return True
 
@@ -173,10 +181,8 @@ def _get_type(context, oid):
 
 
 def _rename_json_field(data_dict):
-    '''
-    rename json type to a corresponding type for the datastore since
-    pre 9.2 postgres versions do not support native json
-    '''
+    '''Rename json type to a corresponding type for the datastore since
+    pre 9.2 postgres versions do not support native json'''
     return _rename_field(data_dict, 'json', 'nested')
 
 
@@ -193,7 +199,8 @@ def _rename_field(data_dict, term, replace):
 
 
 def _guess_type(field):
-    'Simple guess type of field, only allowed are integer, numeric and text'
+    '''Simple guess type of field, only allowed are
+    integer, numeric and text'''
     data_types = set([int, float])
     if isinstance(field, (dict, list)):
         return 'nested'
@@ -252,7 +259,7 @@ def json_get_values(obj, current_list=None):
 
 
 def check_fields(context, fields):
-    'Check if field types are valid.'
+    '''Check if field types are valid.'''
     for field in fields:
         if field.get('type') and not _is_valid_pg_type(context, field['type']):
             raise ValidationError({
@@ -281,7 +288,7 @@ def convert(data, type_name):
 
 
 def create_table(context, data_dict):
-    'Create table from combination of fields and first row of data.'
+    '''Create table from combination of fields and first row of data.'''
 
     datastore_fields = [
         {'id': '_id', 'type': 'serial primary key'},
@@ -331,7 +338,7 @@ def create_table(context, data_dict):
 
 
 def _get_aliases(context, data_dict):
-    ''' Get a list of aliases for a resource. '''
+    '''Get a list of aliases for a resource.'''
     res_id = data_dict['resource_id']
     alias_sql = sqlalchemy.text(
         u'SELECT name FROM "_table_metadata" WHERE alias_of = :id')
@@ -340,8 +347,8 @@ def _get_aliases(context, data_dict):
 
 
 def _get_resources(context, alias):
-    ''' Get a list of resources for an alias. There could be more than one alias
-    in a resource_dict. '''
+    '''Get a list of resources for an alias. There could be more than one alias
+    in a resource_dict.'''
     alias_sql = sqlalchemy.text(
         u'''SELECT alias_of FROM "_table_metadata"
         WHERE name = :alias AND alias_of IS NOT NULL''')
@@ -521,7 +528,7 @@ def alter_table(context, data_dict):
 
 
 def insert_data(context, data_dict):
-    data_dict['method'] = INSERT
+    data_dict['method'] = _INSERT
     return upsert_data(context, data_dict)
 
 
@@ -530,9 +537,9 @@ def upsert_data(context, data_dict):
     if not data_dict.get('records'):
         return
 
-    method = data_dict.get('method', UPSERT)
+    method = data_dict.get('method', _UPSERT)
 
-    if method not in _methods:
+    if method not in [_INSERT, _UPSERT, _UPDATE]:
         raise ValidationError({
             'method': [u'"{0}" is not defined'.format(method)]
         })
@@ -543,7 +550,7 @@ def upsert_data(context, data_dict):
     sql_columns = ", ".join(['"%s"' % name.replace('%', '%%') for name in field_names]
                             + ['"_full_text"'])
 
-    if method == INSERT:
+    if method == _INSERT:
         rows = []
         for num, record in enumerate(records):
             _validate_record(record, num, field_names)
@@ -566,7 +573,7 @@ def upsert_data(context, data_dict):
 
         context['connection'].execute(sql_string, rows)
 
-    elif method in [UPDATE, UPSERT]:
+    elif method in [_UPDATE, _UPSERT]:
         unique_keys = _get_unique_key(context, data_dict)
         if len(unique_keys) < 1:
             raise ValidationError({
@@ -608,7 +615,7 @@ def upsert_data(context, data_dict):
 
             full_text = _to_full_text(fields, record)
 
-            if method == UPDATE:
+            if method == _UPDATE:
                 sql_string = u'''
                     UPDATE "{res_id}"
                     SET ({columns}, "_full_text") = ({values}, to_tsvector(%s))
@@ -629,7 +636,7 @@ def upsert_data(context, data_dict):
                         'key': [u'key "{0}" not found'.format(unique_values)]
                     })
 
-            elif method == UPSERT:
+            elif method == _UPSERT:
                 sql_string = u'''
                     UPDATE "{res_id}"
                     SET ({columns}, "_full_text") = ({values}, to_tsvector(%s))
@@ -700,7 +707,7 @@ def _to_full_text(fields, record):
 
 
 def _where(field_ids, data_dict):
-    'Return a SQL WHERE clause from data_dict filters and q'
+    '''Return a SQL WHERE clause from data_dict filters and q'''
     filters = data_dict.get('filters', {})
 
     if not isinstance(filters, dict):
@@ -786,9 +793,8 @@ def _sort(context, data_dict, field_ids):
 
 
 def _insert_links(data_dict, limit, offset):
-    ''' Adds link to the next/prev part (same limit, offset=offset+limit)
-    and the resource page.
-    '''
+    '''Adds link to the next/prev part (same limit, offset=offset+limit)
+    and the resource page.'''
     data_dict['_links'] = {}
 
     # get the url from the request
@@ -963,23 +969,24 @@ def create(context, data_dict):
         trans.commit()
         return _unrename_json_field(data_dict)
     except IntegrityError, e:
-        if ('duplicate key value violates unique constraint' in str(e)
-                or 'could not create unique index' in str(e)):
+        if int(e.orig.pgcode) == _PG_ERR_CODE['unique_violation']:
             raise ValidationError({
-                'constraints': ['Cannot insert records or create index because of uniqueness constraint'],
+                'constraints': ['Cannot insert records or create index because '
+                                'of uniqueness constraint'],
                 'info': {
                     'details': str(e)
                 }
             })
-        else:
-            raise
-    except Exception, e:
-        trans.rollback()
-        if 'due to statement timeout' in str(e):
+        raise
+    except DBAPIError, e:
+        if int(e.orig.pgcode) == _PG_ERR_CODE['query_canceled']:
             raise ValidationError({
                 'query': ['Query took too long']
             })
         raise
+    except Exception, e:
+        trans.rollback()
+        raise
     finally:
         context['connection'].close()
 
@@ -1005,22 +1012,24 @@ def upsert(context, data_dict):
         trans.commit()
         return _unrename_json_field(data_dict)
     except IntegrityError, e:
-        if 'duplicate key value violates unique constraint' in str(e):
+        if int(e.orig.pgcode) == _PG_ERR_CODE['unique_violation']:
             raise ValidationError({
-                'constraints': ['Cannot insert records because of uniqueness constraint'],
+                'constraints': ['Cannot insert records or create index because '
+                                'of uniqueness constraint'],
                 'info': {
                     'details': str(e)
                 }
             })
-        else:
-            raise
-    except Exception, e:
-        trans.rollback()
-        if 'due to statement timeout' in str(e):
+        raise
+    except DBAPIError, e:
+        if int(e.orig.pgcode) == _PG_ERR_CODE['query_canceled']:
             raise ValidationError({
                 'query': ['Query took too long']
             })
         raise
+    except Exception, e:
+        trans.rollback()
+        raise
     finally:
         context['connection'].close()
 
@@ -1079,8 +1088,8 @@ def search(context, data_dict):
                     data_dict['resource_id'])]
             })
         return search_data(context, data_dict)
-    except Exception, e:
-        if 'due to statement timeout' in str(e):
+    except DBAPIError, e:
+        if int(e.orig.pgcode) == _PG_ERR_CODE['query_canceled']:
             raise ValidationError({
                 'query': ['Search took too long']
             })
@@ -1112,10 +1121,10 @@ def search_sql(context, data_dict):
             'orig': [str(e.orig)]
          }
         })
-    except Exception, e:
-        if 'due to statement timeout' in str(e):
+    except DBAPIError, e:
+        if int(e.orig.pgcode) == _PG_ERR_CODE['query_canceled']:
             raise ValidationError({
-                'query': ['Search took too long']
+                'query': ['Query took too long']
             })
         raise
     finally:
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -1,6 +1,5 @@
 import logging
 import pylons
-from sqlalchemy.exc import ProgrammingError
 
 import ckan.plugins as p
 import ckanext.datastore.logic.action as action
@@ -18,9 +17,6 @@ class DatastoreException(Exception):
 
 
 class DatastorePlugin(p.SingletonPlugin):
-    '''
-    Datastore plugin.
-    '''
     p.implements(p.IConfigurable, inherit=True)
     p.implements(p.IActions)
     p.implements(p.IAuthFunctions)
@@ -51,27 +47,25 @@ def configure(self, config):
         self.write_url = self.config['ckan.datastore.write_url']
         if self.legacy_mode:
             self.read_url = self.write_url
+            log.warn('Legacy mode active. '
+                     'The sql search will not be available.')
         else:
             self.read_url = self.config['ckan.datastore.read_url']
 
-        if model.engine_is_pg():
-            if not self._is_read_only_database():
-                # Make sure that the right permissions are set
-                # so that no harmful queries can be made
-                if not ('debug' in config and config['debug']):
-                    self._check_separate_db()
-                if self.legacy_mode:
-                    log.warn('Legacy mode active. The sql search will not be available.')
-                else:
-                    self._check_read_permissions()
+        if not model.engine_is_pg():
+            log.warn('We detected that you do not use a PostgreSQL '
+                     'database. The DataStore will NOT work and DataStore '
+                     'tests will be skipped.')
+            return
 
-                self._create_alias_table()
-            else:
-                log.warn("We detected that CKAN is running on a read only database. "
-                    "Permission checks and the creation of _table_metadata are skipped.")
+        if self._is_read_only_database():
+            log.warn('We detected that CKAN is running on a read '
+                     'only database. Permission checks and the creation '
+                     'of _table_metadata are skipped.')
         else:
-            log.warn("We detected that you do not use a PostgreSQL database. "
-                    "The DataStore will NOT work and datastore tests will be skipped.")
+            self._check_urls_and_permissions()
+
+            self._create_alias_table()
 
         ## Do light wrapping around action function to add datastore_active
         ## to resource dict.  Not using IAction extension as this prevents
@@ -106,79 +100,77 @@ def new_resource_show(context, data_dict):
             new_resource_show._datastore_wrapped = True
             logic._actions['resource_show'] = new_resource_show
 
+    def _log_or_raise(self, message):
+        if self.config.get('debug'):
+            log.critical(message)
+        else:
+            raise DatastoreException(message)
+
+    def _check_urls_and_permissions(self):
+        # Make sure that the right permissions are set
+        # so that no harmful queries can be made
+
+        if self._same_ckan_and_datastore_db():
+            self._log_or_raise('CKAN and DataStore database '
+                               'cannot be the same.')
+
+        # in legacy mode, the read and write url are ths same (both write url)
+        # consequently the same url check and and write privilege check
+        # don't make sense
+        if not self.legacy_mode:
+            if self._same_read_and_write_url():
+                self._log_or_raise('The write and read-only database '
+                                   'connection urls are the same.')
+
+            if not self._read_connection_has_correct_privileges():
+                self._log_or_raise('The read-only user has write privileges.')
+
     def _is_read_only_database(self):
+        ''' Returns True if no connection has CREATE privileges on the public
+        schema. This is the case if replication is enabled.'''
         for url in [self.ckan_url, self.write_url, self.read_url]:
             connection = db._get_engine(None,
-                {'connection_url': url}).connect()
-            trans = connection.begin()
-            try:
-                sql = u"CREATE TABLE test_readonly(id INTEGER);"
-                connection.execute(sql)
-            except ProgrammingError, e:
-                if 'permission denied' in str(e) or 'read-only transaction' in str(e):
-                    pass
-                else:
-                    raise
-            else:
+                                        {'connection_url': url}).connect()
+            sql = u"SELECT has_schema_privilege('public', 'CREATE')"
+            is_writable = connection.execute(sql).first()[0]
+            if is_writable:
                 return False
-            finally:
-                trans.rollback()
         return True
 
-    def _check_separate_db(self):
-        '''
-        Make sure the datastore is on a separate db. Otherwise one could access
-        all internal tables via the api.
-        '''
-
-        if not self.legacy_mode:
-            if self.write_url == self.read_url:
-                raise Exception("The write and read-only database connection url are the same.")
-
-        if self._get_db_from_url(self.ckan_url) == self._get_db_from_url(self.read_url):
-            raise Exception("The CKAN and datastore database are the same.")
+    def _same_ckan_and_datastore_db(self):
+        '''Returns True if the CKAN and DataStore db are the same'''
+        return self._get_db_from_url(self.ckan_url) == self._get_db_from_url(self.read_url)
 
     def _get_db_from_url(self, url):
         return url[url.rindex("@"):]
 
-    def _check_read_permissions(self):
-        '''
-        Check whether the right permissions are set for the read only user.
+    def _same_read_and_write_url(self):
+        return self.write_url == self.read_url
+
+    def _read_connection_has_correct_privileges(self):
+        ''' Returns True if the right permissions are set for the read only user.
         A table is created by the write user to test the read only user.
         '''
         write_connection = db._get_engine(None,
             {'connection_url': self.write_url}).connect()
-        write_connection.execute(u"DROP TABLE IF EXISTS public._foo;"
-            u"CREATE TABLE public._foo (id INTEGER, name VARCHAR)")
-
         read_connection = db._get_engine(None,
             {'connection_url': self.read_url}).connect()
 
-        statements = [
-            u"CREATE TABLE public._bar (id INTEGER, name VARCHAR)",
-            u"INSERT INTO public._foo VALUES (1, 'okfn')"
-        ]
+        drop_foo_sql = u'DROP TABLE IF EXISTS _foo'
+
+        write_connection.execute(drop_foo_sql)
 
         try:
-            for sql in statements:
-                read_trans = read_connection.begin()
-                try:
-                    read_connection.execute(sql)
-                except ProgrammingError, e:
-                    if 'permission denied' not in str(e):
-                        raise
-                else:
-                    log.info("Connection url {0}".format(self.read_url))
-                    if 'debug' in self.config and self.config['debug']:
-                        log.critical("We have write permissions on the read-only database.")
-                    else:
-                        raise Exception("We have write permissions on the read-only database.")
-                finally:
-                    read_trans.rollback()
-        except Exception:
-            raise
+            write_connection.execute(u'CREATE TABLE _foo ()')
+            for privilege in ['INSERT', 'UPDATE', 'DELETE']:
+                test_privilege_sql = u"SELECT has_table_privilege('_foo', '{privilege}')"
+                sql = test_privilege_sql.format(privilege=privilege)
+                have_privilege = read_connection.execute(sql).first()[0]
+                if have_privilege:
+                    return False
         finally:
-            write_connection.execute("DROP TABLE _foo")
+            write_connection.execute(drop_foo_sql)
+        return True
 
     def _create_alias_table(self):
         mapping_sql = '''


Test patch:
diff --git a/ckanext/datastore/tests/test_configure.py b/ckanext/datastore/tests/test_configure.py
--- a/ckanext/datastore/tests/test_configure.py
+++ b/ckanext/datastore/tests/test_configure.py
@@ -1,17 +1,25 @@
 import unittest
+import nose.tools
+import pyutilib.component.core
 
+import ckan.plugins
 import ckanext.datastore.plugin as plugin
 
 
-class TestTypeGetters(unittest.TestCase):
+class TestConfiguration(unittest.TestCase):
     def setUp(self):
-        self.p = plugin.DatastorePlugin()
+        self._original_plugin = ckan.plugins.unload('datastore')
+        pyutilib.component.core.PluginGlobals.singleton_services()[plugin.DatastorePlugin] = True
+        self.p = pyutilib.component.core.PluginGlobals.singleton_services()[plugin.DatastorePlugin] = ckan.plugins.load('datastore')
+
+    def tearDown(self):
+        ckan.plugins.unload('datastore')
+        pyutilib.component.core.PluginGlobals.singleton_services()[plugin.DatastorePlugin] = self._original_plugin
 
     def test_legacy_mode_default(self):
         assert not self.p.legacy_mode
 
     def test_set_legacy_mode(self):
-        assert not self.p.legacy_mode
         c = {
             'sqlalchemy.url': 'bar',
             'ckan.datastore.write_url': 'foo'
@@ -24,16 +32,87 @@ def test_set_legacy_mode(self):
         assert self.p.write_url == 'foo'
         assert self.p.read_url == 'foo'
 
-    def test_check_separate_db(self):
-        self.p.write_url = 'postgresql://u:pass@localhost/dt'
+    def test_check_separate_write_and_read_url(self):
+        self.p.write_url = 'postgresql://u:pass@localhost/ds'
+        self.p.read_url = 'postgresql://u:pass@localhost/ds'
+        assert self.p._same_read_and_write_url()
+
+        self.p.write_url = 'postgresql://u:pass@localhost/ds'
+        self.p.read_url = 'postgresql://u2:pass@localhost/ds'
+        assert not self.p._same_read_and_write_url()
+
+    def test_same_ckan_and_datastore_db(self):
+        self.p.read_url = 'postgresql://u2:pass@localhost/ckan'
+        self.p.ckan_url = 'postgresql://u:pass@localhost/ckan'
+        assert self.p._same_ckan_and_datastore_db()
+
         self.p.read_url = 'postgresql://u:pass@localhost/dt'
         self.p.ckan_url = 'postgresql://u:pass@localhost/ckan'
+        assert not self.p._same_ckan_and_datastore_db()
+
+    def test_setup_plugin_for_check_urls_and_permissions_tests_should_leave_the_plugin_in_a_valid_state(self):
+        self.setUp_plugin_for_check_urls_and_permissions_tests()
+        self.p._check_urls_and_permissions()  # Should be OK
+
+    def test_check_urls_and_permissions_requires_different_ckan_and_datastore_dbs(self):
+        self.setUp_plugin_for_check_urls_and_permissions_tests()
 
+        self.p._same_ckan_and_datastore_db = lambda: False
+        self.p._check_urls_and_permissions()  # Should be OK
+
+        self.p._same_ckan_and_datastore_db = lambda: True
+        nose.tools.assert_raises(InvalidUrlsOrPermissionsException, self.p._check_urls_and_permissions)
+
+    def test_check_urls_and_permissions_requires_different_read_and_write_urls_when_not_in_legacy_mode(self):
+        self.setUp_plugin_for_check_urls_and_permissions_tests()
+        self.p.legacy_mode = False
+
+        self.p._same_read_and_write_url = lambda: False
+        self.p._check_urls_and_permissions()  # Should be OK
+
+        self.p._same_read_and_write_url = lambda: True
+        nose.tools.assert_raises(InvalidUrlsOrPermissionsException, self.p._check_urls_and_permissions)
+
+    def test_check_urls_and_permissions_doesnt_require_different_read_and_write_urls_when_in_legacy_mode(self):
+        self.setUp_plugin_for_check_urls_and_permissions_tests()
         self.p.legacy_mode = True
-        try:
-            self.p._check_separate_db()
-        except Exception:
-            self.fail("_check_separate_db raise Exception unexpectedly!")
 
+        self.p._same_read_and_write_url = lambda: False
+        self.p._check_urls_and_permissions()  # Should be OK
+
+        self.p._same_read_and_write_url = lambda: True
+        self.p._check_urls_and_permissions()  # Should be OK
+
+    def test_check_urls_and_permissions_requires_read_connection_with_correct_privileges_when_not_in_legacy_mode(self):
+        self.setUp_plugin_for_check_urls_and_permissions_tests()
         self.p.legacy_mode = False
-        self.assertRaises(Exception, self.p._check_separate_db)
+
+        self.p._read_connection_has_correct_privileges = lambda: True
+        self.p._check_urls_and_permissions()  # Should be OK
+
+        self.p._read_connection_has_correct_privileges = lambda: False
+        nose.tools.assert_raises(InvalidUrlsOrPermissionsException, self.p._check_urls_and_permissions)
+
+    def test_check_urls_and_permissions_doesnt_care_about_read_connection_privileges_when_in_legacy_mode(self):
+        self.setUp_plugin_for_check_urls_and_permissions_tests()
+        self.p.legacy_mode = True
+
+        self.p._read_connection_has_correct_privileges = lambda: True
+        self.p._check_urls_and_permissions()  # Should be OK
+
+        self.p._read_connection_has_correct_privileges = lambda: False
+        self.p._check_urls_and_permissions()  # Should be OK
+
+    def setUp_plugin_for_check_urls_and_permissions_tests(self):
+        def _raise_invalid_urls_or_permissions_exception(message):
+            raise InvalidUrlsOrPermissionsException(message)
+
+        self.p._same_ckan_and_datastore_db = lambda: False
+        self.p.legacy_mode = True
+        self.p._same_read_and_write_url = lambda: False
+        self.p._read_connection_has_correct_privileges = lambda: True
+        self.p._log_or_raise = _raise_invalid_urls_or_permissions_exception
+
+
+class InvalidUrlsOrPermissionsException(Exception):
+    pass