Repository: ckan/ckan

Issue:
Remove old paster CLI commands
Once #5108 and #5109 are done we can remove the old paster commands in `ckan/lib/cli.py`.

At least for CKAN 2.9 let's remove the actual commands only and keep the `CkanCommand` class as this is present in the toolkit and needed by extensions to implement paster-based commands. Also utilities like `load_config` as they might be imported from extensions.

There's is some click-based stuff in there which I don't know if it's still relevant (`click_config_option`, `paster_click_group`...). If no longer needed it can be removed.


Discussion:
I would like to take this one.
@EvgeniiaVak great! Let us know if you need any pointers
@amercader  
Yes, some pointers would be helpful. Although I don't know what exactly to ask for. As I understand, there was paster, that was used for everything, now flask is used for creating the server, click - for cli commands.

I can also wait for https://github.com/ckan/ckan/issues/5109 and continue to lean how ckan in general works in the mean time.

I hope this feature is not very urgent, because I don't think I can complete it very soon.
@EvgeniiaVak not a problem and sorry for not coming back to you sooner.

The old paster commands are defined in `ckan/lib/cli.py`. Their new equivalents are written in Click and located in `ckan/cli/cli.py`.

The next version of CKAN will only use the new Click based CLI so we want to remove as much as possible from the old paster ones to avoid confusion. But we can't delete the whole ckan/lib/cli.py module because extensions rely on some classes and methods from that module to write their own commands, and we want to give them time to migrate.

So here's how we could approach it:

- [ ] Remove all `XYZCommand` classes from ckan/lib/cli.py, except the main `CkanCommand` one.
- [ ] Keep the `load_config` and `query_yes_no` methods as these are used by extensions, all the rest of methods and unused imports can be removed
- [ ] Check that the new CLI works fine (`ckan -h`) and that the old CLI still works on a command provided by an extension, eg from ckanext-harvest `paster --plugin=ckanext-harvest harvester sources -c /path/to/ini` (it can be any command from any extension)

Let me know if this helps

@amercader 
Yes, this clarifies things very much, thank you!

Resolving patch:
diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -2,43 +2,24 @@
 
 from __future__ import print_function
 
-import collections
-import csv
-import multiprocessing as mp
 import os
-import datetime
 import sys
-from pprint import pprint
-import re
-import itertools
-import json
-import logging
-from optparse import OptionConflictError
-import traceback
 
-from six import text_type
-from six.moves import input, xrange
-from six.moves.urllib.error import HTTPError
-from six.moves.urllib.parse import urljoin, urlparse
-from six.moves.urllib.request import urlopen
-
-import sqlalchemy as sa
-
-import routes
+import click
 import paste.script
+import routes
 from paste.registry import Registry
 from paste.script.util.logging_config import fileConfig
-import click
-from ckan.cli import load_config as _get_config
+from six.moves import input
+from six.moves.urllib.parse import urlparse
 
 from ckan.config.middleware import make_app
+from ckan.cli import load_config as _get_config
 import ckan.logic as logic
 import ckan.model as model
-import ckan.include.rjsmin as rjsmin
-import ckan.include.rcssmin as rcssmin
-import ckan.plugins as p
 from ckan.common import config
 from ckan.common import asbool
+import ckan.lib.maintain as maintain
 # This is a test Flask request context to be used internally.
 # Do not use it!
 _cli_test_request_context = None
@@ -49,8 +30,11 @@
 #    Otherwise loggers get disabled.
 
 
+@maintain.deprecated('Use @maintain.deprecated instead')
 def deprecation_warning(message=None):
     '''
+    DEPRECATED
+
     Print a deprecation warning to STDERR.
 
     If ``message`` is given it is also printed to STDERR.
@@ -61,8 +45,11 @@ def deprecation_warning(message=None):
     sys.stderr.write(u'\n')
 
 
+@maintain.deprecated()
 def error(msg):
     '''
+    DEPRECATED
+
     Print an error message to STDOUT and exit with return code 1.
     '''
     sys.stderr.write(msg)
@@ -71,7 +58,9 @@ def error(msg):
     sys.exit(1)
 
 
+@maintain.deprecated('Use model.parse_db_config directly instead')
 def _parse_db_config(config_key=u'sqlalchemy.url'):
+    '''Deprecated'''
     db_config = model.parse_db_config(config_key)
     if not db_config:
         raise Exception(
@@ -79,67 +68,13 @@ def _parse_db_config(config_key=u'sqlalchemy.url'):
         )
     return db_config
 
-
-def user_add(args):
-    '''Add new user if we use paster sysadmin add
-    or paster user add
-    '''
-    if len(args) < 1:
-        error('Error: you need to specify the user name.')
-    username = args[0]
-
-    # parse args into data_dict
-    data_dict = {'name': username}
-    for arg in args[1:]:
-        try:
-            field, value = arg.split('=', 1)
-            if field == 'sysadmin':
-                value = asbool(value)
-            data_dict[field] = value
-        except ValueError:
-            raise ValueError(
-                'Could not parse arg: %r (expected "<option>=<value>)"' % arg
-            )
-
-    # Required
-    while '@' not in data_dict.get('email', ''):
-        print('Error: Invalid email address')
-        data_dict['email'] = input('Email address: ').strip()
-
-    if 'password' not in data_dict:
-        data_dict['password'] = UserCmd.password_prompt()
-
-    # Optional
-    if 'fullname' in data_dict:
-        data_dict['fullname'] = data_dict['fullname'].decode(
-            sys.getfilesystemencoding()
-        )
-
-    print('Creating user: %r' % username)
-
-    try:
-        import ckan.logic as logic
-        import ckan.model as model
-        site_user = logic.get_action('get_site_user')({
-            'model': model,
-            'ignore_auth': True},
-            {}
-        )
-        context = {
-            'model': model,
-            'session': model.Session,
-            'ignore_auth': True,
-            'user': site_user['name'],
-        }
-        user_dict = logic.get_action('user_create')(context, data_dict)
-        pprint(user_dict)
-    except logic.ValidationError as e:
-        error(traceback.format_exc())
-
 ## from http://code.activestate.com/recipes/577058/ MIT licence.
 ## Written by Trent Mick
+@maintain.deprecated('Instead you can probably use click.confirm()')
 def query_yes_no(question, default="yes"):
-    """Ask a yes/no question via input() and return their answer.
+    """DEPRECATED
+
+    Ask a yes/no question via input() and return their answer.
 
     "question" is a string that is presented to the user.
     "default" is the presumed answer if the user just hits <Enter>.
@@ -228,8 +163,12 @@ def load_config(config, load_site_user=True):
     return site_user
 
 
+@maintain.deprecated('Instead use ckan.cli.cli.CkanCommand or extensions '
+                     'should use IClick')
 def paster_click_group(summary):
-    '''Return a paster command click.Group for paster subcommands
+    '''DEPRECATED
+
+    Return a paster command click.Group for paster subcommands
 
     :param command: the paster command linked to this function from
         setup.py, used in help text (e.g. "datastore")
@@ -271,7 +210,11 @@ def cli(ctx, plugin, config):
 
 
 class CkanCommand(paste.script.command.Command):
-    '''Base class for classes that implement CKAN paster commands to inherit.'''
+    '''DEPRECATED - Instead use ckan.cli.cli.CkanCommand or extensions
+    should use IClick.
+
+    Base class for classes that implement CKAN paster commands to
+    inherit.'''
     parser = paste.script.command.Command.standard_parser(verbose=True)
     parser.add_option('-c', '--config', dest='config',
                       help='Config file to use.')
@@ -284,2209 +227,3 @@ class CkanCommand(paste.script.command.Command):
 
     def _load_config(self, load_site_user=True):
         self.site_user = load_config(self.options.config, load_site_user)
-
-
-class ManageDb(CkanCommand):
-    '''Perform various tasks on the database.
-
-    db create                      - alias of db upgrade
-    db init                        - create and put in default data
-    db clean                       - clears db (including dropping tables) and
-                                     search index
-    db upgrade [version no.]       - Data migrate
-    db version                     - returns current version of data schema
-    db create-from-model           - create database from the model (indexes not made)
-    db migrate-filestore           - migrate all uploaded data from the 2.1 filesore.
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = None
-    min_args = 1
-
-    def command(self):
-        cmd = self.args[0]
-
-        self._load_config(cmd != 'upgrade')
-
-        import ckan.model as model
-        import ckan.lib.search as search
-
-        if cmd == 'init':
-
-            model.repo.init_db()
-            if self.verbose:
-                print('Initialising DB: SUCCESS')
-        elif cmd == 'clean' or cmd == 'drop':
-
-            # remove any *.pyc version files to prevent conflicts
-            v_path = os.path.join(os.path.dirname(__file__),
-                                  '..', 'migration', 'versions', '*.pyc')
-            import glob
-            filelist = glob.glob(v_path)
-            for f in filelist:
-                os.remove(f)
-
-            model.repo.clean_db()
-            search.clear_all()
-            if self.verbose:
-                print('Cleaning DB: SUCCESS')
-        elif cmd == 'upgrade':
-            model.repo.upgrade_db(*self.args[1:])
-        elif cmd == 'downgrade':
-            model.repo.downgrade_db(*self.args[1:])
-        elif cmd == 'version':
-            self.version()
-        elif cmd == 'create-from-model':
-            model.repo.create_db()
-            if self.verbose:
-                print('Creating DB: SUCCESS')
-        else:
-            error('Command %s not recognized' % cmd)
-
-    def version(self):
-        from ckan.model import Session
-        print(Session.execute('select version from '
-                              'migrate_version;').fetchall())
-
-
-class SearchIndexCommand(CkanCommand):
-    '''Creates a search index for all datasets
-
-    Usage:
-      search-index [-i] [-o] [-r] [-e] [-q] rebuild [dataset_name]  - reindex dataset_name if given, if not then rebuild
-                                                                    full search index (all datasets)
-      search-index rebuild_fast                                     - reindex using multiprocessing using all cores.
-                                                                    This acts in the same way as rubuild -r [EXPERIMENTAL]
-      search-index check                                            - checks for datasets not indexed
-      search-index show DATASET_NAME                                - shows index of a dataset
-      search-index clear [dataset_name]                             - clears the search index for the provided dataset or
-                                                                    for the whole ckan instance
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 2
-    min_args = 0
-
-    def __init__(self, name):
-        super(SearchIndexCommand, self).__init__(name)
-
-        self.parser.add_option('-i', '--force', dest='force',
-                               action='store_true', default=False,
-                               help='Ignore exceptions when rebuilding the index')
-
-        self.parser.add_option('-o', '--only-missing', dest='only_missing',
-                               action='store_true', default=False,
-                               help='Index non indexed datasets only')
-
-        self.parser.add_option('-r', '--refresh', dest='refresh',
-                               action='store_true', default=False,
-                               help='Refresh current index (does not clear the existing one)')
-
-        self.parser.add_option('-q', '--quiet', dest='quiet',
-                               action='store_true', default=False,
-                               help='Do not output index rebuild progress')
-
-        self.parser.add_option('-e', '--commit-each', dest='commit_each',
-                               action='store_true', default=False, help=
-'''Perform a commit after indexing each dataset. This ensures that changes are
-immediately available on the search, but slows significantly the process.
-Default is false.''')
-
-    def command(self):
-        if not self.args:
-            # default to printing help
-            print(self.usage)
-            return
-
-        cmd = self.args[0]
-        # Do not run load_config yet
-        if cmd == 'rebuild_fast':
-            self.rebuild_fast()
-            return
-
-        self._load_config()
-        if cmd == 'rebuild':
-            self.rebuild()
-        elif cmd == 'check':
-            self.check()
-        elif cmd == 'show':
-            self.show()
-        elif cmd == 'clear':
-            self.clear()
-        else:
-            print('Command %s not recognized' % cmd)
-
-    def rebuild(self):
-        from ckan.lib.search import rebuild, commit
-
-        # BY default we don't commit after each request to Solr, as it is
-        # a really heavy operation and slows things a lot
-
-        if len(self.args) > 1:
-            rebuild(self.args[1])
-        else:
-            rebuild(only_missing=self.options.only_missing,
-                    force=self.options.force,
-                    refresh=self.options.refresh,
-                    defer_commit=(not self.options.commit_each),
-                    quiet=self.options.quiet)
-
-        if not self.options.commit_each:
-            commit()
-
-    def check(self):
-        from ckan.lib.search import check
-        check()
-
-    def show(self):
-        from ckan.lib.search import show
-
-        if not len(self.args) == 2:
-            print('Missing parameter: dataset-name')
-            return
-        index = show(self.args[1])
-        pprint(index)
-
-    def clear(self):
-        from ckan.lib.search import clear, clear_all
-        package_id = self.args[1] if len(self.args) > 1 else None
-        if not package_id:
-            clear_all()
-        else:
-            clear(package_id)
-
-    def rebuild_fast(self):
-        ###  Get out config but without starting pylons environment ####
-        conf = _get_config()
-
-        ### Get ids using own engine, otherwise multiprocess will balk
-        db_url = conf['sqlalchemy.url']
-        engine = sa.create_engine(db_url)
-        package_ids = []
-        result = engine.execute("select id from package where state = 'active';")
-        for row in result:
-            package_ids.append(row[0])
-
-        def start(ids):
-            ## load actual enviroment for each subprocess, so each have thier own
-            ## sa session
-            self._load_config()
-            from ckan.lib.search import rebuild, commit
-            rebuild(package_ids=ids)
-            commit()
-
-        def chunks(l, n):
-            """ Yield n successive chunks from l.
-            """
-            newn = int(len(l) / n)
-            for i in xrange(0, n-1):
-                yield l[i*newn:i*newn+newn]
-            yield l[n*newn-newn:]
-
-        processes = []
-        for chunk in chunks(package_ids, mp.cpu_count()):
-            process = mp.Process(target=start, args=(chunk,))
-            processes.append(process)
-            process.daemon = True
-            process.start()
-
-        for process in processes:
-            process.join()
-
-
-class Notification(CkanCommand):
-    '''Send out modification notifications.
-
-    In "replay" mode, an update signal is sent for each dataset in the database.
-
-    Usage:
-      notify replay                        - send out modification signals
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 1
-    min_args = 0
-
-    def command(self):
-        self._load_config()
-        from ckan.model import Session, Package, DomainObjectOperation
-        from ckan.model.modification import DomainObjectModificationExtension
-
-        if not self.args:
-            # default to run
-            cmd = 'replay'
-        else:
-            cmd = self.args[0]
-
-        if cmd == 'replay':
-            dome = DomainObjectModificationExtension()
-            for package in Session.query(Package):
-                dome.notify(package, DomainObjectOperation.changed)
-        else:
-            print('Command %s not recognized' % cmd)
-
-
-class RDFExport(CkanCommand):
-    '''Export active datasets as RDF
-    This command dumps out all currently active datasets as RDF into the
-    specified folder.
-
-    Usage:
-      paster rdf-export /path/to/store/output
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-
-    def command(self):
-        self._load_config()
-
-        if not self.args:
-            # default to run
-            print(RDFExport.__doc__)
-        else:
-            self.export_datasets(self.args[0])
-
-    def export_datasets(self, out_folder):
-        '''
-        Export datasets as RDF to an output folder.
-        '''
-        from ckan.common import config
-        import ckan.model as model
-        import ckan.logic as logic
-        import ckan.lib.helpers as h
-
-        # Create output folder if not exists
-        if not os.path.isdir(out_folder):
-            os.makedirs(out_folder)
-
-        fetch_url = config['ckan.site_url']
-        user = logic.get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
-        context = {'model': model, 'session': model.Session, 'user': user['name']}
-        dataset_names = logic.get_action('package_list')(context, {})
-        for dataset_name in dataset_names:
-            dd = logic.get_action('package_show')(context, {'id': dataset_name})
-            if not dd['state'] == 'active':
-                continue
-
-            url = h.url_for('dataset.read', id=dd['name'])
-
-            url = urljoin(fetch_url, url[1:]) + '.rdf'
-            try:
-                fname = os.path.join(out_folder, dd['name']) + ".rdf"
-                try:
-                    r = urlopen(url).read()
-                except HTTPError as e:
-                    if e.code == 404:
-                        error('Please install ckanext-dcat and enable the ' +
-                              '`dcat` plugin to use the RDF serializations')
-                with open(fname, 'wb') as f:
-                    f.write(r)
-            except IOError as ioe:
-                sys.stderr.write(str(ioe) + "\n")
-
-
-class Sysadmin(CkanCommand):
-    '''Gives sysadmin rights to a named user
-
-    Usage:
-      sysadmin                      - lists sysadmins
-      sysadmin list                 - lists sysadmins
-      sysadmin add USERNAME         - make an existing user into a sysadmin
-      sysadmin add USERNAME [FIELD1=VALUE1 FIELD2=VALUE2 ...]
-                                    - creates a new user that is a sysadmin
-                                      (prompts for password and email if not
-                                      supplied).
-                                      Field can be: apikey
-                                                    email
-                                                    fullname
-                                                    name (this will be the username)
-                                                    password
-      sysadmin remove USERNAME      - removes user from sysadmins
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = None
-    min_args = 0
-
-    def command(self):
-        self._load_config()
-
-        cmd = self.args[0] if self.args else None
-        if cmd is None or cmd == 'list':
-            self.list()
-        elif cmd == 'add':
-            self.add()
-        elif cmd == 'remove':
-            self.remove()
-        else:
-            print('Command %s not recognized' % cmd)
-
-    def list(self):
-        import ckan.model as model
-        print('Sysadmins:')
-        sysadmins = model.Session.query(model.User).filter_by(sysadmin=True,
-                                                              state='active')
-        print('count = %i' % sysadmins.count())
-        for sysadmin in sysadmins:
-            print('%s name=%s email=%s id=%s' % (
-                sysadmin.__class__.__name__,
-                sysadmin.name,
-                sysadmin.email,
-                sysadmin.id))
-
-    def add(self):
-        import ckan.model as model
-
-        if len(self.args) < 2:
-            print('Need name of the user to be made sysadmin.')
-            return
-        username = self.args[1]
-
-        user = model.User.by_name(text_type(username))
-        if not user:
-            print('User "%s" not found' % username)
-            makeuser = input('Create new user: %s? [y/n]' % username)
-            if makeuser == 'y':
-                user_add(self.args[1:])
-                user = model.User.by_name(text_type(username))
-            else:
-                print('Exiting ...')
-                return
-
-        user.sysadmin = True
-        model.Session.add(user)
-        model.repo.commit_and_remove()
-        print('Added %s as sysadmin' % username)
-
-    def remove(self):
-        import ckan.model as model
-
-        if len(self.args) < 2:
-            print('Need name of the user to be made sysadmin.')
-            return
-        username = self.args[1]
-
-        user = model.User.by_name(text_type(username))
-        if not user:
-            print('Error: user "%s" not found!' % username)
-            return
-        user.sysadmin = False
-        model.repo.commit_and_remove()
-
-
-class UserCmd(CkanCommand):
-    '''Manage users
-
-    Usage:
-      user                            - lists users
-      user list                       - lists users
-      user USERNAME                   - shows user properties
-      user add USERNAME [FIELD1=VALUE1 FIELD2=VALUE2 ...]
-                                      - add a user (prompts for email and
-                                        password if not supplied).
-                                        Field can be: apikey
-                                                      email
-                                                      fullname
-                                                      name (this will be the username)
-                                                      password
-      user setpass USERNAME           - set user password (prompts)
-      user remove USERNAME            - removes user from users
-      user search QUERY               - searches for a user name
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = None
-    min_args = 0
-
-    def command(self):
-        self._load_config()
-
-        if not self.args:
-            self.list()
-        else:
-            cmd = self.args[0]
-            if cmd == 'add':
-                self.add()
-            elif cmd == 'remove':
-                self.remove()
-            elif cmd == 'search':
-                self.search()
-            elif cmd == 'setpass':
-                self.setpass()
-            elif cmd == 'list':
-                self.list()
-            else:
-                self.show()
-
-    def get_user_str(self, user):
-        user_str = 'name=%s' % user.name
-        if user.name != user.display_name:
-            user_str += ' display=%s' % user.display_name
-        return user_str
-
-    def list(self):
-        import ckan.model as model
-        print('Users:')
-        users = model.Session.query(model.User).filter_by(state='active')
-        print('count = %i' % users.count())
-        for user in users:
-            print(self.get_user_str(user))
-
-    def show(self):
-        import ckan.model as model
-
-        username = self.args[0]
-        user = model.User.get(text_type(username))
-        print('User: \n', user)
-
-    def setpass(self):
-        import ckan.model as model
-
-        if len(self.args) < 2:
-            print('Need name of the user.')
-            return
-        username = self.args[1]
-        user = model.User.get(username)
-        print('Editing user: %r' % user.name)
-
-        password = self.password_prompt()
-        user.password = password
-        model.repo.commit_and_remove()
-        print('Done')
-
-    def search(self):
-        import ckan.model as model
-
-        if len(self.args) < 2:
-            print('Need user name query string.')
-            return
-        query_str = self.args[1]
-
-        query = model.User.search(query_str)
-        print('%i users matching %r:' % (query.count(), query_str))
-        for user in query.all():
-            print(self.get_user_str(user))
-
-    @classmethod
-    def password_prompt(cls):
-        import getpass
-        password1 = None
-        while not password1:
-            password1 = getpass.getpass('Password: ')
-        password2 = getpass.getpass('Confirm password: ')
-        if password1 != password2:
-            error('Passwords do not match')
-        return password1
-
-    def add(self):
-        user_add(self.args[1:])
-
-    def remove(self):
-        import ckan.model as model
-
-        if len(self.args) < 2:
-            print('Need name of the user.')
-            return
-        username = self.args[1]
-
-        p.toolkit.get_action('user_delete')(
-            {'model': model, 'ignore_auth': True},
-            {'id': username})
-        print('Deleted user: %s' % username)
-
-
-class DatasetCmd(CkanCommand):
-    '''Manage datasets
-
-    Usage:
-      dataset DATASET_NAME|ID            - shows dataset properties
-      dataset show DATASET_NAME|ID       - shows dataset properties
-      dataset list                       - lists datasets
-      dataset delete [DATASET_NAME|ID]   - changes dataset state to 'deleted'
-      dataset purge [DATASET_NAME|ID]    - removes dataset from db entirely
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 3
-    min_args = 0
-
-    def command(self):
-        self._load_config()
-
-        if not self.args:
-            print(self.usage)
-        else:
-            cmd = self.args[0]
-            if cmd == 'delete':
-                self.delete(self.args[1])
-            elif cmd == 'purge':
-                self.purge(self.args[1])
-            elif cmd == 'list':
-                self.list()
-            elif cmd == 'show':
-                self.show(self.args[1])
-            else:
-                self.show(self.args[0])
-
-    def list(self):
-        import ckan.model as model
-        print('Datasets:')
-        datasets = model.Session.query(model.Package)
-        print('count = %i' % datasets.count())
-        for dataset in datasets:
-            state = ('(%s)' % dataset.state) if dataset.state != 'active' else ''
-            print('%s %s %s' % (dataset.id, dataset.name, state))
-
-    def _get_dataset(self, dataset_ref):
-        import ckan.model as model
-        dataset = model.Package.get(text_type(dataset_ref))
-        assert dataset, 'Could not find dataset matching reference: %r' % dataset_ref
-        return dataset
-
-    def show(self, dataset_ref):
-        import pprint
-        dataset = self._get_dataset(dataset_ref)
-        pprint.pprint(dataset.as_dict())
-
-    def delete(self, dataset_ref):
-        import ckan.model as model
-        dataset = self._get_dataset(dataset_ref)
-        old_state = dataset.state
-
-        dataset.delete()
-        model.repo.commit_and_remove()
-        dataset = self._get_dataset(dataset_ref)
-        print('%s %s -> %s' % (dataset.name, old_state, dataset.state))
-
-    def purge(self, dataset_ref):
-        import ckan.logic as logic
-        dataset = self._get_dataset(dataset_ref)
-        name = dataset.name
-
-        site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {})
-        context = {'user': site_user['name']}
-        logic.get_action('dataset_purge')(
-            context, {'id': dataset_ref})
-        print('%s purged' % name)
-
-
-class Ratings(CkanCommand):
-    '''Manage the ratings stored in the db
-
-    Usage:
-      ratings count                 - counts ratings
-      ratings clean                 - remove all ratings
-      ratings clean-anonymous       - remove only anonymous ratings
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 1
-    min_args = 1
-
-    def command(self):
-        self._load_config()
-        import ckan.model as model
-
-        cmd = self.args[0]
-        if cmd == 'count':
-            self.count()
-        elif cmd == 'clean':
-            self.clean()
-        elif cmd == 'clean-anonymous':
-            self.clean(user_ratings=False)
-        else:
-            print('Command %s not recognized' % cmd)
-
-    def count(self):
-        import ckan.model as model
-        q = model.Session.query(model.Rating)
-        print("%i ratings" % q.count())
-        q = q.filter(model.Rating.user_id is None)
-        print("of which %i are anonymous ratings" % q.count())
-
-    def clean(self, user_ratings=True):
-        import ckan.model as model
-        q = model.Session.query(model.Rating)
-        print("%i ratings" % q.count())
-        if not user_ratings:
-            q = q.filter(model.Rating.user_id is None)
-            print("of which %i are anonymous ratings" % q.count())
-        ratings = q.all()
-        for rating in ratings:
-            rating.purge()
-        model.repo.commit_and_remove()
-
-
-## Used by the Tracking class
-_ViewCount = collections.namedtuple("ViewCount", "id name count")
-
-
-class Tracking(CkanCommand):
-    '''Update tracking statistics
-
-    Usage:
-      tracking update [start_date]       - update tracking stats
-      tracking export FILE [start_date]  - export tracking stats to a csv file
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 3
-    min_args = 1
-
-    def command(self):
-        self._load_config()
-        import ckan.model as model
-        engine = model.meta.engine
-
-        cmd = self.args[0]
-        if cmd == 'update':
-            start_date = self.args[1] if len(self.args) > 1 else None
-            self.update_all(engine, start_date)
-        elif cmd == 'export':
-            if len(self.args) <= 1:
-                error(self.__class__.__doc__)
-            output_file = self.args[1]
-            start_date = self.args[2] if len(self.args) > 2 else None
-            self.update_all(engine, start_date)
-            self.export_tracking(engine, output_file)
-        else:
-            error(self.__class__.__doc__)
-
-    def update_all(self, engine, start_date=None):
-        if start_date:
-            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
-        else:
-            # No date given. See when we last have data for and get data
-            # from 2 days before then in case new data is available.
-            # If no date here then use 2011-01-01 as the start date
-            sql = '''SELECT tracking_date from tracking_summary
-                     ORDER BY tracking_date DESC LIMIT 1;'''
-            result = engine.execute(sql).fetchall()
-            if result:
-                start_date = result[0]['tracking_date']
-                start_date += datetime.timedelta(-2)
-                # convert date to datetime
-                combine = datetime.datetime.combine
-                start_date = combine(start_date, datetime.time(0))
-            else:
-                start_date = datetime.datetime(2011, 1, 1)
-        start_date_solrsync = start_date
-        end_date = datetime.datetime.now()
-
-        while start_date < end_date:
-            stop_date = start_date + datetime.timedelta(1)
-            self.update_tracking(engine, start_date)
-            print('tracking updated for %s' % start_date)
-            start_date = stop_date
-
-        self.update_tracking_solr(engine, start_date_solrsync)
-
-    def _total_views(self, engine):
-        sql = '''
-            SELECT p.id,
-                   p.name,
-                   COALESCE(SUM(s.count), 0) AS total_views
-               FROM package AS p
-               LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
-               GROUP BY p.id, p.name
-               ORDER BY total_views DESC
-        '''
-        return [_ViewCount(*t) for t in engine.execute(sql).fetchall()]
-
-    def _recent_views(self, engine, measure_from):
-        sql = '''
-            SELECT p.id,
-                   p.name,
-                   COALESCE(SUM(s.count), 0) AS total_views
-               FROM package AS p
-               LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
-               WHERE s.tracking_date >= %(measure_from)s
-               GROUP BY p.id, p.name
-               ORDER BY total_views DESC
-        '''
-        return [_ViewCount(*t) for t in engine.execute(sql, measure_from=str(measure_from)).fetchall()]
-
-    def export_tracking(self, engine, output_filename):
-        '''Write tracking summary to a csv file.'''
-        HEADINGS = [
-            "dataset id",
-            "dataset name",
-            "total views",
-            "recent views (last 2 weeks)",
-        ]
-
-        measure_from = datetime.date.today() - datetime.timedelta(days=14)
-        recent_views = self._recent_views(engine, measure_from)
-        total_views = self._total_views(engine)
-
-        with open(output_filename, 'w') as fh:
-            f_out = csv.writer(fh)
-            f_out.writerow(HEADINGS)
-            recent_views_for_id = dict((r.id, r.count) for r in recent_views)
-            f_out.writerows([(r.id,
-                              r.name,
-                              r.count,
-                              recent_views_for_id.get(r.id, 0))
-                              for r in total_views])
-
-    def update_tracking(self, engine, summary_date):
-        PACKAGE_URL = '/dataset/'
-        # clear out existing data before adding new
-        sql = '''DELETE FROM tracking_summary
-                 WHERE tracking_date='%s'; ''' % summary_date
-        engine.execute(sql)
-
-        sql = '''SELECT DISTINCT url, user_key,
-                     CAST(access_timestamp AS Date) AS tracking_date,
-                     tracking_type INTO tracking_tmp
-                 FROM tracking_raw
-                 WHERE CAST(access_timestamp as Date)=%s;
-
-                 INSERT INTO tracking_summary
-                   (url, count, tracking_date, tracking_type)
-                 SELECT url, count(user_key), tracking_date, tracking_type
-                 FROM tracking_tmp
-                 GROUP BY url, tracking_date, tracking_type;
-
-                 DROP TABLE tracking_tmp;
-                 COMMIT;'''
-        engine.execute(sql, summary_date)
-
-        # get ids for dataset urls
-        sql = '''UPDATE tracking_summary t
-                 SET package_id = COALESCE(
-                        (SELECT id FROM package p
-                        WHERE p.name = regexp_replace(' ' || t.url, '^[ ]{1}(/\w{2}){0,1}' || %s, ''))
-                     ,'~~not~found~~')
-                 WHERE t.package_id IS NULL
-                 AND tracking_type = 'page';'''
-        engine.execute(sql, PACKAGE_URL)
-
-        # update summary totals for resources
-        sql = '''UPDATE tracking_summary t1
-                 SET running_total = (
-                    SELECT sum(count)
-                    FROM tracking_summary t2
-                    WHERE t1.url = t2.url
-                    AND t2.tracking_date <= t1.tracking_date
-                 )
-                 ,recent_views = (
-                    SELECT sum(count)
-                    FROM tracking_summary t2
-                    WHERE t1.url = t2.url
-                    AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
-                 )
-                 WHERE t1.running_total = 0 AND tracking_type = 'resource';'''
-        engine.execute(sql)
-
-        # update summary totals for pages
-        sql = '''UPDATE tracking_summary t1
-                 SET running_total = (
-                    SELECT sum(count)
-                    FROM tracking_summary t2
-                    WHERE t1.package_id = t2.package_id
-                    AND t2.tracking_date <= t1.tracking_date
-                 )
-                 ,recent_views = (
-                    SELECT sum(count)
-                    FROM tracking_summary t2
-                    WHERE t1.package_id = t2.package_id
-                    AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
-                 )
-                 WHERE t1.running_total = 0 AND tracking_type = 'page'
-                 AND t1.package_id IS NOT NULL
-                 AND t1.package_id != '~~not~found~~';'''
-        engine.execute(sql)
-
-    def update_tracking_solr(self, engine, start_date):
-        sql = '''SELECT package_id FROM tracking_summary
-                where package_id!='~~not~found~~'
-                and tracking_date >= %s;'''
-        results = engine.execute(sql, start_date)
-
-        package_ids = set()
-        for row in results:
-            package_ids.add(row['package_id'])
-
-        total = len(package_ids)
-        not_found = 0
-        print('%i package index%s to be rebuilt starting from %s' % (total, '' if total < 2 else 'es', start_date))
-
-        from ckan.lib.search import rebuild
-        for package_id in package_ids:
-            try:
-                rebuild(package_id)
-            except logic.NotFound:
-                print("Error: package %s not found." % (package_id))
-                not_found += 1
-            except KeyboardInterrupt:
-                print("Stopped.")
-                return
-            except:
-                raise
-        print('search index rebuilding done.' + (' %i not found.' % (not_found) if not_found else ""))
-
-
-class PluginInfo(CkanCommand):
-    '''Provide info on installed plugins.
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 0
-    min_args = 0
-
-    def command(self):
-        self.get_info()
-
-    def get_info(self):
-        ''' print info about current plugins from the .ini file'''
-        import ckan.plugins as p
-        self._load_config()
-        interfaces = {}
-        plugins = {}
-        for name in dir(p):
-            item = getattr(p, name)
-            try:
-                if issubclass(item, p.Interface):
-                    interfaces[item] = {'class': item}
-            except TypeError:
-                pass
-
-        for interface in interfaces:
-            for plugin in p.PluginImplementations(interface):
-                name = plugin.name
-                if name not in plugins:
-                    plugins[name] = {'doc': plugin.__doc__,
-                                     'class': plugin,
-                                     'implements': []}
-                plugins[name]['implements'].append(interface.__name__)
-
-        for plugin in plugins:
-            p = plugins[plugin]
-            print(plugin + ':')
-            print('-' * (len(plugin) + 1))
-            if p['doc']:
-                print(p['doc'])
-            print('Implements:')
-            for i in p['implements']:
-                extra = None
-                if i == 'ITemplateHelpers':
-                    extra = self.template_helpers(p['class'])
-                if i == 'IActions':
-                    extra = self.actions(p['class'])
-                print('    %s' % i)
-                if extra:
-                    print(extra)
-            print()
-
-    def actions(self, cls):
-        ''' Return readable action function info. '''
-        actions = cls.get_actions()
-        return self.function_info(actions)
-
-    def template_helpers(self, cls):
-        ''' Return readable helper function info. '''
-        helpers = cls.get_helpers()
-        return self.function_info(helpers)
-
-    def function_info(self, functions):
-        ''' Take a dict of functions and output readable info '''
-        import inspect
-        output = []
-        for function_name in functions:
-            fn = functions[function_name]
-            args_info = inspect.getargspec(fn)
-            params = args_info.args
-            num_params = len(params)
-            if args_info.varargs:
-                params.append('*' + args_info.varargs)
-            if args_info.keywords:
-                params.append('**' + args_info.keywords)
-            if args_info.defaults:
-                offset = num_params - len(args_info.defaults)
-                for i, v in enumerate(args_info.defaults):
-                    params[i + offset] = params[i + offset] + '=' + repr(v)
-            # is this a classmethod if so remove the first parameter
-            if inspect.ismethod(fn) and inspect.isclass(fn.__self__):
-                params = params[1:]
-            params = ', '.join(params)
-            output.append('        %s(%s)' % (function_name, params))
-            # doc string
-            if fn.__doc__:
-                bits = fn.__doc__.split('\n')
-                for bit in bits:
-                    output.append('            %s' % bit)
-        return ('\n').join(output)
-
-
-class CreateTestDataCommand(CkanCommand):
-    '''Create test data in the database.
-    Tests can also delete the created objects easily with the delete() method.
-
-    create-test-data              - annakarenina and warandpeace
-    create-test-data search       - realistic data to test search
-    create-test-data gov          - government style data
-    create-test-data family       - package relationships data
-    create-test-data user         - create a user 'tester' with api key 'tester'
-    create-test-data translations - annakarenina, warandpeace, and some test
-                                    translations of terms
-    create-test-data vocabs       - annakerenina, warandpeace, and some test
-                                    vocabularies
-    create-test-data hierarchy    - hierarchy of groups
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 1
-    min_args = 0
-
-    def command(self):
-        self._load_config()
-        from ckan.lib.create_test_data import CreateTestData
-
-        if self.args:
-            cmd = self.args[0]
-        else:
-            cmd = 'basic'
-        if self.verbose:
-            print('Creating %s test data' % cmd)
-        if cmd == 'basic':
-            CreateTestData.create_basic_test_data()
-        elif cmd == 'user':
-            CreateTestData.create_test_user()
-            print('Created user %r with password %r and apikey %r' %
-                  ('tester', 'tester', 'tester'))
-        elif cmd == 'search':
-            CreateTestData.create_search_test_data()
-        elif cmd == 'gov':
-            CreateTestData.create_gov_test_data()
-        elif cmd == 'family':
-            CreateTestData.create_family_test_data()
-        elif cmd == 'translations':
-            CreateTestData.create_translations_test_data()
-        elif cmd == 'vocabs':
-            CreateTestData.create_vocabs_test_data()
-        elif cmd == 'hierarchy':
-            CreateTestData.create_group_hierarchy_test_data()
-        else:
-            print('Command %s not recognized' % cmd)
-            raise NotImplementedError
-        if self.verbose:
-            print('Creating %s test data: Complete!' % cmd)
-
-
-class Profile(CkanCommand):
-    '''Code speed profiler
-    Provide a ckan url and it will make the request and record
-    how long each function call took in a file that can be read
-    by pstats.Stats (command-line) or runsnakerun (gui).
-
-    Usage:
-       profile URL [username]
-
-    e.g. profile /data/search
-
-    The result is saved in profile.data.search
-    To view the profile in runsnakerun:
-       runsnakerun ckan.data.search.profile
-
-    You may need to install python module: cProfile
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 2
-    min_args = 1
-
-    def _load_config_into_test_app(self):
-        from paste.deploy import loadapp
-        import paste.fixture
-        if not self.options.config:
-            msg = 'No config file supplied'
-            raise self.BadCommand(msg)
-        self.filename = os.path.abspath(self.options.config)
-        if not os.path.exists(self.filename):
-            raise AssertionError('Config filename %r does not exist.' % self.filename)
-        fileConfig(self.filename)
-
-        wsgiapp = loadapp('config:' + self.filename)
-        self.app = paste.fixture.TestApp(wsgiapp)
-
-    def command(self):
-        self._load_config_into_test_app()
-
-        import paste.fixture
-        import cProfile
-        import re
-
-        url = self.args[0]
-        if self.args[1:]:
-            user = self.args[1]
-        else:
-            user = 'visitor'
-
-        def profile_url(url):
-            try:
-                res = self.app.get(url, status=[200],
-                                   extra_environ={'REMOTE_USER': user})
-            except paste.fixture.AppError:
-                print('App error: ', url.strip())
-            except KeyboardInterrupt:
-                raise
-            except Exception:
-                error(traceback.format_exc())
-
-        output_filename = 'ckan%s.profile' % re.sub('[/?]', '.', url.replace('/', '.'))
-        profile_command = "profile_url('%s')" % url
-        cProfile.runctx(profile_command, globals(), locals(), filename=output_filename)
-        import pstats
-        stats = pstats.Stats(output_filename)
-        stats.sort_stats('cumulative')
-        stats.print_stats(0.1)  # show only top 10% of lines
-        print('Only top 10% of lines shown')
-        print('Written profile to: %s' % output_filename)
-
-
-class CreateColorSchemeCommand(CkanCommand):
-    '''Create or remove a color scheme.
-
-    After running this, you'll need to regenerate the css files. See paster's less command for details.
-
-    color               - creates a random color scheme
-    color clear         - clears any color scheme
-    color <'HEX'>       - uses as base color eg '#ff00ff' must be quoted.
-    color <VALUE>       - a float between 0.0 and 1.0 used as base hue
-    color <COLOR_NAME>  - html color name used for base color eg lightblue
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 1
-    min_args = 0
-
-    rules = [
-        '@layoutLinkColor',
-        '@mastheadBackgroundColor',
-        '@btnPrimaryBackground',
-        '@btnPrimaryBackgroundHighlight',
-    ]
-
-    # list of predefined colors
-    color_list = {
-        'aliceblue': '#f0fff8',
-        'antiquewhite': '#faebd7',
-        'aqua': '#00ffff',
-        'aquamarine': '#7fffd4',
-        'azure': '#f0ffff',
-        'beige': '#f5f5dc',
-        'bisque': '#ffe4c4',
-        'black': '#000000',
-        'blanchedalmond': '#ffebcd',
-        'blue': '#0000ff',
-        'blueviolet': '#8a2be2',
-        'brown': '#a52a2a',
-        'burlywood': '#deb887',
-        'cadetblue': '#5f9ea0',
-        'chartreuse': '#7fff00',
-        'chocolate': '#d2691e',
-        'coral': '#ff7f50',
-        'cornflowerblue': '#6495ed',
-        'cornsilk': '#fff8dc',
-        'crimson': '#dc143c',
-        'cyan': '#00ffff',
-        'darkblue': '#00008b',
-        'darkcyan': '#008b8b',
-        'darkgoldenrod': '#b8860b',
-        'darkgray': '#a9a9a9',
-        'darkgrey': '#a9a9a9',
-        'darkgreen': '#006400',
-        'darkkhaki': '#bdb76b',
-        'darkmagenta': '#8b008b',
-        'darkolivegreen': '#556b2f',
-        'darkorange': '#ff8c00',
-        'darkorchid': '#9932cc',
-        'darkred': '#8b0000',
-        'darksalmon': '#e9967a',
-        'darkseagreen': '#8fbc8f',
-        'darkslateblue': '#483d8b',
-        'darkslategray': '#2f4f4f',
-        'darkslategrey': '#2f4f4f',
-        'darkturquoise': '#00ced1',
-        'darkviolet': '#9400d3',
-        'deeppink': '#ff1493',
-        'deepskyblue': '#00bfff',
-        'dimgray': '#696969',
-        'dimgrey': '#696969',
-        'dodgerblue': '#1e90ff',
-        'firebrick': '#b22222',
-        'floralwhite': '#fffaf0',
-        'forestgreen': '#228b22',
-        'fuchsia': '#ff00ff',
-        'gainsboro': '#dcdcdc',
-        'ghostwhite': '#f8f8ff',
-        'gold': '#ffd700',
-        'goldenrod': '#daa520',
-        'gray': '#808080',
-        'grey': '#808080',
-        'green': '#008000',
-        'greenyellow': '#adff2f',
-        'honeydew': '#f0fff0',
-        'hotpink': '#ff69b4',
-        'indianred ': '#cd5c5c',
-        'indigo ': '#4b0082',
-        'ivory': '#fffff0',
-        'khaki': '#f0e68c',
-        'lavender': '#e6e6fa',
-        'lavenderblush': '#fff0f5',
-        'lawngreen': '#7cfc00',
-        'lemonchiffon': '#fffacd',
-        'lightblue': '#add8e6',
-        'lightcoral': '#f08080',
-        'lightcyan': '#e0ffff',
-        'lightgoldenrodyellow': '#fafad2',
-        'lightgray': '#d3d3d3',
-        'lightgrey': '#d3d3d3',
-        'lightgreen': '#90ee90',
-        'lightpink': '#ffb6c1',
-        'lightsalmon': '#ffa07a',
-        'lightseagreen': '#20b2aa',
-        'lightskyblue': '#87cefa',
-        'lightslategray': '#778899',
-        'lightslategrey': '#778899',
-        'lightsteelblue': '#b0c4de',
-        'lightyellow': '#ffffe0',
-        'lime': '#00ff00',
-        'limegreen': '#32cd32',
-        'linen': '#faf0e6',
-        'magenta': '#ff00ff',
-        'maroon': '#800000',
-        'mediumaquamarine': '#66cdaa',
-        'mediumblue': '#0000cd',
-        'mediumorchid': '#ba55d3',
-        'mediumpurple': '#9370d8',
-        'mediumseagreen': '#3cb371',
-        'mediumslateblue': '#7b68ee',
-        'mediumspringgreen': '#00fa9a',
-        'mediumturquoise': '#48d1cc',
-        'mediumvioletred': '#c71585',
-        'midnightblue': '#191970',
-        'mintcream': '#f5fffa',
-        'mistyrose': '#ffe4e1',
-        'moccasin': '#ffe4b5',
-        'navajowhite': '#ffdead',
-        'navy': '#000080',
-        'oldlace': '#fdf5e6',
-        'olive': '#808000',
-        'olivedrab': '#6b8e23',
-        'orange': '#ffa500',
-        'orangered': '#ff4500',
-        'orchid': '#da70d6',
-        'palegoldenrod': '#eee8aa',
-        'palegreen': '#98fb98',
-        'paleturquoise': '#afeeee',
-        'palevioletred': '#d87093',
-        'papayawhip': '#ffefd5',
-        'peachpuff': '#ffdab9',
-        'peru': '#cd853f',
-        'pink': '#ffc0cb',
-        'plum': '#dda0dd',
-        'powderblue': '#b0e0e6',
-        'purple': '#800080',
-        'red': '#ff0000',
-        'rosybrown': '#bc8f8f',
-        'royalblue': '#4169e1',
-        'saddlebrown': '#8b4513',
-        'salmon': '#fa8072',
-        'sandybrown': '#f4a460',
-        'seagreen': '#2e8b57',
-        'seashell': '#fff5ee',
-        'sienna': '#a0522d',
-        'silver': '#c0c0c0',
-        'skyblue': '#87ceeb',
-        'slateblue': '#6a5acd',
-        'slategray': '#708090',
-        'slategrey': '#708090',
-        'snow': '#fffafa',
-        'springgreen': '#00ff7f',
-        'steelblue': '#4682b4',
-        'tan': '#d2b48c',
-        'teal': '#008080',
-        'thistle': '#d8bfd8',
-        'tomato': '#ff6347',
-        'turquoise': '#40e0d0',
-        'violet': '#ee82ee',
-        'wheat': '#f5deb3',
-        'white': '#ffffff',
-        'whitesmoke': '#f5f5f5',
-        'yellow': '#ffff00',
-        'yellowgreen': '#9acd32',
-    }
-
-    def create_colors(self, hue, num_colors=5, saturation=None, lightness=None):
-        if saturation is None:
-            saturation = 0.9
-        if lightness is None:
-            lightness = 40
-        else:
-            lightness *= 100
-
-        import math
-        saturation -= math.trunc(saturation)
-
-        print(hue, saturation)
-        import colorsys
-        ''' Create n related colours '''
-        colors = []
-        for i in xrange(num_colors):
-            ix = i * (1.0/num_colors)
-            _lightness = (lightness + (ix * 40))/100.
-            if _lightness > 1.0:
-                _lightness = 1.0
-            color = colorsys.hls_to_rgb(hue, _lightness, saturation)
-            hex_color = '#'
-            for part in color:
-                hex_color += '%02x' % int(part * 255)
-            # check and remove any bad values
-            if not re.match('^\#[0-9a-f]{6}$', hex_color):
-                hex_color = '#FFFFFF'
-            colors.append(hex_color)
-        return colors
-
-    def command(self):
-
-        hue = None
-        saturation = None
-        lightness = None
-
-        public = config.get(u'ckan.base_public_folder')
-        path = os.path.dirname(__file__)
-        path = os.path.join(path, '..', public, 'base', 'less', 'custom.less')
-
-        if self.args:
-            arg = self.args[0]
-            rgb = None
-            if arg == 'clear':
-                os.remove(path)
-                print('custom colors removed.')
-            elif arg.startswith('#'):
-                color = arg[1:]
-                if len(color) == 3:
-                    rgb = [int(x, 16) * 16 for x in color]
-                elif len(color) == 6:
-                    rgb = [int(x, 16) for x in re.findall('..', color)]
-                else:
-                    print('ERROR: invalid color')
-            elif arg.lower() in self.color_list:
-                color = self.color_list[arg.lower()][1:]
-                rgb = [int(x, 16) for x in re.findall('..', color)]
-            else:
-                try:
-                    hue = float(self.args[0])
-                except ValueError:
-                    print('ERROR argument `%s` not recognised' % arg)
-            if rgb:
-                import colorsys
-                hue, lightness, saturation = colorsys.rgb_to_hls(*rgb)
-                lightness = lightness / 340
-                # deal with greys
-                if not (hue == 0.0 and saturation == 0.0):
-                    saturation = None
-        else:
-            import random
-            hue = random.random()
-        if hue is not None:
-            f = open(path, 'w')
-            colors = self.create_colors(hue, saturation=saturation, lightness=lightness)
-            for i in xrange(len(self.rules)):
-                f.write('%s: %s;\n' % (self.rules[i], colors[i]))
-                print('%s: %s;\n' % (self.rules[i], colors[i]))
-            f.close
-            print('Color scheme has been created.')
-        print('Make sure less is run for changes to take effect.')
-
-
-class TranslationsCommand(CkanCommand):
-    '''Translation helper functions
-
-    trans js      - generate the javascript translations
-    trans mangle  - mangle the zh_TW translations for testing
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    max_args = 1
-    min_args = 1
-
-    def command(self):
-        self._load_config()
-        from ckan.common import config
-        from ckan.lib.i18n import build_js_translations
-        ckan_path = os.path.join(os.path.dirname(__file__), '..')
-        self.i18n_path = config.get('ckan.i18n_directory',
-                                    os.path.join(ckan_path, 'i18n'))
-        command = self.args[0]
-        if command == 'mangle':
-            self.mangle_po()
-        elif command == 'js':
-            build_js_translations()
-        else:
-            print('command not recognised')
-
-    def mangle_po(self):
-        ''' This will mangle the zh_TW translations for translation coverage
-        testing.
-
-        NOTE: This will destroy the current translations fot zh_TW
-        '''
-        import polib
-        pot_path = os.path.join(self.i18n_path, 'ckan.pot')
-        po = polib.pofile(pot_path)
-        # we don't want to mangle the following items in strings
-        # %(...)s  %s %0.3f %1$s %2$0.3f [1:...] {...} etc
-
-        # sprintf bit after %
-        spf_reg_ex = "\+?(0|'.)?-?\d*(.\d*)?[\%bcdeufosxX]"
-
-        extract_reg_ex = '(\%\([^\)]*\)' + spf_reg_ex + \
-                         '|\[\d*\:[^\]]*\]' + \
-                         '|\{[^\}]*\}' + \
-                         '|<[^>}]*>' + \
-                         '|\%((\d)*\$)?' + spf_reg_ex + ')'
-
-        for entry in po:
-            msg = entry.msgid.encode('utf-8')
-            matches = re.finditer(extract_reg_ex, msg)
-            length = len(msg)
-            position = 0
-            translation = u''
-            for match in matches:
-                translation += '-' * (match.start() - position)
-                position = match.end()
-                translation += match.group(0)
-            translation += '-' * (length - position)
-            entry.msgstr = translation
-        out_dir = os.path.join(self.i18n_path, 'zh_TW', 'LC_MESSAGES')
-        try:
-            os.makedirs(out_dir)
-        except OSError:
-            pass
-        po.metadata['Plural-Forms'] = "nplurals=1; plural=0\n"
-        out_po = os.path.join(out_dir, 'ckan.po')
-        out_mo = os.path.join(out_dir, 'ckan.mo')
-        po.save(out_po)
-        po.save_as_mofile(out_mo)
-        print('zh_TW has been mangled')
-
-
-class MinifyCommand(CkanCommand):
-    '''Create minified versions of the given Javascript and CSS files.
-
-    Usage:
-
-        paster minify [--clean] PATH
-
-    for example:
-
-        paster minify ckan/public/base
-        paster minify ckan/public/base/css/*.css
-        paster minify ckan/public/base/css/red.css
-
-    if the --clean option is provided any minified files will be removed.
-
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    min_args = 1
-
-    exclude_dirs = ['vendor']
-
-    def __init__(self, name):
-
-        super(MinifyCommand, self).__init__(name)
-
-        self.parser.add_option('--clean', dest='clean',
-                               action='store_true', default=False,
-                               help='remove any minified files in the path')
-
-    def command(self):
-        clean = getattr(self.options, 'clean', False)
-        self._load_config()
-        for base_path in self.args:
-            if os.path.isfile(base_path):
-                if clean:
-                    self.clear_minifyed(base_path)
-                else:
-                    self.minify_file(base_path)
-            elif os.path.isdir(base_path):
-                for root, dirs, files in os.walk(base_path):
-                    dirs[:] = [d for d in dirs if not d in self.exclude_dirs]
-                    for filename in files:
-                        path = os.path.join(root, filename)
-                        if clean:
-                            self.clear_minifyed(path)
-                        else:
-                            self.minify_file(path)
-            else:
-                # Path is neither a file or a dir?
-                continue
-
-    def clear_minifyed(self, path):
-        path_only, extension = os.path.splitext(path)
-
-        if extension not in ('.css', '.js'):
-            # This is not a js or css file.
-            return
-
-        if path_only.endswith('.min'):
-            print('removing %s' % path)
-            os.remove(path)
-
-    def minify_file(self, path):
-        '''Create the minified version of the given file.
-
-        If the file is not a .js or .css file (e.g. it's a .min.js or .min.css
-        file, or it's some other type of file entirely) it will not be
-        minifed.
-
-        :param path: The path to the .js or .css file to minify
-
-        '''
-        import ckan.lib.fanstatic_resources as fanstatic_resources
-
-        path_only, extension = os.path.splitext(path)
-
-        if path_only.endswith('.min'):
-            # This is already a minified file.
-            return
-
-        if extension not in ('.css', '.js'):
-            # This is not a js or css file.
-            return
-
-        path_min = fanstatic_resources.min_path(path)
-
-        source = open(path, 'r').read()
-        f = open(path_min, 'w')
-        if path.endswith('.css'):
-            f.write(rcssmin.cssmin(source))
-        elif path.endswith('.js'):
-            f.write(rjsmin.jsmin(source))
-        f.close()
-        print("Minified file '{0}'".format(path))
-
-
-class LessCommand(CkanCommand):
-    '''Compile all root less documents into their CSS counterparts
-
-    Usage:
-
-        paster less
-
-    '''
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    min_args = 0
-
-    def command(self):
-        self._load_config()
-        self.less()
-
-    custom_css = {
-        'fuchsia': '''
-            @layoutLinkColor: #E73892;
-            @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
-            @footerLinkColor: @footerTextColor;
-            @mastheadBackgroundColor: @layoutLinkColor;
-            @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
-            @btnPrimaryBackgroundHighlight: @layoutLinkColor;
-            ''',
-
-        'green': '''
-            @layoutLinkColor: #2F9B45;
-            @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
-            @footerLinkColor: @footerTextColor;
-            @mastheadBackgroundColor: @layoutLinkColor;
-            @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
-            @btnPrimaryBackgroundHighlight: @layoutLinkColor;
-            ''',
-
-        'red': '''
-            @layoutLinkColor: #C14531;
-            @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
-            @footerLinkColor: @footerTextColor;
-            @mastheadBackgroundColor: @layoutLinkColor;
-            @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
-            @btnPrimaryBackgroundHighlight: @layoutLinkColor;
-            ''',
-
-        'maroon': '''
-            @layoutLinkColor: #810606;
-            @footerTextColor: mix(#FFF, @layoutLinkColor, 60%);
-            @footerLinkColor: @footerTextColor;
-            @mastheadBackgroundColor: @layoutLinkColor;
-            @btnPrimaryBackground: lighten(@layoutLinkColor, 10%);
-            @btnPrimaryBackgroundHighlight: @layoutLinkColor;
-            ''',
-    }
-
-    def less(self):
-        ''' Compile less files '''
-        import subprocess
-        command = ('npm', 'bin')
-        process = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            universal_newlines=True)
-        output = process.communicate()
-        directory = output[0].strip()
-        if not directory:
-            raise error('Command "{}" returned nothing. Check that npm is '
-                        'installed.'.format(' '.join(command)))
-        less_bin = os.path.join(directory, 'lessc')
-
-        public = config.get(u'ckan.base_public_folder')
-
-        root = os.path.join(os.path.dirname(__file__), '..', public, 'base')
-        root = os.path.abspath(root)
-        custom_less = os.path.join(root, 'less', 'custom.less')
-        for color in self.custom_css:
-            f = open(custom_less, 'w')
-            f.write(self.custom_css[color])
-            f.close()
-            self.compile_less(root, less_bin, color)
-        f = open(custom_less, 'w')
-        f.write('// This file is needed in order for `gulp build` to compile in less 1.3.1+\n')
-        f.close()
-        self.compile_less(root, less_bin, 'main')
-
-    def compile_less(self, root, less_bin, color):
-        print('compile %s.css' % color)
-        import subprocess
-        main_less = os.path.join(root, 'less', 'main.less')
-        main_css = os.path.join(root, 'css', '%s.css' % color)
-
-        command = (less_bin, main_less, main_css)
-        process = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            universal_newlines=True)
-        output = process.communicate()
-        print(output)
-
-
-class FrontEndBuildCommand(CkanCommand):
-    '''Creates and minifies css and JavaScript files
-
-    Usage:
-
-        paster front-end-build
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    min_args = 0
-
-    def command(self):
-        self._load_config()
-
-        # Less css
-        cmd = LessCommand('less')
-        cmd.options = self.options
-        cmd.command()
-
-        # js translation strings
-        cmd = TranslationsCommand('trans')
-        cmd.options = self.options
-        cmd.args = ('js',)
-        cmd.command()
-
-        # minification
-        cmd = MinifyCommand('minify')
-        cmd.options = self.options
-        public = config.get(u'ckan.base_public_folder')
-        root = os.path.join(os.path.dirname(__file__), '..', public, 'base')
-        root = os.path.abspath(root)
-        ckanext = os.path.join(os.path.dirname(__file__), '..', '..', 'ckanext')
-        ckanext = os.path.abspath(ckanext)
-        cmd.args = (root, ckanext)
-        cmd.command()
-
-
-class ViewsCommand(CkanCommand):
-    '''Manage resource views.
-
-    Usage:
-
-        paster views create [options] [type1] [type2] ...
-
-            Create views on relevant resources. You can optionally provide
-            specific view types (eg `recline_view`, `image_view`). If no types
-            are provided, the default ones will be used. These are generally
-            the ones defined in the `ckan.views.default_views` config option.
-            Note that on either case, plugins must be loaded (ie added to
-            `ckan.plugins`), otherwise the command will stop.
-
-        paster views clear [options] [type1] [type2] ...
-
-            Permanently delete all views or the ones with the provided types.
-
-        paster views clean
-
-            Permanently delete views for all types no longer present in the
-            `ckan.plugins` configuration option.
-
-    '''
-
-    summary = __doc__.split('\n')[0]
-    usage = __doc__
-    min_args = 1
-
-    def __init__(self, name):
-
-        super(ViewsCommand, self).__init__(name)
-
-        self.parser.add_option('-y', '--yes', dest='assume_yes',
-                               action='store_true',
-                               default=False,
-                               help='''Automatic yes to prompts. Assume "yes"
-as answer to all prompts and run non-interactively''')
-
-        self.parser.add_option('-d', '--dataset', dest='dataset_id',
-                               action='append',
-                               help='''Create views on a particular dataset.
-You can use the dataset id or name, and it can be defined multiple times.''')
-
-        self.parser.add_option('--no-default-filters',
-                               dest='no_default_filters',
-                               action='store_true',
-                               default=False,
-                               help='''Do not add default filters for relevant
-resource formats for the view types provided. Note that filters are not added
-by default anyway if an unsupported view type is provided or when using the
-`-s` or `-d` options.''')
-
-        self.parser.add_option('-s', '--search', dest='search_params',
-                               action='store',
-                               default=False,
-                               help='''Extra search parameters that will be
-used for getting the datasets to create the resource views on. It must be a
-JSON object like the one used by the `package_search` API call. Supported
-fields are `q`, `fq` and `fq_list`. Check the documentation for examples.
-Not used when using the `-d` option.''')
-
-    def command(self):
-        self._load_config()
-        if not self.args:
-            print(self.usage)
-        elif self.args[0] == 'create':
-            view_plugin_types = self.args[1:]
-            self.create_views(view_plugin_types)
-        elif self.args[0] == 'clear':
-            view_plugin_types = self.args[1:]
-            self.clear_views(view_plugin_types)
-        elif self.args[0] == 'clean':
-            self.clean_views()
-        else:
-            print(self.usage)
-
-    _page_size = 100
-
-    def _get_view_plugins(self, view_plugin_types,
-                          get_datastore_views=False):
-        '''
-        Returns the view plugins that were succesfully loaded
-
-        Views are provided as a list of ``view_plugin_types``. If no types are
-        provided, the default views defined in the ``ckan.views.default_views``
-        will be created. Only in this case (when the default view plugins are
-        used) the `get_datastore_views` parameter can be used to get also view
-        plugins that require data to be in the DataStore.
-
-        If any of the provided plugins could not be loaded (eg it was not added
-        to `ckan.plugins`) the command will stop.
-
-        Returns a list of loaded plugin names.
-        '''
-        from ckan.lib.datapreview import (get_view_plugins,
-                                          get_default_view_plugins
-                                          )
-
-        log = logging.getLogger(__name__)
-
-        view_plugins = []
-
-        if not view_plugin_types:
-            log.info('No view types provided, using default types')
-            view_plugins = get_default_view_plugins()
-            if get_datastore_views:
-                view_plugins.extend(
-                    get_default_view_plugins(get_datastore_views=True))
-        else:
-            view_plugins = get_view_plugins(view_plugin_types)
-
-        loaded_view_plugins = [view_plugin.info()['name']
-                               for view_plugin in view_plugins]
-
-        plugins_not_found = list(set(view_plugin_types) -
-                                 set(loaded_view_plugins))
-
-        if plugins_not_found:
-            error('View plugin(s) not found : {0}. '.format(plugins_not_found)
-                  + 'Have they been added to the `ckan.plugins` configuration'
-                  + ' option?')
-
-        return loaded_view_plugins
-
-    def _add_default_filters(self, search_data_dict, view_types):
-        '''
-        Adds extra filters to the `package_search` dict for common view types
-
-        It basically adds `fq` parameters that filter relevant resource formats
-        for the view types provided. For instance, if one of the view types is
-        `pdf_view` the following will be added to the final query:
-
-            fq=res_format:"pdf" OR res_format:"PDF"
-
-        This obviously should only be used if all view types are known and can
-        be filtered, otherwise we want all datasets to be returned. If a
-        non-filterable view type is provided, the search params are not
-        modified.
-
-        Returns the provided data_dict for `package_search`, optionally
-        modified with extra filters.
-        '''
-
-        from ckanext.imageview.plugin import DEFAULT_IMAGE_FORMATS
-        from ckanext.textview.plugin import get_formats as get_text_formats
-        from ckanext.datapusher.plugin import DEFAULT_FORMATS as \
-            datapusher_formats
-
-        filter_formats = []
-
-        for view_type in view_types:
-            if view_type == 'image_view':
-
-                for _format in DEFAULT_IMAGE_FORMATS:
-                    filter_formats.extend([_format, _format.upper()])
-
-            elif view_type == 'text_view':
-                formats = get_text_formats(config)
-                for _format in itertools.chain.from_iterable(formats.values()):
-                    filter_formats.extend([_format, _format.upper()])
-
-            elif view_type == 'pdf_view':
-                filter_formats.extend(['pdf', 'PDF'])
-
-            elif view_type in ['recline_view', 'recline_grid_view',
-                               'recline_graph_view', 'recline_map_view']:
-
-                if datapusher_formats[0] in filter_formats:
-                    continue
-
-                for _format in datapusher_formats:
-                    if '/' not in _format:
-                        filter_formats.extend([_format, _format.upper()])
-            else:
-                # There is another view type provided so we can't add any
-                # filter
-                return search_data_dict
-
-        filter_formats_query = ['+res_format:"{0}"'.format(_format)
-                                for _format in filter_formats]
-        search_data_dict['fq_list'].append(' OR '.join(filter_formats_query))
-
-        return search_data_dict
-
-    def _update_search_params(self, search_data_dict):
-        '''
-        Update the `package_search` data dict with the user provided parameters
-
-        Supported fields are `q`, `fq` and `fq_list`.
-
-        If the provided JSON object can not be parsed the process stops with
-        an error.
-
-        Returns the updated data dict
-        '''
-
-        log = logging.getLogger(__name__)
-
-        if not self.options.search_params:
-            return search_data_dict
-
-        try:
-            user_search_params = json.loads(self.options.search_params)
-        except ValueError as e:
-            error('Unable to parse JSON search parameters: {0}'.format(e))
-
-        if user_search_params.get('q'):
-            search_data_dict['q'] = user_search_params['q']
-
-        if user_search_params.get('fq'):
-            if search_data_dict['fq']:
-                search_data_dict['fq'] += ' ' + user_search_params['fq']
-            else:
-                search_data_dict['fq'] = user_search_params['fq']
-
-        if (user_search_params.get('fq_list') and
-                isinstance(user_search_params['fq_list'], list)):
-            search_data_dict['fq_list'].extend(user_search_params['fq_list'])
-
-    def _search_datasets(self, page=1, view_types=[]):
-        '''
-        Perform a query with `package_search` and return the result
-
-        Results can be paginated using the `page` parameter
-        '''
-
-        n = self._page_size
-
-        search_data_dict = {
-            'q': '',
-            'fq': '',
-            'fq_list': [],
-            'include_private': True,
-            'rows': n,
-            'start': n * (page - 1),
-        }
-
-        if self.options.dataset_id:
-
-            search_data_dict['q'] = ' OR '.join(
-                ['id:{0} OR name:"{0}"'.format(dataset_id)
-                 for dataset_id in self.options.dataset_id]
-            )
-
-        elif self.options.search_params:
-
-            self._update_search_params(search_data_dict)
-
-        elif not self.options.no_default_filters:
-
-            self._add_default_filters(search_data_dict, view_types)
-
-        if not search_data_dict.get('q'):
-            search_data_dict['q'] = '*:*'
-
-        query = p.toolkit.get_action('package_search')(
-            {}, search_data_dict)
-
-        return query
-
-    def create_views(self, view_plugin_types=[]):
-
-        from ckan.lib.datapreview import add_views_to_dataset_resources
-
-        log = logging.getLogger(__name__)
-
-        datastore_enabled = 'datastore' in config['ckan.plugins'].split()
-
-        loaded_view_plugins = self._get_view_plugins(view_plugin_types,
-                                                     datastore_enabled)
-
-        context = {'user': self.site_user['name']}
-
-        page = 1
-        while True:
-            query = self._search_datasets(page, loaded_view_plugins)
-
-            if page == 1 and query['count'] == 0:
-                error('No datasets to create resource views on, exiting...')
-
-            elif page == 1 and not self.options.assume_yes:
-
-                msg = ('\nYou are about to check {0} datasets for the ' +
-                       'following view plugins: {1}\n' +
-                       ' Do you want to continue?')
-
-                confirm = query_yes_no(msg.format(query['count'],
-                                                  loaded_view_plugins))
-
-                if confirm == 'no':
-                    error('Command aborted by user')
-
-            if query['results']:
-                for dataset_dict in query['results']:
-
-                    if not dataset_dict.get('resources'):
-                        continue
-
-                    views = add_views_to_dataset_resources(
-                        context,
-                        dataset_dict,
-                        view_types=loaded_view_plugins)
-
-                    if views:
-                        view_types = list({view['view_type']
-                                           for view in views})
-                        msg = ('Added {0} view(s) of type(s) {1} to ' +
-                               'resources from dataset {2}')
-                        log.debug(msg.format(len(views),
-                                             ', '.join(view_types),
-                                             dataset_dict['name']))
-
-                if len(query['results']) < self._page_size:
-                    break
-
-                page += 1
-            else:
-                break
-
-        log.info('Done')
-
-    def clear_views(self, view_plugin_types=[]):
-
-        log = logging.getLogger(__name__)
-
-        if not self.options.assume_yes:
-            if view_plugin_types:
-                msg = 'Are you sure you want to delete all resource views ' + \
-                      'of type {0}?'.format(', '.join(view_plugin_types))
-            else:
-                msg = 'Are you sure you want to delete all resource views?'
-
-            result = query_yes_no(msg, default='no')
-
-            if result == 'no':
-                error('Command aborted by user')
-
-        context = {'user': self.site_user['name']}
-        logic.get_action('resource_view_clear')(
-            context, {'view_types': view_plugin_types})
-
-        log.info('Done')
-
-    def clean_views(self):
-        names = []
-        for plugin in p.PluginImplementations(p.IResourceView):
-            names.append(str(plugin.info()['name']))
-
-        results = model.ResourceView.get_count_not_in_view_types(names)
-
-        if not results:
-            print('No resource views to delete')
-            return
-
-        print('This command will delete.\n')
-        for row in results:
-            print('%s of type %s' % (row[1], row[0]))
-
-        result = query_yes_no('Do you want to delete these resource views:', default='no')
-
-        if result == 'no':
-            print('Not Deleting.')
-            return
-
-        model.ResourceView.delete_not_in_view_types(names)
-        model.Session.commit()
-        print('Deleted resource views.')
-
-
-class ConfigToolCommand(paste.script.command.Command):
-    '''Tool for editing options in a CKAN config file
-
-    paster config-tool <default.ini> <key>=<value> [<key>=<value> ...]
-    paster config-tool <default.ini> -f <custom_options.ini>
-
-    Examples:
-      paster config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC'
-      paster config-tool default.ini -s server:main -e port=8080
-      paster config-tool default.ini -f custom_options.ini
-    '''
-    parser = paste.script.command.Command.standard_parser(verbose=True)
-    default_verbosity = 1
-    group_name = 'ckan'
-    usage = __doc__
-    summary = usage.split('\n')[0]
-
-    parser.add_option('-s', '--section', dest='section',
-                      default='app:main', help='Section of the config file')
-    parser.add_option(
-        '-e', '--edit', action='store_true', dest='edit', default=False,
-        help='Checks the option already exists in the config file')
-    parser.add_option(
-        '-f', '--file', dest='merge_filepath', metavar='FILE',
-        help='Supply an options file to merge in')
-
-    def command(self):
-        from ckan.lib import config_tool
-        if len(self.args) < 1:
-            self.parser.error('Not enough arguments (got %i, need at least 1)'
-                              % len(self.args))
-        config_filepath = self.args[0]
-        if not os.path.exists(config_filepath):
-            self.parser.error('Config filename %r does not exist.' %
-                              config_filepath)
-        if self.options.merge_filepath:
-            config_tool.config_edit_using_merge_file(
-                config_filepath, self.options.merge_filepath)
-        options = self.args[1:]
-        if not (options or self.options.merge_filepath):
-            self.parser.error('No options provided')
-        if options:
-            for option in options:
-                if '=' not in option:
-                    error(
-                        'An option does not have an equals sign: %r '
-                        'It should be \'key=value\'. If there are spaces '
-                        'you\'ll need to quote the option.\n' % option)
-            try:
-                config_tool.config_edit_using_option_strings(
-                    config_filepath, options, self.options.section,
-                    edit=self.options.edit)
-            except config_tool.ConfigToolError as e:
-                error(traceback.format_exc())
-
-
-class JobsCommand(CkanCommand):
-    '''Manage background jobs
-
-    Usage:
-
-        paster jobs worker [--burst] [QUEUES]
-
-            Start a worker that fetches jobs from queues and executes
-            them. If no queue names are given then the worker listens
-            to the default queue, this is equivalent to
-
-                paster jobs worker default
-
-            If queue names are given then the worker listens to those
-            queues and only those:
-
-                paster jobs worker my-custom-queue
-
-            Hence, if you want the worker to listen to the default queue
-            and some others then you must list the default queue explicitly:
-
-                paster jobs worker default my-custom-queue
-
-            If the `--burst` option is given then the worker will exit
-            as soon as all its queues are empty.
-
-        paster jobs list [QUEUES]
-
-                List currently enqueued jobs from the given queues. If no queue
-                names are given then the jobs from all queues are listed.
-
-        paster jobs show ID
-
-                Show details about a specific job.
-
-        paster jobs cancel ID
-
-                Cancel a specific job. Jobs can only be canceled while they are
-                enqueued. Once a worker has started executing a job it cannot
-                be aborted anymore.
-
-        paster jobs clear [QUEUES]
-
-                Cancel all jobs on the given queues. If no queue names are
-                given then ALL queues are cleared.
-
-        paster jobs test [QUEUES]
-
-                Enqueue a test job. If no queue names are given then the job is
-                added to the default queue. If queue names are given then a
-                separate test job is added to each of the queues.
-    '''
-
-    summary = __doc__.split(u'\n')[0]
-    usage = __doc__
-    min_args = 0
-
-
-    def __init__(self, *args, **kwargs):
-        super(JobsCommand, self).__init__(*args, **kwargs)
-        try:
-            self.parser.add_option(u'--burst', action='store_true',
-                                   default=False,
-                                   help=u'Start worker in burst mode.')
-        except OptionConflictError:
-            # Option has already been added in previous call
-            pass
-
-    def command(self):
-        self._load_config()
-        try:
-            cmd = self.args.pop(0)
-        except IndexError:
-            print(self.__doc__)
-            sys.exit(0)
-        if cmd == u'worker':
-            self.worker()
-        elif cmd == u'list':
-            self.list()
-        elif cmd == u'show':
-            self.show()
-        elif cmd == u'cancel':
-            self.cancel()
-        elif cmd == u'clear':
-            self.clear()
-        elif cmd == u'test':
-            self.test()
-        else:
-            error(u'Unknown command "{}"'.format(cmd))
-
-    def worker(self):
-        from ckan.lib.jobs import Worker
-        Worker(self.args).work(burst=self.options.burst)
-
-    def list(self):
-        data_dict = {
-            u'queues': self.args,
-        }
-        jobs = p.toolkit.get_action(u'job_list')({}, data_dict)
-        for job in jobs:
-            if job[u'title'] is None:
-                job[u'title'] = ''
-            else:
-                job[u'title'] = u'"{}"'.format(job[u'title'])
-            print(u'{created} {id} {queue} {title}'.format(**job))
-
-    def show(self):
-        if not self.args:
-            error(u'You must specify a job ID')
-        id = self.args[0]
-        try:
-            job = p.toolkit.get_action(u'job_show')({}, {u'id': id})
-        except logic.NotFound:
-            error(u'There is no job with ID "{}"'.format(id))
-        print(u'ID:      {}'.format(job[u'id']))
-        if job[u'title'] is None:
-            title = u'None'
-        else:
-            title = u'"{}"'.format(job[u'title'])
-        print(u'Title:   {}'.format(title))
-        print(u'Created: {}'.format(job[u'created']))
-        print(u'Queue:   {}'.format(job[u'queue']))
-
-    def cancel(self):
-        if not self.args:
-            error(u'You must specify a job ID')
-        id = self.args[0]
-        try:
-            p.toolkit.get_action(u'job_cancel')({}, {u'id': id})
-        except logic.NotFound:
-            error(u'There is no job with ID "{}"'.format(id))
-        print(u'Cancelled job {}'.format(id))
-
-    def clear(self):
-        data_dict = {
-            u'queues': self.args,
-        }
-        queues = p.toolkit.get_action(u'job_clear')({}, data_dict)
-        queues = (u'"{}"'.format(q) for q in queues)
-        print(u'Cleared queue(s) {}'.format(u', '.join(queues)))
-
-    def test(self):
-        from ckan.lib.jobs import DEFAULT_QUEUE_NAME, enqueue, test_job
-        for queue in (self.args or [DEFAULT_QUEUE_NAME]):
-            job = enqueue(test_job, [u'A test job'], title=u'A test job', queue=queue)
-            print(u'Added test job {} to queue "{}"'.format(job.id, queue))
diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -7,7 +7,7 @@
     _parse_db_config,
     paster_click_group,
 )
-from ckan.lib.cli.cli import click_config_option
+from ckan.cli.cli import click_config_option
 
 from ckanext.datastore.backend.postgres import identifier
 from ckanext.datastore.blueprint import DUMP_FORMATS, dump_to
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -51,29 +51,8 @@ def parse_version(s):
         'main = ckan.config.install:CKANInstaller',
     ],
     'paste.paster_command': [
-        'db = ckan.lib.cli:ManageDb',
-        'create-test-data = ckan.lib.cli:CreateTestDataCommand',
-        'sysadmin = ckan.lib.cli:Sysadmin',
-        'user = ckan.lib.cli:UserCmd',
-        'dataset = ckan.lib.cli:DatasetCmd',
-        'search-index = ckan.lib.cli:SearchIndexCommand',
-        'ratings = ckan.lib.cli:Ratings',
-        'notify = ckan.lib.cli:Notification',
-        'rdf-export = ckan.lib.cli:RDFExport',
-        'tracking = ckan.lib.cli:Tracking',
-        'plugin-info = ckan.lib.cli:PluginInfo',
-        'profile = ckan.lib.cli:Profile',
-        'color = ckan.lib.cli:CreateColorSchemeCommand',
-        'check-po-files = ckan.i18n.check_po_files:CheckPoFiles',
-        'trans = ckan.lib.cli:TranslationsCommand',
-        'minify = ckan.lib.cli:MinifyCommand',
-        'less = ckan.lib.cli:LessCommand',
         'datastore = ckanext.datastore.commands:datastore_group',
         'datapusher = ckanext.datapusher.cli:DatapusherCommand',
-        'front-end-build = ckan.lib.cli:FrontEndBuildCommand',
-        'views = ckan.lib.cli:ViewsCommand',
-        'config-tool = ckan.lib.cli:ConfigToolCommand',
-        'jobs = ckan.lib.cli:JobsCommand',
     ],
     'console_scripts': [
         'ckan = ckan.cli.cli:ckan',


Test patch:
diff --git a/ckan/tests/cli/test_jobs.py b/ckan/tests/cli/test_jobs.py
--- a/ckan/tests/cli/test_jobs.py
+++ b/ckan/tests/cli/test_jobs.py
@@ -1,22 +1,36 @@
 # -*- coding: utf-8 -*-
 
-import os
 import datetime
+import os
+import os.path
 import tempfile
 
-import ckan.lib.jobs as jobs
 from ckan.cli.cli import ckan
-from ckan.tests.helpers import RQTestBase
+import ckan.lib.jobs as jobs
+import ckan.tests.helpers as helpers
+
+
+def test_unknown_command(cli):
+    """
+    Test error handling for unknown ``ckan jobs`` sub-command.
+    """
+    result = cli.invoke(ckan, [u"jobs", u"does-not-exist"])
+    assert result.exit_code != 0
+    assert u"No such command" in result.output
+
 
+class TestJobsList(helpers.RQTestBase):
+    """
+    Tests for ``ckan jobs list``.
+    """
 
-class TestJobShow(RQTestBase):
     def test_list_default_queue(self, cli):
         """
         Test output of ``jobs list`` for default queue.
         """
         job = self.enqueue()
-        result = cli.invoke(ckan, [u"jobs", u"list"])
-        fields = result.output.split()
+        stdout = cli.invoke(ckan, [u"jobs", u"list"]).output
+        fields = stdout.split(u"\n")[-2].split()
         assert len(fields) == 3
         dt = datetime.datetime.strptime(fields[0], u"%Y-%m-%dT%H:%M:%S")
         now = datetime.datetime.utcnow()
@@ -29,8 +43,8 @@ def test_list_other_queue(self, cli):
         Test output of ``jobs.list`` for non-default queue.
         """
         job = self.enqueue(queue=u"my_queue")
-        result = cli.invoke(ckan, [u"jobs", u"list"])
-        fields = result.output.split()
+        stdout = cli.invoke(ckan, [u"jobs", u"list"]).output
+        fields = stdout.split(u"\n")[-2].split()
         assert len(fields) == 3
         assert fields[2] == u"my_queue"
 
@@ -39,8 +53,8 @@ def test_list_title(self, cli):
         Test title output of ``jobs list``.
         """
         job = self.enqueue(title=u"My_Title")
-        result = cli.invoke(ckan, [u"jobs", u"list"])
-        fields = result.output.split()
+        stdout = cli.invoke(ckan, [u"jobs", u"list"]).output
+        fields = stdout.split(u"\n")[-2].split()
         assert len(fields) == 4
         assert fields[3] == u'"My_Title"'
 
@@ -51,15 +65,15 @@ def test_list_filter(self, cli):
         job1 = self.enqueue(queue=u"q1")
         job2 = self.enqueue(queue=u"q2")
         job3 = self.enqueue(queue=u"q3")
-        result = cli.invoke(ckan, [u"jobs", u"list", u"q1", u"q2"])
-        assert u"q1" in result.output
-        assert u"q2" in result.output
-        assert u"q3" not in result.output
+        stdout = cli.invoke(ckan, [u"jobs", u"list", u"q1", u"q2"]).output
+        assert u"q1" in stdout
+        assert u"q2" in stdout
+        assert u"q3" not in stdout
 
 
-class TestJobShow(RQTestBase):
+class TestJobShow(helpers.RQTestBase):
     """
-    Tests for ``paster jobs show``.
+    Tests for ``ckan jobs show``.
     """
 
     def test_show_existing(self, cli):
@@ -67,21 +81,22 @@ def test_show_existing(self, cli):
         Test ``jobs show`` for an existing job.
         """
         job = self.enqueue(queue=u"my_queue", title=u"My Title")
-        result = cli.invoke(ckan, [u"jobs", u"show", job.id])
-        assert job.id in result.output
-        assert jobs.remove_queue_name_prefix(job.origin) in result.output
+        output = cli.invoke(ckan, [u"jobs", u"show", job.id]).output
+        assert job.id in output
+        assert jobs.remove_queue_name_prefix(job.origin) in output
 
     def test_show_missing_id(self, cli):
         """
         Test ``jobs show`` with a missing ID.
         """
         result = cli.invoke(ckan, [u"jobs", u"show"])
-        assert result.exit_code
+        assert result.exit_code != 0
+        assert u'Error: Missing argument "id".' in result.output
 
 
-class TestJobsCancel(RQTestBase):
+class TestJobsCancel(helpers.RQTestBase):
     """
-    Tests for ``paster jobs cancel``.
+    Tests for ``ckan jobs cancel``.
     """
 
     def test_cancel_existing(self, cli):
@@ -90,18 +105,19 @@ def test_cancel_existing(self, cli):
         """
         job1 = self.enqueue()
         job2 = self.enqueue()
-        result = cli.invoke(ckan, [u"jobs", u"cancel", job1.id])
+        stdout = cli.invoke(ckan, [u"jobs", u"cancel", job1.id]).output
         all_jobs = self.all_jobs()
         assert len(all_jobs) == 1
         assert all_jobs[0].id == job2.id
-        assert job1.id in result.output
+        assert job1.id in stdout
 
     def test_cancel_not_existing(self, cli):
         """
         Test ``jobs cancel`` for a not existing job.
         """
         result = cli.invoke(ckan, [u"jobs", u"cancel", u"does-not-exist"])
-        assert result.exit_code
+        # FIXME: after https://github.com/ckan/ckan/issues/5158
+        # assert result.exit_code != 0
         assert u"does-not-exist" in result.output
 
     def test_cancel_missing_id(self, cli):
@@ -109,12 +125,13 @@ def test_cancel_missing_id(self, cli):
         Test ``jobs cancel`` with a missing ID.
         """
         result = cli.invoke(ckan, [u"jobs", u"cancel"])
-        assert result.exit_code
+        assert result.exit_code != 0
+        assert u'Error: Missing argument "id".' in result.output
 
 
-class TestJobsClear(RQTestBase):
+class TestJobsClear(helpers.RQTestBase):
     """
-    Tests for ``paster jobs clear``.
+    Tests for ``ckan jobs clear``.
     """
 
     def test_clear_all_queues(self, cli):
@@ -125,10 +142,10 @@ def test_clear_all_queues(self, cli):
         self.enqueue()
         self.enqueue(queue=u"q1")
         self.enqueue(queue=u"q2")
-        result = cli.invoke(ckan, [u"jobs", u"clear"])
-        assert jobs.DEFAULT_QUEUE_NAME in result.output
-        assert u"q1" in result.output
-        assert u"q2" in result.output
+        stdout = cli.invoke(ckan, [u"jobs", u"clear"]).output
+        assert jobs.DEFAULT_QUEUE_NAME in stdout
+        assert u"q1" in stdout
+        assert u"q2" in stdout
         assert self.all_jobs() == []
 
     def test_clear_specific_queues(self, cli):
@@ -140,25 +157,25 @@ def test_clear_specific_queues(self, cli):
         self.enqueue(queue=u"q2")
         self.enqueue(queue=u"q2")
         self.enqueue(queue=u"q3")
-        result = cli.invoke(ckan, [u"jobs", u"clear", u"q2", u"q3"])
-        assert u"q2" in result.output
-        assert u"q3" in result.output
-        assert jobs.DEFAULT_QUEUE_NAME not in result.output
-        assert u"q1" not in result.output
+        stdout = cli.invoke(ckan, [u"jobs", u"clear", u"q2", u"q3"]).output
+        assert u"q2" in stdout
+        assert u"q3" in stdout
+        assert jobs.DEFAULT_QUEUE_NAME not in stdout
+        assert u"q1" not in stdout
         all_jobs = self.all_jobs()
         assert set(all_jobs) == {job1, job2}
 
 
-class TestJobsTest(RQTestBase):
+class TestJobsTest(helpers.RQTestBase):
     """
-    Tests for ``paster jobs test``.
+    Tests for ``ckan jobs test``.
     """
 
     def test_test_default_queue(self, cli):
         """
         Test ``jobs test`` for the default queue.
         """
-        cli.invoke(ckan, [u"jobs", u"test"])
+        stdout = cli.invoke(ckan, [u"jobs", u"test"]).output
         all_jobs = self.all_jobs()
         assert len(all_jobs) == 1
         assert (
@@ -170,8 +187,7 @@ def test_test_specific_queues(self, cli):
         """
         Test ``jobs test`` for specific queues.
         """
-
-        cli.invoke(ckan, [u"jobs", u"test", u"q1", u"q2"])
+        stdout = cli.invoke(ckan, [u"jobs", u"test", u"q1", u"q2"]).output
         all_jobs = self.all_jobs()
         assert len(all_jobs) == 2
         assert {jobs.remove_queue_name_prefix(j.origin) for j in all_jobs} == {
@@ -180,9 +196,9 @@ def test_test_specific_queues(self, cli):
         }
 
 
-class TestJobsWorker(RQTestBase):
+class TestJobsWorker(helpers.RQTestBase):
     """
-    Tests for ``paster jobs worker``.
+    Tests for ``ckan jobs worker``.
     """
 
     # All tests of ``jobs worker`` must use the ``--burst`` option to
diff --git a/ckan/tests/cli/test_user.py b/ckan/tests/cli/test_user.py
--- a/ckan/tests/cli/test_user.py
+++ b/ckan/tests/cli/test_user.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import pytest
+
 from ckan.cli.cli import ckan
 
 
@@ -19,7 +20,8 @@ def test_cli_user_add_valid_args(self, cli):
             u"email=berty@example.com",
         ]
         result = cli.invoke(ckan, args)
-        assert not result.exit_code
+
+        assert result.exit_code == 0
 
     def test_cli_user_add_no_args(self, cli):
         """Command with no args raises SystemExit.