Repository: ckan/ckan

Issue:
Paster command db simple-dump-json fails with out of memory error
When dumping large datasets (~17.000) the paster command `db simple-dump-json` fails with an out of memory error. 

System: 
- Ubuntu 14.04.2 LTS 
- 64-bit platform
- RAM 8 GB

I tried to patch the local install with an implementation writing only some JSON chunks to the output file, which still fails in the way that the process gets killed, also by an out of memory error.

``` python
def dump_json(self, dump_file_obj, query):
    for pkg in query:
        pkg_dict = pkg.as_dict()
        for chunk in json.JSONEncoder(indent=4).iterencode(pkg_dict):
            dump_file_obj.write(chunk)

    dump_file_obj.close()
```

My way of avoiding failing dumps is currently using a combination of `ckanapi dump datasets` and some post `jq` manipulation via `cat dump.jsonl | jq --slurp . > dump.json`.



Discussion:

Resolving patch:
diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -186,9 +186,6 @@ class ManageDb(CkanCommand):
     db upgrade [version no.]       - Data migrate
     db version                     - returns current version of data schema
     db dump FILE_PATH              - dump to a pg_dump file
-    db simple-dump-csv FILE_PATH   - dump just datasets in CSV format
-    db simple-dump-json FILE_PATH  - dump just datasets in JSON format
-    db user-dump-csv FILE_PATH     - dump user information to a CSV file
     db load FILE_PATH              - load a pg_dump from a file
     db load-only FILE_PATH         - load a pg_dump from a file but don\'t do
                                      the schema upgrade or search indexing
@@ -239,12 +236,6 @@ def command(self):
             self.load()
         elif cmd == 'load-only':
             self.load(only_load=True)
-        elif cmd == 'simple-dump-csv':
-            self.simple_dump_csv()
-        elif cmd == 'simple-dump-json':
-            self.simple_dump_json()
-        elif cmd == 'user-dump-csv':
-            self.user_dump_csv()
         elif cmd == 'create-from-model':
             model.repo.create_db()
             if self.verbose:
@@ -326,35 +317,6 @@ def load(self, only_load=False):
             print 'Now remember you have to call \'db upgrade\' and then \'search-index rebuild\'.'
         print 'Done'
 
-    def simple_dump_csv(self):
-        import ckan.model as model
-        if len(self.args) < 2:
-            print 'Need csv file path'
-            return
-        dump_filepath = self.args[1]
-        import ckan.lib.dumper as dumper
-        dump_file = open(dump_filepath, 'w')
-        dumper.SimpleDumper().dump(dump_file, format='csv')
-
-    def simple_dump_json(self):
-        import ckan.model as model
-        if len(self.args) < 2:
-            print 'Need json file path'
-            return
-        dump_filepath = self.args[1]
-        import ckan.lib.dumper as dumper
-        dump_file = open(dump_filepath, 'w')
-        dumper.SimpleDumper().dump(dump_file, format='json')
-
-    def user_dump_csv(self):
-        if len(self.args) < 2:
-            print 'Need csv file path'
-            return
-        dump_filepath = self.args[1]
-        import ckan.lib.dumper as dumper
-        dump_file = open(dump_filepath, 'w')
-        dumper.UserDumper().dump(dump_file)
-
     def migrate_filestore(self):
         from ckan.model import Session
         import requests
diff --git a/ckan/lib/dumper.py b/ckan/lib/dumper.py
deleted file mode 100644
--- a/ckan/lib/dumper.py
+++ /dev/null
@@ -1,108 +0,0 @@
-import csv
-
-import ckan.model as model
-from ckan.common import json, OrderedDict
-
-
-class SimpleDumper(object):
-    '''Dumps just package data but including tags, groups, license text etc'''
-    def dump(self, dump_file_obj, format='json', query=None):
-        if query is None:
-            query = model.Session.query(model.Package)
-            active = model.State.ACTIVE
-            query = query.filter_by(state=active)
-        if format == 'csv':
-            self.dump_csv(dump_file_obj, query)
-        elif format == 'json':
-            self.dump_json(dump_file_obj, query)
-        else:
-            raise Exception('Unknown format: %s' % format)
-
-    def dump_csv(self, dump_file_obj, query):
-        row_dicts = []
-        for pkg in query:
-            pkg_dict = pkg.as_dict()
-            # flatten dict
-            for name, value in pkg_dict.items()[:]:
-                if isinstance(value, (list, tuple)):
-                    if value and isinstance(value[0], dict) and \
-                            name == 'resources':
-                        for i, res in enumerate(value):
-                            prefix = 'resource-%i' % i
-                            pkg_dict[prefix + '-url'] = res['url']
-                            pkg_dict[prefix + '-format'] = res['format']
-                            pkg_dict[prefix + '-description'] = \
-                                res['description']
-                    else:
-                        pkg_dict[name] = ' '.join(value)
-                if isinstance(value, dict):
-                    for name_, value_ in value.items():
-                        pkg_dict[name_] = value_
-                    del pkg_dict[name]
-            row_dicts.append(pkg_dict)
-        writer = CsvWriter(row_dicts)
-        writer.save(dump_file_obj)
-
-    def dump_json(self, dump_file_obj, query):
-        pkgs = []
-        for pkg in query:
-            pkg_dict = pkg.as_dict()
-            pkgs.append(pkg_dict)
-        json.dump(pkgs, dump_file_obj, indent=4)
-
-
-class CsvWriter:
-    def __init__(self, package_dict_list=None):
-        self._rows = []
-        self._col_titles = []
-        for row_dict in package_dict_list:
-            for key in row_dict.keys():
-                if key not in self._col_titles:
-                    self._col_titles.append(key)
-        for row_dict in package_dict_list:
-            self._add_row_dict(row_dict)
-
-    def _add_row_dict(self, row_dict):
-        row = []
-        for title in self._col_titles:
-            if title in row_dict:
-                if isinstance(row_dict[title], int):
-                    row.append(row_dict[title])
-                elif isinstance(row_dict[title], unicode):
-                    row.append(row_dict[title].encode('utf8'))
-                else:
-                    row.append(row_dict[title])
-            else:
-                row.append(None)
-        self._rows.append(row)
-
-    def save(self, file_obj):
-        writer = csv.writer(file_obj, quotechar='"',
-                            quoting=csv.QUOTE_NONNUMERIC)
-        writer.writerow(self._col_titles)
-        for row in self._rows:
-            writer.writerow(row)
-
-
-class UserDumper(object):
-    def dump(self, dump_file_obj):
-        query = model.Session.query(model.User)
-        query = query.order_by(model.User.created.asc())
-
-        columns = (('id', 'name', 'openid', 'fullname', 'email', 'created',
-                    'about'))
-        row_dicts = []
-        for user in query:
-            row = OrderedDict()
-            for col in columns:
-                value = getattr(user, col)
-                if not value:
-                    value = ''
-                if col == 'created':
-                    value = str(value)  # or maybe dd/mm/yyyy?
-                row[col] = value
-            row_dicts.append(row)
-
-        writer = CsvWriter(row_dicts)
-        writer.save(dump_file_obj)
-        dump_file_obj.close()


Test patch:
diff --git a/ckan/tests/legacy/lib/test_cli.py b/ckan/tests/legacy/lib/test_cli.py
--- a/ckan/tests/legacy/lib/test_cli.py
+++ b/ckan/tests/legacy/lib/test_cli.py
@@ -10,45 +10,6 @@
 
 from ckan.lib.search import index_for,query_for, clear_all
 
-class TestDb:
-    @classmethod
-    def setup_class(cls):
-        cls.db = ManageDb('db')
-        CreateTestData.create()
-
-        # delete warandpeace
-        rev = model.repo.new_revision()
-        model.Package.by_name(u'warandpeace').delete()
-        model.repo.commit_and_remove()
-
-    @classmethod
-    def teardown_class(cls):
-        model.repo.rebuild_db()
-
-    def test_simple_dump_csv(self):
-        csv_filepath = '/tmp/dump.tmp'
-        self.db.args = ('simple-dump-csv %s' % csv_filepath).split()
-        self.db.simple_dump_csv()
-        assert os.path.exists(csv_filepath), csv_filepath
-        f_obj = open(csv_filepath, "r")
-        reader = csv.reader(f_obj)
-        rows = [row for row in reader]
-        assert_equal(rows[0][:3], ['id', 'name', 'title'])
-        pkg_names = set(row[1] for row in rows[1:])
-        assert 'annakarenina' in pkg_names, pkg_names
-        assert 'warandpeace' not in pkg_names, pkg_names
-
-    def test_simple_dump_json(self):
-        json_filepath = '/tmp/dump.tmp'
-        self.db.args = ('simple-dump-json %s' % json_filepath).split()
-        self.db.simple_dump_json()
-        assert os.path.exists(json_filepath), json_filepath
-        f_obj = open(json_filepath, "r")
-        rows = json.loads(f_obj.read())
-        assert set(rows[0].keys()) > set(('id', 'name', 'title')), rows[0].keys()
-        pkg_names = set(row['name'] for row in rows)
-        assert 'annakarenina' in pkg_names, pkg_names
-        assert 'warandpeace' not in pkg_names, pkg_names
 
 class FakeOptions():
     def __init__(self,**kwargs):
diff --git a/ckan/tests/legacy/test_dumper.py b/ckan/tests/legacy/test_dumper.py
deleted file mode 100644
--- a/ckan/tests/legacy/test_dumper.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import tempfile
-
-from ckan.tests.legacy import TestController, CreateTestData
-import ckan.model as model
-import ckan.lib.dumper as dumper
-simple_dumper = dumper.SimpleDumper()
-
-
-class TestSimpleDump(TestController):
-
-    @classmethod
-    def setup_class(self):
-        model.repo.rebuild_db()
-        CreateTestData.create()
-
-    @classmethod
-    def teardown_class(self):
-        model.Session.remove()
-        model.repo.rebuild_db()
-
-    def test_simple_dump_csv(self):
-        dump_file = tempfile.TemporaryFile()
-        simple_dumper.dump(dump_file, 'csv')
-        dump_file.seek(0)
-        res = dump_file.read()
-        assert 'annakarenina' in res, res
-        assert 'tolstoy' in res, res
-        assert 'russian' in res, res
-        assert 'genre' in res, res
-        assert 'romantic novel' in res, res
-        assert 'datahub.io/download' in res, res
-        assert 'Index of the novel' in res, res
-        assert 'joeadmin' not in res, res
-        self.assert_correct_field_order(res)
-
-    def test_simple_dump_json(self):
-        dump_file = tempfile.TemporaryFile()
-        simple_dumper.dump(dump_file, 'json')
-        dump_file.seek(0)
-        res = dump_file.read()
-        assert 'annakarenina' in res, res
-        assert '"russian"' in res, res
-        assert 'genre' in res, res
-        assert 'romantic novel' in res, res
-        assert 'joeadmin' not in res, res
-        self.assert_correct_field_order(res)
-
-    def assert_correct_field_order(self, res):
-        correct_field_order = ('id', 'name', 'title', 'version', 'url')
-        field_position = [res.find('"%s"' % field) for field in correct_field_order]
-        field_position_sorted = field_position[:]
-        field_position_sorted.sort()
-        assert field_position == field_position_sorted, field_position