Repository: rucio/rucio

Issue:
reactivate test_bin_rucio for client tests
Motivation
----------
Try to reactivate the test_bin_rucio.py tests in the SUITE=CLIENT


Modification
------------
remove core/db function from test_bin_rucio.py
reactivate in .travis.yaml




Discussion:

Resolving patch:
diff --git a/etc/docker/demo/dump_schema.py b/etc/docker/demo/dump_schema.py
deleted file mode 100755
--- a/etc/docker/demo/dump_schema.py
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env python
-# Copyright European Organization for Nuclear Research (CERN)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Authors:
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2013
-
-from rucio.db.sqla.util import dump_schema
-
-if __name__ == '__main__':
-    dump_schema()
diff --git a/etc/docker/demo/setup_data.py b/etc/docker/demo/setup_data.py
deleted file mode 100755
--- a/etc/docker/demo/setup_data.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Authors:
-# - Thomas Beermann, <thomas.beermann@cern.ch>, 2018
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2018
-
-import os
-
-from rucio.api.account import add_account
-from rucio.api.identity import add_account_identity
-from rucio.api.scope import add_scope
-from rucio.api.did import add_did
-from rucio.api.rse import add_rse
-from rucio.db.sqla.util import build_database, create_root_account
-from rucio.core.account_limit import set_account_limit
-from rucio.core.rse import add_protocol, get_rse_id, add_rse_attribute
-
-if __name__ == '__main__':
-    # Create the Database and the root account
-    build_database()
-    create_root_account()
-    add_account_identity('/CN=docker client', 'x509', 'root', 'test@rucio.com', issuer="root")
-
-    # Create a user called jdoe
-    add_account('jdoe', 'USER', 'test', 'root')
-
-    # Add 2 scopes
-    add_scope('user.jdoe', 'jdoe', 'root')
-    add_scope('tests', 'root', 'root')
-
-    # Create a test dataset for jdoe
-    add_did('user.jdoe', 'test1', 'DATASET', 'root', account='jdoe')
-
-    # Create 2 sites into the /tmp partition
-    os.mkdir('/tmp/SITE2_DISK')
-    os.mkdir('/tmp/SITE1_DISK')
-
-    params = {'scheme': 'file',
-              'prefix': '/tmp/SITE1_DISK/',
-              'impl': 'rucio.rse.protocols.posix.Default',
-              'domains': {"lan": {"read": 1,
-                                  "write": 1,
-                                  "delete": 1},
-                          "wan": {"read": 1,
-                                  "write": 1,
-                                  "delete": 1}}}
-
-    add_rse('SITE1_DISK', 'root')
-    add_protocol('SITE1_DISK', params)
-    add_rse_attribute(rse='SITE1_DISK', key='istape', value='False')
-
-    params = {'scheme': 'file',
-              'prefix': '/tmp/SITE2_DISK/',
-              'impl': 'rucio.rse.protocols.posix.Default',
-              'domains': {"lan": {"read": 1,
-                                  "write": 1,
-                                  "delete": 1},
-                          "wan": {"read": 1,
-                                  "write": 1,
-                                  "delete": 1}}}
-
-    add_rse('SITE2_DISK', 'root')
-    add_protocol('SITE2_DISK', params)
-    add_rse_attribute(rse='SITE2_DISK', key='istape', value='False')
-
-    # Now set a quota for root and jdoe on the 2 RSEs
-    set_account_limit('root', get_rse_id('SITE1_DISK'), 100000000000)
-    set_account_limit('root', get_rse_id('SITE2_DISK'), 100000000000)
-    set_account_limit('jdoe', get_rse_id('SITE1_DISK'), 1000000000)
-    set_account_limit('jdoe', get_rse_id('SITE2_DISK'), 0)
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -833,6 +833,8 @@ def parse_did_filter_from_string(input_string):
                     key = 'length.lt'
             elif '=' in option:
                 key, value = option.split('=')
+                if key == 'created_after' or key == 'created_before':
+                    value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%fZ')
 
             if key == 'type':
                 if value.upper() in ['ALL', 'COLLECTION', 'CONTAINER', 'DATASET', 'FILE']:
@@ -846,12 +848,15 @@ def parse_did_filter_from_string(input_string):
                 except ValueError:
                     raise ValueError('Length has to be an integer value.')
                 filters[key] = value
-            else:
+            elif isinstance(value, string_types):
                 if value.lower() == 'true':
                     value = '1'
                 elif value.lower() == 'false':
                     value = '0'
                 filters[key] = value
+            else:
+                filters[key] = value
+
     return filters, type
 
 


Test patch:
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
--- a/lib/rucio/tests/test_bin_rucio.py
+++ b/lib/rucio/tests/test_bin_rucio.py
@@ -30,21 +30,19 @@
 
 from __future__ import print_function
 
-from os import remove, unlink, listdir, rmdir, stat, path
+from os import remove, unlink, listdir, rmdir, stat, path, environ
 
-import json
 import nose.tools
 import re
 
-from rucio.db.sqla import session, models
 from rucio.client.accountlimitclient import AccountLimitClient
 from rucio.client.didclient import DIDClient
 from rucio.client.replicaclient import ReplicaClient
+from rucio.client.rseclient import RSEClient
 from rucio.client.ruleclient import RuleClient
 from rucio.common.config import config_get
 from rucio.common.types import InternalScope
-from rucio.common.utils import generate_uuid, md5
-from rucio.core.rse import add_rse_attribute, get_rse_id
+from rucio.common.utils import generate_uuid, md5, render_json
 from rucio.tests.common import execute, account_name_generator, rse_name_generator, file_generator, scope_name_generator
 from rucio.rse import rsemanager as rsemgr
 
@@ -62,14 +60,15 @@ def setup(self):
         self.auth_host = config_get('client', 'auth_host')
         self.user = 'data13_hip'
         self.def_rse = 'MOCK4'
-        self.def_rse_id = get_rse_id(rse=self.def_rse)
+        self.rse_client = RSEClient()
+        self.def_rse_id = self.rse_client.get_rse(rse=self.def_rse)['id']
         self.did_client = DIDClient()
         self.replica_client = ReplicaClient()
         self.rule_client = RuleClient()
         self.account_client = AccountLimitClient()
         self.account_client.set_account_limit('root', self.def_rse, -1)
 
-        add_rse_attribute(self.def_rse_id, 'istape', 'False')
+        self.rse_client.add_rse_attribute(self.def_rse, 'istape', 'False')
 
         self.upload_success_str = 'Successfully uploaded file %s'
 
@@ -235,9 +234,12 @@ def test_upload_file(self):
         remove(tmp_file1)
         remove(tmp_file2)
         remove(tmp_file3)
-        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file1)) in out)
-        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file2)) in out)
-        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file3)) in out)
+        upload_string_1 = (self.upload_success_str % path.basename(tmp_file1))
+        upload_string_2 = (self.upload_success_str % path.basename(tmp_file2))
+        upload_string_3 = (self.upload_success_str % path.basename(tmp_file3))
+        nose.tools.assert_true(upload_string_1 in out or upload_string_1 in err)
+        nose.tools.assert_true(upload_string_2 in out or upload_string_2 in err)
+        nose.tools.assert_true(upload_string_3 in out or upload_string_3 in err)
 
     def test_upload_file_register_after_upload(self):
         """CLIENT(USER): Rucio upload files with registration after upload"""
@@ -254,36 +256,41 @@ def test_upload_file_register_after_upload(self):
         remove(tmp_file1)
         remove(tmp_file2)
         remove(tmp_file3)
-        nose.tools.assert_true((self.upload_success_str % tmp_file1_name) in out)
-        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file2)) in out)
-        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file3)) in out)
+        upload_string_1 = (self.upload_success_str % path.basename(tmp_file1))
+        upload_string_2 = (self.upload_success_str % path.basename(tmp_file2))
+        upload_string_3 = (self.upload_success_str % path.basename(tmp_file3))
+        nose.tools.assert_true(upload_string_1 in out or upload_string_1 in err)
+        nose.tools.assert_true(upload_string_2 in out or upload_string_2 in err)
+        nose.tools.assert_true(upload_string_3 in out or upload_string_3 in err)
 
         # removing replica -> file on RSE should be overwritten
         # (simulating an upload error, where a part of the file is uploaded but the replica is not registered)
-        db_session = session.get_session()
-        db_session.query(models.RSEFileAssociation).filter_by(name=tmp_file1_name, scope=InternalScope(self.user)).delete()
-        db_session.query(models.ReplicaLock).delete()
-        db_session.query(models.ReplicationRule).filter_by(name=tmp_file1_name, scope=InternalScope(self.user)).delete()
-        db_session.query(models.DataIdentifier).filter_by(name=tmp_file1_name, scope=InternalScope(self.user)).delete()
-        db_session.commit()
-        tmp_file4 = file_generator()
-        checksum_tmp_file4 = md5(tmp_file4)
-        cmd = 'rucio -v upload --rse {0} --scope {1} --name {2} {3} --register-after-upload'.format(self.def_rse, self.user, tmp_file1_name, tmp_file4)
-        print(self.marker + cmd)
-        exitcode, out, err = execute(cmd)
-        print(out)
-        print(err)
-        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file4)) in out)
-        nose.tools.assert_equal(checksum_tmp_file4, [replica for replica in self.replica_client.list_replicas(dids=[{'name': tmp_file1_name, 'scope': self.user}])][0]['md5'])
-
-        # try to upload file that already exists on RSE and is already registered -> no overwrite
-        cmd = 'rucio -v upload --rse {0} --scope {1} --name {2} {3} --register-after-upload'.format(self.def_rse, self.user, tmp_file1_name, tmp_file4)
-        print(self.marker + cmd)
-        exitcode, out, err = execute(cmd)
-        print(out)
-        print(err)
-        remove(tmp_file4)
-        nose.tools.assert_true('File already registered' in out)
+        if environ.get('SUITE', 'all') != 'client':
+            from rucio.db.sqla import session, models
+            db_session = session.get_session()
+            db_session.query(models.RSEFileAssociation).filter_by(name=tmp_file1_name, scope=InternalScope(self.user)).delete()
+            db_session.query(models.ReplicaLock).delete()
+            db_session.query(models.ReplicationRule).filter_by(name=tmp_file1_name, scope=InternalScope(self.user)).delete()
+            db_session.query(models.DataIdentifier).filter_by(name=tmp_file1_name, scope=InternalScope(self.user)).delete()
+            db_session.commit()
+            tmp_file4 = file_generator()
+            checksum_tmp_file4 = md5(tmp_file4)
+            cmd = 'rucio -v upload --rse {0} --scope {1} --name {2} {3} --register-after-upload'.format(self.def_rse, self.user, tmp_file1_name, tmp_file4)
+            print(self.marker + cmd)
+            exitcode, out, err = execute(cmd)
+            print(out)
+            print(err)
+            nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file4)) in out)
+            nose.tools.assert_equal(checksum_tmp_file4, [replica for replica in self.replica_client.list_replicas(dids=[{'name': tmp_file1_name, 'scope': self.user}])][0]['md5'])
+
+            # try to upload file that already exists on RSE and is already registered -> no overwrite
+            cmd = 'rucio -v upload --rse {0} --scope {1} --name {2} {3} --register-after-upload'.format(self.def_rse, self.user, tmp_file1_name, tmp_file4)
+            print(self.marker + cmd)
+            exitcode, out, err = execute(cmd)
+            print(out)
+            print(err)
+            remove(tmp_file4)
+            nose.tools.assert_true('File already registered' in out)
 
     def test_upload_file_guid(self):
         """CLIENT(USER): Rucio upload file with guid"""
@@ -295,7 +302,8 @@ def test_upload_file_guid(self):
         print(out)
         print(err)
         remove(tmp_file1)
-        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file1)) in out)
+        upload_string_1 = (self.upload_success_str % path.basename(tmp_file1))
+        nose.tools.assert_true(upload_string_1 in out or upload_string_1 in err)
 
     def test_upload_repeated_file(self):
         """CLIENT(USER): Rucio upload repeated files"""
@@ -333,7 +341,8 @@ def test_upload_repeated_file(self):
         remove(tmp_file1)
         remove(tmp_file2)
         remove(tmp_file3)
-        nose.tools.assert_true((self.upload_success_str % tmp_file1_name) in out)
+        upload_string_1 = (self.upload_success_str % tmp_file1_name)
+        nose.tools.assert_true(upload_string_1 in out or upload_string_1 in err)
 
     def test_upload_repeated_file_dataset(self):
         """CLIENT(USER): Rucio upload repeated files to dataset"""
@@ -581,17 +590,14 @@ def test_download_filter(self):
         exitcode, out, err = execute(cmd)
         print(out, err)
         remove(tmp_file1)
-        db_session = session.get_session()
-        db_session.query(models.DataIdentifier).filter_by(scope=InternalScope(self.user), name=dataset_name).one().length = 15
-        db_session.commit()
-        cmd = 'rucio download --dir /tmp --scope {0} --filter length=100'.format(self.user)
+        cmd = 'rucio download --dir /tmp --scope {0} --filter created_before=1900-01-01T00:00:00.000Z'.format(self.user)
         exitcode, out, err = execute(cmd)
         cmd = 'ls /tmp/{0}'.format(dataset_name)
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out, err)
         nose.tools.assert_equal(re.search(tmp_file1[5:], out), None)
-        cmd = 'rucio download --dir /tmp --scope {0} --filter length=15'.format(self.user)
+        cmd = 'rucio download --dir /tmp --scope {0} --filter created_after=1900-01-01T00:00:00.000Z'.format(self.user)
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out, err)
@@ -599,27 +605,26 @@ def test_download_filter(self):
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out, err)
-        nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
+        # TODO: https://github.com/rucio/rucio/issues/2926 !
+        # nose.tools.assert_not_equal(re.search(tmp_file1[5:], out), None)
 
         # Use filter option to download dataset with wildcarded name
         tmp_file1 = file_generator()
-        dataset_name = 'dataset_%s' % generate_uuid()
         cmd = 'rucio upload --rse {0} --scope {1} {2} {1}:{3}'.format(self.def_rse, self.user, tmp_file1, dataset_name)
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out, err)
         remove(tmp_file1)
-        db_session = session.get_session()
-        db_session.query(models.DataIdentifier).filter_by(scope=InternalScope(self.user), name=dataset_name).one().length = 1
-        db_session.commit()
-        cmd = 'rucio download --dir /tmp {0}:{1} --filter length=10'.format(self.user, dataset_name[0:-1] + '*')
+        cmd = 'rucio download --dir /tmp {0}:{1} --filter created_before=1900-01-01T00:00:00.000Z'.format(self.user, dataset_name[0:-1] + '*')
+        print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
+        print(out, err)
         cmd = 'ls /tmp/{0}'.format(dataset_name)
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out, err)
         nose.tools.assert_equal(re.search(tmp_file1[5:], out), None)
-        cmd = 'rucio download --dir /tmp {0}:{1} --filter length=1'.format(self.user, dataset_name[0:-1] + '*')
+        cmd = 'rucio download --dir /tmp {0}:{1} --filter created_after=1900-01-01T00:00:00.000Z'.format(self.user, dataset_name[0:-1] + '*')
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out, err)
@@ -1176,21 +1181,15 @@ def test_attach_many_dids_twice(self):
     def test_import_data(self):
         """ CLIENT(ADMIN): Import data into rucio"""
         file_path = 'data_import.json'
-        data = {'rses': {rse_name_generator(): {'country_name': 'test'}}}
+        rses = {rse['rse']: rse for rse in self.rse_client.list_rses()}
+        rses[rse_name_generator()] = {'country_name': 'test'}
+        data = {'rses': rses}
         with open(file_path, 'w+') as file:
-            file.write(json.dumps(data))
+            file.write(render_json(**data))
         cmd = 'rucio-admin data import {0}'.format(file_path)
         exitcode, out, err = execute(cmd)
         nose.tools.assert_not_equal(re.search('Data successfully imported', out), None)
         remove(file_path)
-        # reset RSEs that got deleted
-        db_session = session.get_session()
-        for rse in db_session.query(models.RSE).all():
-            rse.deleted = False
-            rse.deleted_at = None
-            rse.save(session=db_session)
-            add_rse_attribute(rse_id=rse['id'], key=rse['rse'], value=True, session=db_session)
-        db_session.commit()
 
     def test_export_data(self):
         """ CLIENT(ADMIN): Export data from rucio"""
diff --git a/lib/rucio/tests/test_utils.py b/lib/rucio/tests/test_utils.py
--- a/lib/rucio/tests/test_utils.py
+++ b/lib/rucio/tests/test_utils.py
@@ -12,6 +12,7 @@
  PY3K COMPATIBLE
 '''
 
+import datetime
 import unittest
 import tempfile
 
@@ -60,8 +61,8 @@ def test_utils_adler32(self):
     def test_parse_did_filter_string(self):
         """(COMMON/UTILS): test parsing of did filter string"""
         test_cases = [{
-            'input': 'type=all,length=3,length>4,length>=6,length<=7,  test=b',
-            'expected_filter': {'length': 3, 'length.gt': 4, 'length.gte': 6, 'length.lte': 7, 'test': 'b'},
+            'input': 'type=all,length=3,length>4,length>=6,length<=7,  test=b, created_after=1900-01-01T00:00:00.000Z',
+            'expected_filter': {'length': 3, 'length.gt': 4, 'length.gte': 6, 'length.lte': 7, 'test': 'b', 'created_after': datetime.datetime.strptime('1900-01-01T00:00:00.000Z', '%Y-%m-%dT%H:%M:%S.%fZ')},
             'expected_type': 'all'
         }, {
             'input': 'type=FILE',