Repository: rucio/rucio

Issue:
Stale files after test
Motivation
----------

Properly remove leftover stale files when running tests.



Discussion:

Resolving patch:
diff --git a/lib/rucio/common/dumper/consistency.py b/lib/rucio/common/dumper/consistency.py
--- a/lib/rucio/common/dumper/consistency.py
+++ b/lib/rucio/common/dumper/consistency.py
@@ -1,16 +1,25 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2015-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
-# - Fernando Lopez, <felopez@cern.ch>, 2015
+# - Fernando Lopez <fernando.e.lopez@gmail.com>, 2015
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2019
 #
 # PY3K COMPATIBLE
 
-from rucio.common import dumper
-from rucio.common.dumper import error, DUMPS_CACHE_DIR, data_models, path_parsing
 import datetime
 import logging
 import os
@@ -18,6 +27,9 @@
 import subprocess
 import tempfile
 
+from rucio.common import dumper
+from rucio.common.dumper import error, DUMPS_CACHE_DIR, data_models, path_parsing
+
 
 subcommands = ['consistency', 'consistency-manual']
 
@@ -35,9 +47,9 @@ def dump(cls, subcommand, ddm_endpoint, storage_dump, prev_date_fname=None, next
         logger = logging.getLogger('auditor.consistency')
         if subcommand == 'consistency':
             prev_date_fname = data_models.Replica.download(
-                ddm_endpoint, prev_date)
+                ddm_endpoint, prev_date, cache_dir=cache_dir)
             next_date_fname = data_models.Replica.download(
-                ddm_endpoint, next_date)
+                ddm_endpoint, next_date, cache_dir=cache_dir)
             assert prev_date_fname is not None
             assert next_date_fname is not None
         else:
diff --git a/lib/rucio/core/account.py b/lib/rucio/core/account.py
--- a/lib/rucio/core/account.py
+++ b/lib/rucio/core/account.py
@@ -1,19 +1,27 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2012-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
-# - Thomas Beermann, <thomas.beermann@cern.ch>, 2012
-# - Angelos Molfetas, <angelos.molfetas@cern.ch>, 2012
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2013, 2017
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2015
-# - Martin Barisits, <martin.barisits@cern.ch>, 2014
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2014-2019
-# - Joaquin Bogado, <joaquin.bogado@cern.ch>, 2015
-# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018-2019
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2012
+# - Angelos Molfetas <Angelos.Molfetas@cern.ch>, 2012
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2019
+# - Vincent Garonne <vgaronne@gmail.com>, 2012-2015
+# - Martin Barisits <martin.barisits@cern.ch>, 2014
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2015
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 #
 # PY3K COMPATIBLE
 
@@ -254,7 +262,8 @@ def add_account_attribute(account, key, value, session=None):
            or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
            or match('.*IntegrityError.*UNIQUE constraint failed: account_attr_map.account, account_attr_map.key.*', error.args[0]) \
            or error.args[0] == "(IntegrityError) column account/key is not unique" \
-           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]):
+           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
+           or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]):
             raise exception.Duplicate('Key {0} already exist for account {1}!'.format(key, account))
     except Exception:
         raise exception.RucioException(str(format_exc()))


Test patch:
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
--- a/lib/rucio/tests/test_bin_rucio.py
+++ b/lib/rucio/tests/test_bin_rucio.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,17 +14,17 @@
 #
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2019
 # - Angelos Molfetas <Angelos.Molfetas@cern.ch>, 2012
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
 # - Cheng-Hsi Chao <cheng-hsi.chao@cern.ch>, 2014
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2015
-# - Martin Barisits <martin.barisits@cern.ch>, 2015-2016
+# - Martin Barisits <martin.barisits@cern.ch>, 2015-2019
 # - Frank Berghaus <frank.berghaus@cern.ch>, 2017-2018
 # - Tobias Wegner <twegner@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 #
 # PY3K COMPATIBLE
 
@@ -660,6 +660,7 @@ def test_download_metalink_file(self):
             metalink_file.write(replica_file)
         cmd = 'rucio download --dir /tmp --metalink {0}'.format(metalink_file_path)
         exitcode, out, err = execute(cmd)
+        remove(metalink_file_path)
         cmd = 'ls /tmp/{0}'.format(scope)
         exitcode, out, err = execute(cmd)
         nose.tools.assert_not_equal(re.search(tmp_file_name, out), None)
diff --git a/lib/rucio/tests/test_download.py b/lib/rucio/tests/test_download.py
--- a/lib/rucio/tests/test_download.py
+++ b/lib/rucio/tests/test_download.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,10 +14,13 @@
 #
 # Authors:
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
+# - Tobias Wegner <twegner@cern.ch>, 2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2019
 #
 # PY3K COMPATIBLE
 
 import logging
+import shutil
 
 import nose.tools
 import os.path
@@ -39,35 +42,37 @@ def setup(self):
         self.upload_client = UploadClient(_client=self.client, logger=logger)
         self.download_client = DownloadClient(client=self.client, logger=logger)
 
-    def create_and_upload_tmp_file(self, rse, scope='mock'):
-        file_path = file_generator()
-        item = {'path': file_path,
-                'rse': rse,
-                'did_scope': scope,
-                'did_name': os.path.basename(file_path),
-                'guid': generate_uuid()}
+        self.file_path = file_generator()
+        self.scope = 'mock'
+        self.name = os.path.basename(self.file_path)
+        self.rse = 'MOCK4'
+        self.guid = generate_uuid()
+
+        item = {'path': self.file_path,
+                'rse': self.rse,
+                'did_scope': self.scope,
+                'did_name': self.name,
+                'guid': self.guid}
         nose.tools.assert_equal(self.upload_client.upload([item]), 0)
-        return item
+
+    def teardown(self):
+        shutil.rmtree('mock')
 
     def test_download_item(self):
-        """ DOWNLOAD (CLIENT): download DIDs. """
-        item = self.create_and_upload_tmp_file('MOCK4')
-        scope = item['did_scope']
-        name = item['did_name']
-        uuid = item['guid']
+        """ DOWNLOAD (CLIENT): Download DIDs """
 
         # Download specific DID
-        result = self.download_client.download_dids([{'did': '%s:%s' % (scope, name)}])
+        result = self.download_client.download_dids([{'did': '%s:%s' % (self.scope, self.name)}])
         nose.tools.assert_true(result)
 
         # Download with wildcard
-        result = self.download_client.download_dids([{'did': '%s:%s' % (scope, name[:-2] + '*')}])
+        result = self.download_client.download_dids([{'did': '%s:%s' % (self.scope, self.name[:-2] + '*')}])
         nose.tools.assert_true(result)
 
         # Download with filter
-        result = self.download_client.download_dids([{'filters': {'guid': uuid, 'scope': scope}}])
+        result = self.download_client.download_dids([{'filters': {'guid': self.guid, 'scope': self.scope}}])
         nose.tools.assert_true(result)
 
         # Download with wildcard and name
-        result = self.download_client.download_dids([{'did': '%s:%s' % (scope, '*'), 'filters': {'guid': uuid}}])
+        result = self.download_client.download_dids([{'did': '%s:%s' % (self.scope, '*'), 'filters': {'guid': self.guid}}])
         nose.tools.assert_true(result)