Repository: rucio/rucio

Issue:
 Adding existing files to archive fails
Motivation
----------

```
Traceback (most recent call last):
 File "archive.py", line 21, in <module>
   c.add_files_to_archive(name=zipfile,scope='user.walkerr',files=flist)
 File "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/1.14.11/lib/python2.7/site-packages/rucio/client/didclient.py", line 281, in add_files_to_archive
   return self.attach_dids(scope=scope, name=name, dids=files)
 File "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/1.14.11/lib/python2.7/site-packages/rucio/client/didclient.py", line 187, in attach_dids
   raise exc_cls(exc_msg)
rucio.common.exception.RucioException: An unknown exception occurred.
Details: [u'(cx_Oracle.IntegrityError) ORA-00001: unique constraint (ATLAS_RUCIO.DIDS_PK) violated\n']
```



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -230,20 +230,23 @@ def __add_files_to_archive(scope, name, files, account, ignore_duplicate=False,
     :param ignore_duplicate: If True, ignore duplicate entries.
     :param session: The database session in use.
     """
-    # lookup for files
+    # lookup for existing files
     files_query = session.query(models.DataIdentifier.scope, models.DataIdentifier.name,
                                 models.DataIdentifier.bytes, models.DataIdentifier.guid,
-                                models.DataIdentifier.events, models.DataIdentifier.availability,
+                                models.DataIdentifier.events,
+                                models.DataIdentifier.availability,
                                 models.DataIdentifier.adler32, models.DataIdentifier.md5).\
         filter(models.DataIdentifier.did_type == DIDType.FILE).\
         with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle')
 
     file_condition = []
     for file in files:
-        file_condition.append(and_(models.DataIdentifier.scope == file['scope'], models.DataIdentifier.name == file['name']))
+        file_condition.append(and_(models.DataIdentifier.scope == file['scope'],
+                                   models.DataIdentifier.name == file['name']))
 
-    existing_content = []
+    existing_content, existing_files = [], {}
     if ignore_duplicate:
+        # lookup for existing content
         content_query = session.query(models.ConstituentAssociation.scope,
                                       models.ConstituentAssociation.name,
                                       models.ConstituentAssociation.child_scope,
@@ -258,55 +261,63 @@ def __add_files_to_archive(scope, name, files, account, ignore_duplicate=False,
         for row in content_query.filter(or_(*content_condition)):
             existing_content.append(row)
 
-    contents = []
     for row in files_query.filter(or_(*file_condition)):
-        contents.append({'child_scope': row.scope,
-                         'child_name': row.name,
-                         'scope': scope,
-                         'name': name,
-                         'bytes': row.bytes,
-                         'adler32': row.adler32,
-                         'md5': row.md5,
-                         'guid': row.guid,
-                         'length': row.events})
-
-    new_files, existing_files = [], []
+        existing_files[row.scope + ':' + row.name] = {'child_scope': row.scope,
+                                                      'child_name': row.name,
+                                                      'scope': scope,
+                                                      'name': name,
+                                                      'bytes': row.bytes,
+                                                      'adler32': row.adler32,
+                                                      'md5': row.md5,
+                                                      'guid': row.guid,
+                                                      'length': row.events}
+
+    contents = []
+    new_files, existing_files_condition = [], []
     for file in files:
-        if not existing_content or (scope, name, file['scope'], file['name']) not in existing_content:
-            content = {'child_scope': file['scope'],
-                       'child_name': file['name'],
-                       'scope': scope,
-                       'name': name,
-                       'bytes': file['bytes'],
-                       'adler32': file.get('adler32'),
-                       'md5': file.get('md5'),
-                       'guid': file.get('guid'),
-                       'length': file.get('events')}
-            if content not in contents:
-                file['constituent'] = True
-                file['did_type'] = DIDType.FILE
-                file['account'] = account
-                for key in file.get('meta', {}):
-                    file[key] = file['meta'][key]
-                new_files.append(file)
-                contents.append(content)
-            else:
-                existing_files.append(and_(models.DataIdentifier.scope == file['scope'],
-                                           models.DataIdentifier.name == file['name']))
+
+        if file['scope'] + ':' + file['name'] not in existing_files:
+            # For non existing files
+            # Add them to the content
+            contents.append({'child_scope': file['scope'],
+                             'child_name': file['name'],
+                             'scope': scope,
+                             'name': name,
+                             'bytes': file['bytes'],
+                             'adler32': file.get('adler32'),
+                             'md5': file.get('md5'),
+                             'guid': file.get('guid'),
+                             'length': file.get('events')})
+
+            file['constituent'] = True
+            file['did_type'] = DIDType.FILE
+            file['account'] = account
+            for key in file.get('meta', {}):
+                file[key] = file['meta'][key]
+            # Prepare new file registrations
+            new_files.append(file)
+        else:
+            # For existing files
+            # Prepare the dids updates
+            existing_files_condition.append(and_(models.DataIdentifier.scope == file['scope'],
+                                                 models.DataIdentifier.name == file['name']))
+            # Check if they are not already in the content
+            if not existing_content or (scope, name, file['scope'], file['name']) not in existing_content:
+                contents.append(existing_files[file['scope'] + ':' + file['name']])
 
     # insert into archive_contents
     try:
         new_files and session.bulk_insert_mappings(models.DataIdentifier, new_files)
-        if existing_files:
+        if existing_files_condition:
             session.query(models.DataIdentifier).\
                 filter(models.DataIdentifier.did_type == DIDType.FILE).\
-                filter(or_(*existing_files)).update({'constituent': True})
+                filter(or_(*existing_files_condition)).update({'constituent': True})
 
+        print contents
         contents and session.bulk_insert_mappings(models.ConstituentAssociation, contents)
         session.flush()
     except IntegrityError, error:
         raise exception.RucioException(error.args)
-    pass
 
 
 @transactional_session


Test patch:
diff --git a/lib/rucio/tests/test_archive.py b/lib/rucio/tests/test_archive.py
--- a/lib/rucio/tests/test_archive.py
+++ b/lib/rucio/tests/test_archive.py
@@ -34,7 +34,7 @@ def __init__(self):
     def test_add_and_list_archive(self):
         """  ARCHIVE (CLIENT): Add files to archive and list the content."""
         scope, rse = 'mock', 'MOCK'
-        archive_file = 'file_' + generate_uuid() + '.zip'
+        archive_files = ['file_' + generate_uuid() + '.zip' for _ in range(2)]
         files = []
         for i in range(10):
             files.append({'scope': scope, 'name': 'lfn.%s' % str(generate_uuid()),
@@ -42,17 +42,16 @@ def test_add_and_list_archive(self):
                           'adler32': '0cc737eb',
                           'type': 'FILE',
                           'meta': {'guid': str(generate_uuid())}})
+        for archive_file in archive_files:
 
-        self.replica_client.add_replicas(rse=rse, files=[{'scope': scope,
-                                                          'name': archive_file,
-                                                          'bytes': 1,
-                                                          'adler32': '0cc737eb'}])
+            self.replica_client.add_replicas(rse=rse, files=[{'scope': scope,
+                                                              'name': archive_file,
+                                                              'bytes': 1,
+                                                              'adler32': '0cc737eb'}])
 
-        self.did_client.add_files_to_archive(scope=scope,
-                                             name=archive_file,
-                                             files=files)
+            self.did_client.add_files_to_archive(scope=scope, name=archive_file, files=files)
 
-        content = [fil for fil in self.did_client.list_archive_content(scope=scope,
-                                                                       name=archive_file)]
+            content = [fil for fil in self.did_client.list_archive_content(scope=scope,
+                                                                           name=archive_file)]
 
-        assert_equal(len(content), 10)
+            assert_equal(len(content), 10)