Repository: rucio/rucio
Issue: addfile method fails with internal error is a file is attached to a container

Motivation
----------
The addfile method in `dirac.py` fails with internal error is a file is attached to a container or if a container tries to be attached to a dataset. A proper specific exception should be raised in that case.


Modification
------------
Introduce a new exception



Changed files:
lib/rucio/core/dirac.py, lib/rucio/web/rest/flaskapi/v1/dirac.py, lib/rucio/web/rest/webpy/v1/dirac.py

diff --git a/lib/rucio/core/dirac.py b/lib/rucio/core/dirac.py
index caf8bf099..a738b7627 100644
--- a/lib/rucio/core/dirac.py
+++ b/lib/rucio/core/dirac.py
@@ -22,12 +22,13 @@ from __future__ import print_function
 from sqlalchemy.orm.exc import NoResultFound
 from rucio.db.sqla import models
 from rucio.db.sqla.session import transactional_session, read_session
-from rucio.db.sqla.constants import DIDType, RuleGrouping
-from rucio.common.exception import InvalidType
+from rucio.db.sqla.constants import DIDType
+from rucio.common.exception import InvalidType, UnsupportedOperation
 from rucio.common.types import InternalScope, InternalAccount
 from rucio.common.utils import extract_scope
 from rucio.core.did import add_did, attach_dids_to_dids
 from rucio.core.replica import add_replicas
+from rucio.core.rule import add_rule
 from rucio.core.scope import list_scopes
 
 
@@ -41,11 +42,11 @@ def _exists(scope, name, session=None):
     :session: The session used
     """
     try:
-        session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\\
+        res = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\\
             with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one()
-        return True
+        return True, res.did_type
     except NoResultFound:
-        return False
+        return False, None
 
 
 @transactional_session
@@ -71,7 +72,9 @@ def add_files(lfns, account, ignore_availability, session=None):
         filename = lfn['lfn']
         lfn_scope, _ = extract_scope(filename, scopes)
         lfn_scope = InternalScope(lfn_scope)
-        if _exists(lfn_scope, filename):
+
+        exists, did_type = _exists(lfn_scope, filename)
+        if exists:
             continue
 
         # Get all the ascendants of the file
@@ -84,7 +87,10 @@ def add_files(lfns, account, ignore_availability, session=None):
         dsn_name = lpns[0]
         dsn_scope, _ = extract_scope(dsn_name, scopes)
         dsn_scope = InternalScope(dsn_scope)
-        if (dsn_name not in exist_lfn) and (not _exists(dsn_scope, dsn_name)):
+        exists, did_type = _exists(dsn_scope, dsn_name)
+        if exists and did_type == DIDType.CONTAINER:
+            raise UnsupportedOperation('Cannot create %s as dataset' % dsn_name)
+        if (dsn_name not in exist_lfn) and not exists:
             print('Will create %s' % dsn_name)
             add_did(scope=dsn_scope,
                     name=dsn_name,
@@ -92,7 +98,7 @@ def add_files(lfns, account, ignore_availability, session=None):
                     account=InternalAccount(account),
                     statuses=None,
                     meta=None,
-                    rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account), 'lifetime': None, 'GROUPING': RuleGrouping.NONE}],
+                    rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account), 'lifetime': None, 'GROUPING': 'NONE'}],
                     lifetime=None,
                     dids=None,
                     rse_id=None,
@@ -122,13 +128,26 @@ def add_files(lfns, account, ignore_availability, session=None):
                      account=InternalAccount(account),
                      ignore_availability=ignore_availability,
                      session=session)
+        add_rule(dids=[{'scope': lfn_scope, 'name': filename}],
+                 account=InternalAccount(account),
+                 copies=1,
+                 rse_expression=lfn['rse'],
+                 grouping=None,
+                 weight=None,
+                 lifetime=86400,
+                 locked=None,
+                 subscription_id=None,
+                 session=session)
         attachments.append({'scope': dsn_scope, 'name': dsn_name, 'dids': [{'scope': lfn_scope, 'name': filename}]})
 
         # Now loop over the ascendants of the dataset and created them
         for lpn in lpns[1:]:
             child_scope, _ = extract_scope(lpn, scopes)
             child_scope = InternalScope(child_scope)
-            if (lpn not in exist_lfn) and (not _exists(child_scope, lpn)):
+            exists, did_type = _exists(child_scope, lpn)
+            if exists and did_type == DIDType.DATASET:
+                raise UnsupportedOperation('Cannot create %s as container' % lpn)
+            if (lpn not in exist_lfn) and not exists:
                 print('Will create %s' % lpn)
                 add_did(scope=child_scope,
                         name=lpn,
diff --git a/lib/rucio/web/rest/flaskapi/v1/dirac.py b/lib/rucio/web/rest/flaskapi/v1/dirac.py
index 750e14623..56ad9d489 100755
--- a/lib/rucio/web/rest/flaskapi/v1/dirac.py
+++ b/lib/rucio/web/rest/flaskapi/v1/dirac.py
@@ -30,7 +30,7 @@ from rucio.api.dirac import add_files
 from rucio.common.exception import (AccessDenied, DataIdentifierAlreadyExists, InvalidType,
                                     DatabaseException, Duplicate, InvalidPath,
                                     ResourceTemporaryUnavailable, RucioException,
-                                    RSENotFound)
+                                    RSENotFound, UnsupportedOperation)
 from rucio.common.utils import generate_http_error_flask
 
 
@@ -46,8 +46,10 @@ class AddFiles(MethodView):
 
         HTTP Error:
             401 Unauthorized
+            405 Method Not Allowed
             409 Conflict
             500 Internal Error
+            503 Service Unavailable
         """
         try:
             json_data = loads(request.data)
@@ -62,6 +64,8 @@ class AddFiles(MethodView):
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
         except RSENotFound as error:
             return generate_http_error_flask(404, 'RSENotFound', error.args[0])
+        except UnsupportedOperation as error:
+            return generate_http_error_flask(405, 'UnsupportedOperation', error.args[0])
         except DataIdentifierAlreadyExists as error:
             return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
         except Duplicate as error:
diff --git a/lib/rucio/web/rest/webpy/v1/dirac.py b/lib/rucio/web/rest/webpy/v1/dirac.py
index 0458ad31b..00d546b46 100755
--- a/lib/rucio/web/rest/webpy/v1/dirac.py
+++ b/lib/rucio/web/rest/webpy/v1/dirac.py
@@ -26,7 +26,7 @@ from rucio.api.dirac import add_files
 from rucio.common.exception import (AccessDenied, DataIdentifierAlreadyExists, InvalidType,
                                     DatabaseException, Duplicate, InvalidPath,
                                     ResourceTemporaryUnavailable, RucioException,
-                                    RSENotFound)
+                                    RSENotFound, UnsupportedOperation)
 from rucio.common.utils import generate_http_error, parse_response
 from rucio.web.rest.common import rucio_loadhook, rucio_unloadhook, RucioController
 
@@ -44,8 +44,10 @@ class AddFiles(RucioController):
 
         HTTP Error:
             401 Unauthorized
+            405 Method Not Allowed
             409 Conflict
             500 Internal Error
+            503 Service Unavailable
         """
         json_data = data()
         try:
@@ -59,6 +61,8 @@ class AddFiles(RucioController):
             raise generate_http_error(400, 'InvalidPath', error.args[0])
         except AccessDenied as error:
             raise generate_http_error(401, 'AccessDenied', error.args[0])
+        except UnsupportedOperation as error:
+            raise generate_http_error(405, 'UnsupportedOperation', error.args[0])
         except Duplicate as error:
             raise generate_http_error(409, 'Duplicate', error.args[0])
         except DataIdentifierAlreadyExists as error:
@@ -70,6 +74,7 @@ class AddFiles(RucioController):
         except ResourceTemporaryUnavailable as error:
             raise generate_http_error(503, 'ResourceTemporaryUnavailable', error.args[0])
         except RucioException as error:
+            print(format_exc())
             raise generate_http_error(500, error.__class__.__name__, error.args[0])
         except Exception as error:
             print(format_exc())