Repository: rucio/rucio

Issue:
adding a replica with non-existent scope should raise ScopeNotFound
Motivation
----------

I would like to automatically add scopes which don't already exist when they are used to register new replicas with something like
```
try:
    add_replica(...)
except ScopeNotFound:
    add_scope(...)
    add_replica(...)
```
But this actually raises a generic rucio exception
```
[2020-09-02 16:01:23,223] [aCTLDMXRegister.py:223] [WARNING] - Rucio exception: An unknown exception occurred.
Details: ['(psycopg2.IntegrityError) insert or update on table "dids" violates foreign key constraint "DIDS_SCOPE_FK"\nDETAIL:  Key (scope)=(test-v2.2) is not present in table "scopes".\n']
```

Modification
------------

Missing scope should raise ScopeNotFound.

adding a replica with non-existent scope should raise ScopeNotFound
Motivation
----------

I would like to automatically add scopes which don't already exist when they are used to register new replicas with something like
```
try:
    add_replica(...)
except ScopeNotFound:
    add_scope(...)
    add_replica(...)
```
But this actually raises a generic rucio exception
```
[2020-09-02 16:01:23,223] [aCTLDMXRegister.py:223] [WARNING] - Rucio exception: An unknown exception occurred.
Details: ['(psycopg2.IntegrityError) insert or update on table "dids" violates foreign key constraint "DIDS_SCOPE_FK"\nDETAIL:  Key (scope)=(test-v2.2) is not present in table "scopes".\n']
```

Modification
------------

Missing scope should raise ScopeNotFound.



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1182,8 +1182,20 @@ def __bulk_add_new_file_dids(files, account, dataset_meta=None, session=None):
     try:
         session.flush()
     except IntegrityError as error:
+        if match('.*IntegrityError.*02291.*integrity constraint.*DIDS_SCOPE_FK.*violated - parent key not found.*', error.args[0]) \
+                or match('.*IntegrityError.*FOREIGN KEY constraint failed.*', error.args[0]) \
+                or match('.*IntegrityError.*1452.*Cannot add or update a child row: a foreign key constraint fails.*', error.args[0]) \
+                or match('.*IntegrityError.*02291.*integrity constraint.*DIDS_SCOPE_FK.*violated - parent key not found.*', error.args[0]) \
+                or match('.*IntegrityError.*insert or update on table.*violates foreign key constraint "DIDS_SCOPE_FK".*', error.args[0]) \
+                or match('.*ForeignKeyViolation.*insert or update on table.*violates foreign key constraint.*', error.args[0]) \
+                or match('.*IntegrityError.*foreign key constraints? failed.*', error.args[0]):
+            raise exception.ScopeNotFound('Scope not found!')
+
         raise exception.RucioException(error.args)
     except DatabaseError as error:
+        if match('.*(DatabaseError).*ORA-14400.*inserted partition key does not map to any partition.*', error.args[0]):
+            raise exception.ScopeNotFound('Scope not found!')
+
         raise exception.RucioException(error.args)
     except FlushError as error:
         if match('New instance .* with identity key .* conflicts with persistent instance', error.args[0]):
diff --git a/lib/rucio/web/rest/flaskapi/v1/replica.py b/lib/rucio/web/rest/flaskapi/v1/replica.py
--- a/lib/rucio/web/rest/flaskapi/v1/replica.py
+++ b/lib/rucio/web/rest/flaskapi/v1/replica.py
@@ -37,8 +37,8 @@
 from geoip2.errors import AddressNotFoundError
 from six import string_types
 
-from rucio.api.replica import (add_replicas, list_replicas, list_dataset_replicas, list_dataset_replicas_bulk,
-                               delete_replicas,
+from rucio.api.replica import (add_replicas, list_replicas, list_dataset_replicas,
+                               list_dataset_replicas_bulk, delete_replicas,
                                get_did_from_pfns, update_replicas_states,
                                declare_bad_file_replicas, add_bad_pfns, get_suspicious_files,
                                declare_suspicious_file_replicas, list_bad_replicas_status,
@@ -49,7 +49,8 @@
 from rucio.common.exception import (AccessDenied, DataIdentifierAlreadyExists, InvalidType,
                                     DataIdentifierNotFound, Duplicate, InvalidPath,
                                     ResourceTemporaryUnavailable, RucioException,
-                                    RSENotFound, UnsupportedOperation, ReplicaNotFound, InvalidObject)
+                                    RSENotFound, UnsupportedOperation, ReplicaNotFound,
+                                    InvalidObject, ScopeNotFound)
 from rucio.common.replica_sorter import sort_random, sort_geoip, sort_closeness, sort_dynamic, sort_ranking
 from rucio.common.utils import parse_response, APIEncoder, render_json_list
 from rucio.db.sqla.constants import BadFilesStatus
@@ -197,6 +198,7 @@ def post(self):
         :status 400: Invalid Path.
         :status 401: Invalid auth token.
         :status 404: RSE not found.
+        :status 404: Scope not found.
         :status 409: Replica already exists.
         :status 409: DID already exists.
         :status 503: Resource Temporary Unavailable.
@@ -221,6 +223,8 @@ def post(self):
             return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
         except RSENotFound as error:
             return generate_http_error_flask(404, 'RSENotFound', error.args[0])
+        except ScopeNotFound as error:
+            return generate_http_error_flask(404, 'ScopeNotFound', error.args[0])
         except ResourceTemporaryUnavailable as error:
             return generate_http_error_flask(503, 'ResourceTemporaryUnavailable', error.args[0])
         except RucioException as error:
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -51,7 +51,8 @@
 from rucio.common.exception import (AccessDenied, DataIdentifierAlreadyExists, InvalidType,
                                     DataIdentifierNotFound, Duplicate, InvalidPath,
                                     ResourceTemporaryUnavailable, RucioException,
-                                    RSENotFound, UnsupportedOperation, ReplicaNotFound, InvalidObject)
+                                    RSENotFound, UnsupportedOperation, ReplicaNotFound,
+                                    InvalidObject, ScopeNotFound)
 from rucio.common.replica_sorter import sort_random, sort_geoip, sort_closeness, sort_dynamic, sort_ranking
 from rucio.common.schema import get_schema_value
 from rucio.common.utils import parse_response, APIEncoder, render_json_list
@@ -228,6 +229,8 @@ def POST(self):
             raise generate_http_error(409, 'DataIdentifierAlreadyExists', error.args[0])
         except RSENotFound as error:
             raise generate_http_error(404, 'RSENotFound', error.args[0])
+        except ScopeNotFound as error:
+            raise generate_http_error(404, 'ScopeNotFound', error.args[0])
         except ResourceTemporaryUnavailable as error:
             raise generate_http_error(503, 'ResourceTemporaryUnavailable', error.args[0])
         except RucioException as error:


Test patch:
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -47,7 +47,7 @@
 from rucio.client.ruleclient import RuleClient
 from rucio.common.config import config_get, config_get_bool
 from rucio.common.exception import (DataIdentifierNotFound, AccessDenied, UnsupportedOperation,
-                                    RucioException, ReplicaIsLocked, ReplicaNotFound)
+                                    RucioException, ReplicaIsLocked, ReplicaNotFound, ScopeNotFound)
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import generate_uuid, clean_surls
 from rucio.core.did import add_did, attach_dids, get_did, set_status, list_files, get_did_atime
@@ -838,6 +838,12 @@ def test_add_list_replicas(self):
         for i in range(nbfiles):
             assert 'MOCK3' in replicas[i]['rses']
 
+    def test_add_replica_scope_not_found(self):
+        """ REPLICA (CLIENT): Add replica with missing scope """
+        files = [{'scope': 'nonexistingscope', 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb'}]
+        with pytest.raises(ScopeNotFound):
+            self.replica_client.add_replicas(rse='MOCK', files=files)
+
     def test_delete_replicas(self):
         """ REPLICA (CLIENT): Add and delete file replicas """
         tmp_scope = 'mock'