Repository: rucio/rucio

Issue:
REST/Client : set-metadata should raise DataIdentifierNotFound when the DID doesn't exist
Motivation
----------
```
rucio set-metadata --did blahblah:qwerty --key lifetime --value 100000
2018-06-19 15:13:04,853 ERROR   The resource doesn't support the requested operation.
Details: lifetime for blahblah:qwerty cannot be updated
```
docker builds fail with pip install error (pycurl/m2crypto)
Motivation
----------
Currently the docker server and daemon builds fail while installing rucio because of the pycurl dependency needing openssl-devel installed.

Modification
------------
add yum install openssl-devel.x86_64


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -1176,6 +1176,12 @@ def set_metadata(scope, name, key, value, type=None, did=None,
     :param recursive: Option to propagate the metadata change to content.
     :param session: The database session in use.
     """
+    try:
+        rowcount = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\
+            with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one()
+    except NoResultFound:
+        raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name))
+
     if key == 'lifetime':
         try:
             expired_at = None
diff --git a/lib/rucio/web/rest/flaskapi/v1/did.py b/lib/rucio/web/rest/flaskapi/v1/did.py
--- a/lib/rucio/web/rest/flaskapi/v1/did.py
+++ b/lib/rucio/web/rest/flaskapi/v1/did.py
@@ -19,7 +19,7 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2016
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2018
 # - Yun-Pin Sun <yun-pin.sun@cern.ch>, 2013
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2015
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
 # - Martin Baristis <martin.barisits@cern.ch>, 2014-2015
 
 from json import dumps, loads
@@ -93,9 +93,9 @@ def get(self, scope):
             for did in scope_list(scope=scope, name=name, recursive=recursive):
                 data += render_json(**did) + '\n'
             return Response(data, content_type='application/x-json-stream')
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -155,11 +155,11 @@ def get(self, scope):
             for did in list_dids(scope=scope, filters=filters, type=type, long=long):
                 data += dumps(did) + '\n'
             return Response(data, content_type='application/x-json-stream')
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except KeyNotFound, error:
+        except KeyNotFound as error:
             return generate_http_error_flask(404, 'KeyNotFound', error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -213,21 +213,21 @@ def post(self):
         try:
             print json_data
             add_dids(json_data, issuer=request.environ.get('issuer'))
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except DuplicateContent, error:
+        except DuplicateContent as error:
             return generate_http_error_flask(409, 'DuplicateContent', error.args[0])
-        except DataIdentifierAlreadyExists, error:
+        except DataIdentifierAlreadyExists as error:
             return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except DatabaseException, error:
+        except DatabaseException as error:
             return generate_http_error_flask(500, 'DatabaseException', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
         return "Created", 201
@@ -257,19 +257,19 @@ def post(self):
 
         try:
             attach_dids_to_dids(attachments=attachments, ignore_duplicate=ignore_duplicate, issuer=request.environ.get('issuer'))
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except DuplicateContent, error:
+        except DuplicateContent as error:
             return generate_http_error_flask(409, 'DuplicateContent', error.args[0])
-        except DataIdentifierAlreadyExists, error:
+        except DataIdentifierAlreadyExists as error:
             return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -316,13 +316,13 @@ def get(self, scope, name):
                 dynamic = True
             did = get_did(scope=scope, name=name, dynamic=dynamic)
             return Response(render_json(**did), content_type='application/json')
-        except ScopeNotFound, error:
+        except ScopeNotFound as error:
             return generate_http_error_flask(404, 'ScopeNotFound', error.args[0])
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -379,26 +379,26 @@ def post(self, scope, name):
                 rse = json_data['rse']
         except ValueError:
             return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter list')
-        except KeyError, error:
+        except KeyError as error:
             return generate_http_error_flask(400, 'ValueError', str(error))
 
         try:
             add_did(scope=scope, name=name, type=type, statuses=statuses, meta=meta, rules=rules, lifetime=lifetime, dids=dids, rse=rse, issuer=request.environ.get('issuer'))
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except DuplicateContent, error:
+        except DuplicateContent as error:
             return generate_http_error_flask(409, 'DuplicateContent', error.args[0])
-        except DataIdentifierAlreadyExists, error:
+        except DataIdentifierAlreadyExists as error:
             return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except DatabaseException, error:
+        except DatabaseException as error:
             return generate_http_error_flask(500, 'DatabaseException', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
         return "Created", 201
@@ -441,17 +441,17 @@ def put(self, scope, name):
 
         try:
             set_status(scope=scope, name=name, issuer=request.environ.get('issuer'), **kwargs)
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except UnsupportedStatus, error:
+        except UnsupportedStatus as error:
             return generate_http_error_flask(409, 'UnsupportedStatus', error.args[0])
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -497,11 +497,11 @@ def get(self, scope, name):
             for did in list_content(scope=scope, name=name):
                 data += render_json(**did) + '\n'
             return Response(data, content_type="application/x-json-stream")
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -546,19 +546,19 @@ def post(self, scope, name):
 
         try:
             attach_dids(scope=scope, name=name, attachment=json_data, issuer=request.environ.get('issuer'))
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except DuplicateContent, error:
+        except DuplicateContent as error:
             return generate_http_error_flask(409, 'DuplicateContent', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except RSENotFound, error:
+        except RSENotFound as error:
             return generate_http_error_flask(404, 'RSENotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -588,13 +588,13 @@ def delete(self, scope, name):
 
         try:
             detach_dids(scope=scope, name=name, dids=dids, issuer=request.environ.get('issuer'))
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -623,11 +623,11 @@ def get(self, scope, name):
             for did in list_content_history(scope=scope, name=name):
                 data += render_json(**did) + '\n'
             return Response(data, content_type="application/x-json-stream")
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -658,11 +658,11 @@ def get(self, scope, name):
             for file in list_files(scope=scope, name=name, long=long):
                 data += dumps(file) + "\n"
             return Response(data, content_type="application/x-json-stream")
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -688,11 +688,11 @@ def get(self, scope, name):
             for dataset in list_parent_dids(scope=scope, name=name):
                 data += render_json(**dataset) + "\n"
             return Response(data, content_type="application/x-json-stream")
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -717,11 +717,11 @@ def get(self, scope, name):
         try:
             meta = get_metadata(scope=scope, name=name)
             return Response(render_json(**meta), content_type='application/json')
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -735,6 +735,7 @@ def post(self, scope, name, key):
             201 Created
 
         HTTP Error:
+            400 Bad Request
             401 Unauthorized
             404 Not Found
             409 Conflict
@@ -755,17 +756,19 @@ def post(self, scope, name, key):
         try:
             set_metadata(scope=scope, name=name, key=key, value=value,
                          issuer=request.environ.get('issuer'), recursive=recursive)
-        except Duplicate, error:
+        except DataIdentifierNotFound as error:
+            return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
+        except Duplicate as error:
             return generate_http_error_flask(409, 'Duplicate', error.args[0])
-        except KeyNotFound, error:
+        except KeyNotFound as error:
             return generate_http_error_flask(400, 'KeyNotFound', error.args[0])
-        except InvalidMetadata, error:
+        except InvalidMetadata as error:
             return generate_http_error_flask(400, 'InvalidMetadata', error.args[0])
-        except InvalidValueForKey, error:
+        except InvalidValueForKey as error:
             return generate_http_error_flask(400, 'InvalidValueForKey', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
 
@@ -795,11 +798,11 @@ def get(self, scope, name):
             for rule in list_replication_rules({'scope': scope, 'name': name}):
                 data += dumps(rule, cls=APIEncoder) + '\n'
             return Response(data, content_type="application/x-json-stream")
-        except RuleNotFound, error:
+        except RuleNotFound as error:
             return generate_http_error_flask(404, 'RuleNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             return error, 500
 
 
@@ -825,9 +828,9 @@ def get(self, scope, name):
             for rule in list_associated_replication_rules_for_file(scope=scope, name=name):
                 data += dumps(rule, cls=APIEncoder) + '\n'
             return Response(data, content_type="application/x-json-stream")
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             return error, 500
 
 
@@ -852,11 +855,11 @@ def get(self, guid):
             for dataset in get_dataset_by_guid(guid):
                 data += dumps(dataset, cls=APIEncoder) + '\n'
             return Response(data, content_type="application/x-json-stream")
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             return error, 500
 
 
@@ -886,21 +889,21 @@ def post(self, input_scope, input_name, output_scope, output_name, nbfiles):
         """
         try:
             create_did_sample(input_scope=input_scope, input_name=input_name, output_scope=output_scope, output_name=output_name, issuer=request.environ.get('issuer'), nbfiles=nbfiles)
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except DuplicateContent, error:
+        except DuplicateContent as error:
             return generate_http_error_flask(409, 'DuplicateContent', error.args[0])
-        except DataIdentifierAlreadyExists, error:
+        except DataIdentifierAlreadyExists as error:
             return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except DatabaseException, error:
+        except DatabaseException as error:
             return generate_http_error_flask(500, 'DatabaseException', error.args)
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
         return "Created", 201
@@ -929,9 +932,9 @@ def get(self):
             for did in list_new_dids(type):
                 data += dumps(did, cls=APIEncoder) + '\n'
             return Response(data, content_type="application/x-json-stream")
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             return error, 500
 
 
@@ -962,21 +965,21 @@ def post(self):
 
         try:
             resurrect(dids=dids, issuer=request.environ.get('issuer'))
-        except DataIdentifierNotFound, error:
+        except DataIdentifierNotFound as error:
             return generate_http_error_flask(404, 'DataIdentifierNotFound', error.args[0])
-        except DuplicateContent, error:
+        except DuplicateContent as error:
             return generate_http_error_flask(409, 'DuplicateContent', error.args[0])
-        except DataIdentifierAlreadyExists, error:
+        except DataIdentifierAlreadyExists as error:
             return generate_http_error_flask(409, 'DataIdentifierAlreadyExists', error.args[0])
-        except AccessDenied, error:
+        except AccessDenied as error:
             return generate_http_error_flask(401, 'AccessDenied', error.args[0])
-        except UnsupportedOperation, error:
+        except UnsupportedOperation as error:
             return generate_http_error_flask(409, 'UnsupportedOperation', error.args[0])
-        except DatabaseException, error:
+        except DatabaseException as error:
             return generate_http_error_flask(500, 'DatabaseException', error.args[0])
-        except RucioException, error:
+        except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
-        except Exception, error:
+        except Exception as error:
             print format_exc()
             return error, 500
         return "Created", 201
diff --git a/lib/rucio/web/rest/webpy/v1/did.py b/lib/rucio/web/rest/webpy/v1/did.py
--- a/lib/rucio/web/rest/webpy/v1/did.py
+++ b/lib/rucio/web/rest/webpy/v1/did.py
@@ -19,7 +19,7 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2016
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2018
 # - Yun-Pin Sun <yun-pin.sun@cern.ch>, 2013
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2015
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
 # - Martin Baristis <martin.barisits@cern.ch>, 2014-2015
 
 from json import dumps, loads
@@ -581,6 +581,7 @@ def POST(self, scope, name, key):
             201 Created
 
         HTTP Error:
+            400 Bad Request
             401 Unauthorized
             404 Not Found
             409 Conflict
@@ -601,6 +602,8 @@ def POST(self, scope, name, key):
         try:
             set_metadata(scope=scope, name=name, key=key, value=value,
                          issuer=ctx.env.get('issuer'), recursive=recursive)
+        except DataIdentifierNotFound as error:
+            raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
         except Duplicate as error:
             raise generate_http_error(409, 'Duplicate', error.args[0])
         except KeyNotFound as error:


Test patch:
diff --git a/lib/rucio/tests/test_did.py b/lib/rucio/tests/test_did.py
--- a/lib/rucio/tests/test_did.py
+++ b/lib/rucio/tests/test_did.py
@@ -16,7 +16,7 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2015
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013-2018
@@ -119,7 +119,7 @@ def test_update_dids(self):
         set_metadata(scope=tmp_scope, name=lfn, key='adler32', value='0cc737ee')
         assert_equal(get_metadata(scope=tmp_scope, name=lfn)['adler32'], '0cc737ee')
 
-        with assert_raises(UnsupportedOperation):
+        with assert_raises(DataIdentifierNotFound):
             set_metadata(scope=tmp_scope, name='Nimportnawak', key='adler32', value='0cc737ee')
 
         set_metadata(scope=tmp_scope, name=lfn, key='bytes', value=724963577)