Repository: rucio/rucio

Issue:
Configuration: set doesn't work properly if '.' is provided as a value
Motivation
----------
The configuration set method doesn't work properly if '.' is provided as a value :
`rucio-admin config set --section automatix --option separator --value '.'`
But when trying to retrieve it : 
```
[Wed Feb 05 03:19:09.493072 2020] [:info] [pid 3368:tid 139652002477824] [client 188.184.29.109:36844] mod_wsgi (pid=3368, process='', application='rucio-server-int-02.cern.ch|/config'): Loading WSGI script '/usr/lib/python2.7/site-packages/rucio/web/rest/config.py'.
[Wed Feb 05 03:19:11.931687 2020] [:error] [pid 3368:tid 139652002477824] 'NoneType' object has no attribute 'lower'
[Wed Feb 05 03:19:11.931765 2020] [:error] [pid 3368:tid 139652002477824] <type 'exceptions.AttributeError'>
[Wed Feb 05 03:19:11.932690 2020] [:error] [pid 3368:tid 139652002477824] Traceback (most recent call last):
[Wed Feb 05 03:19:11.932702 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/rucio/web/rest/common.py", line 103, in decorated
[Wed Feb 05 03:19:11.932705 2020] [:error] [pid 3368:tid 139652002477824]     return f(*args, **kwargs)
[Wed Feb 05 03:19:11.932708 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/rucio/web/rest/common.py", line 155, in decorated
[Wed Feb 05 03:19:11.932710 2020] [:error] [pid 3368:tid 139652002477824]     return f(*args, **kwargs)
[Wed Feb 05 03:19:11.932713 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/rucio/web/rest/config.py", line 68, in GET
[Wed Feb 05 03:19:11.932715 2020] [:error] [pid 3368:tid 139652002477824]     for item in config.items(section, issuer=ctx.env.get('issuer')):
[Wed Feb 05 03:19:11.932718 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/rucio/api/config.py", line 132, in items
[Wed Feb 05 03:19:11.932734 2020] [:error] [pid 3368:tid 139652002477824]     return config.items(section)
[Wed Feb 05 03:19:11.932737 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/retrying.py", line 49, in wrapped_f
[Wed Feb 05 03:19:11.932739 2020] [:error] [pid 3368:tid 139652002477824]     return Retrying(*dargs, **dkw).call(f, *args, **kw)
[Wed Feb 05 03:19:11.932741 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/retrying.py", line 206, in call
[Wed Feb 05 03:19:11.932744 2020] [:error] [pid 3368:tid 139652002477824]     return attempt.get(self._wrap_exception)
[Wed Feb 05 03:19:11.932746 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/retrying.py", line 247, in get
[Wed Feb 05 03:19:11.932748 2020] [:error] [pid 3368:tid 139652002477824]     six.reraise(self.value[0], self.value[1], self.value[2])
[Wed Feb 05 03:19:11.932750 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/retrying.py", line 200, in call
[Wed Feb 05 03:19:11.932752 2020] [:error] [pid 3368:tid 139652002477824]     attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
[Wed Feb 05 03:19:11.932755 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 284, in new_funct
[Wed Feb 05 03:19:11.932757 2020] [:error] [pid 3368:tid 139652002477824]     return function(*args, **kwargs)
[Wed Feb 05 03:19:11.932759 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/rucio/core/config.py", line 192, in items
[Wed Feb 05 03:19:11.932761 2020] [:error] [pid 3368:tid 139652002477824]     return [(item[0], __convert_type(item[1])) for item in items]
[Wed Feb 05 03:19:11.932763 2020] [:error] [pid 3368:tid 139652002477824]   File "/usr/lib/python2.7/site-packages/rucio/core/config.py", line 267, in __convert_type
[Wed Feb 05 03:19:11.932766 2020] [:error] [pid 3368:tid 139652002477824]     if value.lower() in ['true', 'yes', 'on']:
[Wed Feb 05 03:19:11.932768 2020] [:error] [pid 3368:tid 139652002477824] AttributeError: 'NoneType' object has no attribute 'lower'
```
The value is converted to None. The value is set properly if one use directly the set value in core. Problem probably at the REST level






Discussion:

Resolving patch:
diff --git a/lib/rucio/client/configclient.py b/lib/rucio/client/configclient.py
--- a/lib/rucio/client/configclient.py
+++ b/lib/rucio/client/configclient.py
@@ -20,9 +20,12 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 #
 # PY3K COMPATIBLE
 
+from json import dumps
+
 from requests.status_codes import codes
 
 try:
@@ -71,19 +74,37 @@ def get_config(self, section=None, option=None):
             exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
             raise exc_cls(exc_msg)
 
-    def set_config_option(self, section, option, value):
+    def set_config_option(self, section, option, value, use_body_for_params=True):
         """
         Sends the request to create or set an option within a section. Missing sections will be created.
 
         :param section: the name of the section.
         :param option: the name of the option.
+        :param value: the value to set on the config option
+        :param use_body_for_params: send parameters in a json-encoded request body instead of url-encoded
+        TODO: remove this parameter
+        The format of the /config endpoint was recently changed. We migrated from performing a PUT on
+        "/config/<section>/<option>/<value>" to sending the parameters using a json-encoded body.
+        This was done to fix multiple un-wanted side effects related to how the middleware treats
+        values encoded in a path.
+        For a smooth transition, we allow both cases for now, but we should migrate to only passing
+        values via the request body.
         :return: True if option was removed successfully. False otherwise.
         """
 
-        path = '/'.join([self.CONFIG_BASEURL, section, option, value])
-        url = build_url(choice(self.list_hosts), path=path)
+        if use_body_for_params:
+            url = build_url(choice(self.list_hosts), path=self.CONFIG_BASEURL)
+            data = dumps({
+                section: {
+                    option: value
+                }
+            })
+            r = self._send_request(url, type='POST', data=data)
+        else:
+            path = '/'.join([self.CONFIG_BASEURL, section, option, value])
+            url = build_url(choice(self.list_hosts), path=path)
+            r = self._send_request(url, type='PUT')
 
-        r = self._send_request(url, type='PUT')
         if r.status_code == codes.created:
             return True
         else:
diff --git a/lib/rucio/web/rest/flaskapi/v1/config.py b/lib/rucio/web/rest/flaskapi/v1/config.py
--- a/lib/rucio/web/rest/flaskapi/v1/config.py
+++ b/lib/rucio/web/rest/flaskapi/v1/config.py
@@ -22,8 +22,10 @@
 # - Muhammad Aditya Hilmy <didithilmy@gmail.com>, 2020
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 import logging
+from json import loads
 
 from flask import Flask, Blueprint, request as request, jsonify
 from flask.views import MethodView
@@ -34,6 +36,19 @@
 from rucio.web.rest.utils import generate_http_error_flask
 
 
+def _config_items_from_input(input_data):
+    """
+    extract section, option, value from a dict with format {section: {option: value}}.
+    """
+    try:
+        for section, section_config in input_data.items():
+            for option, value in section_config.items():
+                return section, option, value
+    except ValueError:
+        return None, None, None
+    return None, None, None
+
+
 class Config(MethodView):
     """ REST API for full configuration. """
 
@@ -64,6 +79,39 @@ def get(self):
             logging.exception("Internal Error")
             return str(error), 500
 
+    def post(self):
+        """
+        Create or set the configuration option in the requested section.
+        The request body is expected to contain a json {"section": {"option": "value"}}.
+
+        .. :quickref: Config; set config value
+
+        :status 201: Option successfully created or updated.
+        :status 400: The input data is invalid or incomplete.
+        :status 401: Invalid Auth Token.
+        :status 500: Configuration Error.
+        """
+        json_data = request.data.decode()
+        try:
+            input_data = loads(json_data)
+        except ValueError:
+            return generate_http_error_flask(400, 'ValueError', 'Cannot decode json parameter dictionary')
+
+        section, option, value = _config_items_from_input(input_data)
+        if section is None or option is None or value is None:
+            return generate_http_error_flask(400, 'ValueError', 'Invalid input')
+
+        try:
+            config.set(section=section, option=option, value=value, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+            return 'Created', 201
+        except ConfigurationError:
+            return generate_http_error_flask(500, 'ConfigurationError', 'Could not set value \'%s\' for section \'%s\' option \'%s\'' % (value, section, option))
+        except RucioException as error:
+            return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
+        except Exception as error:
+            logging.exception("Internal Error")
+            return str(error), 500
+
 
 class Section(MethodView):
     """ REST API for the sections in the configuration. """
@@ -142,6 +190,7 @@ def delete(self, section, option):
         """
         try:
             config.remove_option(section=section, option=option, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
+            return 'OK', 200
         except RucioException as error:
             return generate_http_error_flask(500, error.__class__.__name__, error.args[0])
         except Exception as error:
@@ -156,6 +205,7 @@ def put(self, section, option, value):
         """
         Set the value of an option.
         If the option does not exist, create it.
+        TODO: remove this endpoint after migrating all clients to pass input data via a json body
 
         .. :quickref: OptionSet; set config value.
 
@@ -185,7 +235,7 @@ def blueprint():
     section_view = Section.as_view('section')
     bp.add_url_rule('/<section>', view_func=section_view, methods=['get', ])
     config_view = Config.as_view('config')
-    bp.add_url_rule('', view_func=config_view, methods=['get', ])
+    bp.add_url_rule('', view_func=config_view, methods=['get', 'post'])
 
     bp.before_request(request_auth_env)
     bp.after_request(response_headers)
diff --git a/lib/rucio/web/rest/webpy/v1/config.py b/lib/rucio/web/rest/webpy/v1/config.py
--- a/lib/rucio/web/rest/webpy/v1/config.py
+++ b/lib/rucio/web/rest/webpy/v1/config.py
@@ -21,12 +21,13 @@
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 import json
 from logging import getLogger, StreamHandler, DEBUG
 from traceback import format_exc
 
-from web import application, ctx, Created, loadhook, header, InternalError
+from web import application, ctx, data, Created, loadhook, header, InternalError
 
 from rucio.api import config
 from rucio.common.exception import ConfigurationError
@@ -44,6 +45,19 @@
         '', 'Config')
 
 
+def _config_items_from_input(input_data):
+    """
+    extract section, option, value from a dict with format {section: {option: value}}.
+    """
+    try:
+        for section, section_config in input_data.items():
+            for option, value in section_config.items():
+                return section, option, value
+    except ValueError:
+        return None, None, None
+    return None, None, None
+
+
 class Config(RucioController):
     """ REST API for full configuration. """
 
@@ -71,6 +85,43 @@ def GET(self):
 
         return json.dumps(res)
 
+    @exception_wrapper
+    def POST(self):
+        """
+        Set the value of an option.
+        If the option does not exist, create it.
+
+        HTTP Success:
+            200 OK
+
+        HTTP Error:
+            401 Unauthorized
+            400 The input data is invalid or incomplete
+            500 ConfigurationError
+
+        :param Rucio-Auth-Account: Account identifier.
+        :param Rucio-Auth-Token: 32 character hex string.
+        """
+
+        json_data = data().decode()
+        try:
+            input_data = json.loads(json_data)
+        except ValueError:
+            raise generate_http_error(400, 'ValueError', 'Cannot decode json parameter dictionary')
+
+        section, option, value = _config_items_from_input(input_data)
+        if section is None or option is None or value is None:
+            raise generate_http_error(400, 'ValueError', 'Invalid input data')
+
+        try:
+            config.set(section=section, option=option, value=value, issuer=ctx.env.get('issuer'), vo=ctx.env.get('vo'))
+        except ConfigurationError:
+            raise generate_http_error(500, 'ConfigurationError', 'Could not set value \'%s\' for section \'%s\' option \'%s\'' % (value, section, option))
+        except Exception as error:
+            print(format_exc())
+            raise InternalError(error)
+        raise Created()
+
 
 class Section(RucioController):
     """ REST API for the sections in the configuration. """
@@ -154,6 +205,7 @@ def PUT(self, section, option, value):
         """
         Set the value of an option.
         If the option does not exist, create it.
+        TODO: remove this endpoint after migrating all clients to pass input data via a json body
 
         HTTP Success:
             200 OK


Test patch:
diff --git a/lib/rucio/tests/test_config.py b/lib/rucio/tests/test_config.py
--- a/lib/rucio/tests/test_config.py
+++ b/lib/rucio/tests/test_config.py
@@ -130,3 +130,31 @@ def test_get_config_section_option(self):
 
         with pytest.raises(exception.ConfigNotFound):
             self.c.get_config(self.test_section_1, 'no_option')
+
+    def test_set_and_get_config_value_special_strings(self):
+        for test_option_description, option_value in [
+            ('dot', '.'),
+            ('slash', '/'),
+            ('aPath', 'a/b/c/../'),
+            ('percentEncodedSpecialChar', '%2E'),
+            ('urlParameters', 'a?x=y'),
+        ]:
+            self.c.set_config_option(self.test_section_1, test_option_description, option_value)
+            retrieved_value = self.c.get_config(self.test_section_1, test_option_description)
+            assert retrieved_value == option_value
+
+    def test_set_config_option_via_deprecated_url(self):
+        """
+        The format of the /config endpoint was recently changed, but we still support the old
+        format for API calls for the transition period.
+        TODO: remove this test
+        """
+        self.c.set_config_option(self.test_section_1, self.test_option_s + 'ViaUrl', self.test_option_sv, use_body_for_params=False)
+        self.c.set_config_option(self.test_section_1, self.test_option_b + 'ViaUrl', self.test_option_bv, use_body_for_params=False)
+        self.c.set_config_option(self.test_section_2, self.test_option_i + 'ViaUrl', self.test_option_iv, use_body_for_params=False)
+        self.c.set_config_option(self.test_section_2, self.test_option_f + 'ViaUrl', self.test_option_fv, use_body_for_params=False)
+        tmp = self.c.get_config(None, None)
+        assert self.test_option_s + 'ViaUrl' in tmp[self.test_section_1]
+        assert self.test_option_b + 'ViaUrl' in tmp[self.test_section_1]
+        assert self.test_option_i + 'ViaUrl' in tmp[self.test_section_2]
+        assert self.test_option_f + 'ViaUrl' in tmp[self.test_section_2]