Repository: rucio/rucio

Issue:
1.20.4rc3 fts bug
Motivation
----------
In 1.20.4rc3, the conveyor reported a crash with the trace:
```
2019-08-21 15:56:55,991 3229    ERROR   Thread [1/31] :  Failed to submit a job with error mc16_13TeV is not JSON serializable: Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/rucio/daemons/conveyor/common.py", line 125, in submit_transfer
    user_transfer_job=user_transfer_job)
  File "/usr/lib/python2.7/site-packages/rucio/core/transfer.py", line 90, in submit_bulk_transfers
    transfer_id = FTS3Transfertool(external_host=external_host).submit(files=job_files, job_params=job_params, timeout=timeout)
  File "/usr/lib/python2.7/site-packages/rucio/transfertool/fts3.py", line 186, in submit
    params_str = json.dumps(params_dict)
  File "/usr/lib64/python2.7/json/__init__.py", line 243, in dumps
    return _default_encoder.encode(obj)
  File "/usr/lib64/python2.7/json/encoder.py", line 207, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/lib64/python2.7/json/encoder.py", line 270, in iterencode
    return _iterencode(o, 0)
  File "/usr/lib64/python2.7/json/encoder.py", line 184, in default
    raise TypeError(repr(o) + " is not JSON serializable")
TypeError: mc16_13TeV is not JSON serializableh 
```



Modification
------------
Utilise rucio.common.utils.APIEncoder in the fts transfertools files




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -583,7 +583,7 @@ def delete_dids(dids, account, expire_rules=False, session=None):
                                                   models.CollectionReplica.name == did['name']))
 
         # ATLAS LOCALGROUPDISK Archive policy
-        if did['did_type'] == DIDType.DATASET and did['scope'] != 'archive':
+        if did['did_type'] == DIDType.DATASET and did['scope'].external != 'archive':
             try:
                 rucio.core.rule.archive_localgroupdisk_datasets(scope=did['scope'], name=did['name'], session=session)
             except exception.UndefinedPolicy:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -3111,7 +3111,7 @@ def archive_localgroupdisk_datasets(scope, name, session=None):
                                    session=session)
             rucio.core.did.attach_dids(scope=archive, name=name, dids=content, account=did['account'], session=session)
             if not did['open']:
-                rucio.core.did.set_status(scope='archive', name=name, open=False, session=session)
+                rucio.core.did.set_status(scope=archive, name=name, open=False, session=session)
 
             for rse in rses_to_rebalance:
                 add_rule(dids=[{'scope': archive, 'name': name}],
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -14,6 +14,7 @@
 # - Diego Ciangottini <diego.ciangottini@pg.infn.it>, 2018
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 #
 # PY3K COMPATIBLE
 
@@ -48,6 +49,7 @@
     pass
 from rucio.common.config import config_get, config_get_bool
 from rucio.common.exception import TransferToolTimeout, TransferToolWrongAnswer
+from rucio.common.utils import APIEncoder
 from rucio.core.monitor import record_counter, record_timer
 from rucio.db.sqla.constants import FTSState
 from rucio.transfertool.transfertool import Transfertool
@@ -182,7 +184,7 @@ def submit(self, files, job_params, timeout=None):
 
         # bulk submission
         params_dict = {'files': files, 'params': job_params}
-        params_str = json.dumps(params_dict)
+        params_str = json.dumps(params_dict, cls=APIEncoder)
 
         post_result = None
         try:
@@ -259,7 +261,7 @@ def update_priority(self, transfer_id, priority, timeout=None):
 
         job = None
         params_dict = {"params": {"priority": priority}}
-        params_str = json.dumps(params_dict)
+        params_str = json.dumps(params_dict, cls=APIEncoder)
 
         job = requests.post('%s/jobs/%s' % (self.external_host, transfer_id),
                             verify=self.verify,
@@ -524,7 +526,7 @@ def set_se_config(self, storage_element, inbound_max_active=None, outbound_max_a
             except KeyError:
                 params_dict[storage_element]['se_info'] = {'outbound_max_throughput': outbound_max_throughput}
 
-        params_str = json.dumps(params_dict)
+        params_str = json.dumps(params_dict, cls=APIEncoder)
 
         try:
             result = requests.post('%s/config/se' % (self.external_host),
@@ -561,7 +563,7 @@ def set_se_status(self, storage_element, message, ban=True, timeout=None):
             params_dict['timeout'] = timeout
             status = 'WAIT'
         params_dict['status'] = status
-        params_str = json.dumps(params_dict)
+        params_str = json.dumps(params_dict, cls=APIEncoder)
 
         result = None
         if ban:


Test patch: