Repository: rucio/rucio

Issue:
Add file to content history when the last replica is deleted by the reaper
When the latest replica (eligible for deletion) is removed, the file is removed from the content but not archived.

_Migrated from JIRA [RUCIO-2570](https://its.cern.ch/jira/browse/RUCIO-2570)_


Discussion:
Needs to be checked
Needs to be done 😄 
Starting today, I will be marking each time DDM Ops is affected by this issue.
:fire:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2019
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
@@ -44,7 +44,7 @@
 from sqlalchemy.exc import DatabaseError, IntegrityError, CompileError, InvalidRequestError
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import not_, func
-from sqlalchemy.sql.expression import bindparam, case, text, Insert, select, true
+from sqlalchemy.sql.expression import bindparam, case, text, select, true
 
 import rucio.core.rule
 import rucio.core.replica  # import add_replicas
@@ -344,7 +344,7 @@ def __add_files_to_dataset(scope, name, files, account, rse_id, ignore_duplicate
     # Get metadata from dataset
     try:
         dataset_meta = validate_name(scope=scope, name=name, did_type='D')
-    except:
+    except Exception:
         dataset_meta = None
 
     if rse_id:
@@ -596,31 +596,11 @@ def delete_dids(dids, account, expire_rules=False, session=None):
 
             # Archive content
             # Disable for postgres
-            if session.bind.dialect.name != 'postgresql':
-                q = session.query(models.DataIdentifierAssociation.scope,
-                                  models.DataIdentifierAssociation.name,
-                                  models.DataIdentifierAssociation.child_scope,
-                                  models.DataIdentifierAssociation.child_name,
-                                  models.DataIdentifierAssociation.did_type,
-                                  models.DataIdentifierAssociation.child_type,
-                                  models.DataIdentifierAssociation.bytes,
-                                  models.DataIdentifierAssociation.adler32,
-                                  models.DataIdentifierAssociation.md5,
-                                  models.DataIdentifierAssociation.guid,
-                                  models.DataIdentifierAssociation.events,
-                                  models.DataIdentifierAssociation.rule_evaluation,
-                                  bindparam("did_created_at", did.get('created_at')),
-                                  models.DataIdentifierAssociation.created_at,
-                                  models.DataIdentifierAssociation.updated_at,
-                                  bindparam('deleted_at', datetime.utcnow())).\
-                    filter(and_(models.DataIdentifierAssociation.scope == did['scope'],
-                                models.DataIdentifierAssociation.name == did['name']))
-                ins = Insert(table=models.DataIdentifierAssociationHistory, inline=True).\
-                    from_select(('scope', 'name', 'child_scope', 'child_name', 'did_type',
-                                 'child_type', 'bytes', 'adler32', 'md5', 'guid', 'events',
-                                 'rule_evaluation', 'did_created_at', 'created_at', 'updated_at',
-                                 'deleted_at'), q)
-                session.execute(ins)
+            insert_content_history(content_clause=[and_(models.DataIdentifierAssociation.scope == did['scope'],
+                                                        models.DataIdentifierAssociation.name == did['name'])],
+                                   did_created_at=did.get('created_at'),
+                                   session=session)
+
         parent_content_clause.append(and_(models.DataIdentifierAssociation.child_scope == did['scope'], models.DataIdentifierAssociation.child_name == did['name']))
         rule_id_clause.append(and_(models.ReplicationRule.scope == did['scope'], models.ReplicationRule.name == did['name']))
 
@@ -1941,7 +1921,6 @@ def remove_dids_from_followed(dids, account, session=None):
             session.query(models.DidsFollowed).\
                 filter_by(scope=did['scope'], name=did['name'], account=account).\
                 delete(synchronize_session=False)
-
     except NoResultFound:
         raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (did['scope'], did['name']))
 
@@ -2015,3 +1994,49 @@ def create_reports(total_workers, worker_number, session=None):
 
     except NoResultFound:
         raise exception.AccountNotFound("No email found for given account.")
+
+
+@transactional_session
+def insert_content_history(content_clause, did_created_at, session=None):
+    """
+    Insert into content history a list of did
+
+    :param content_clause: Content clause of the files to archive
+    :param did_created_at: Creation date of the did
+    :param session: The database session in use.
+    """
+    query = session.query(models.DataIdentifierAssociation.scope,
+                          models.DataIdentifierAssociation.name,
+                          models.DataIdentifierAssociation.child_scope,
+                          models.DataIdentifierAssociation.child_name,
+                          models.DataIdentifierAssociation.did_type,
+                          models.DataIdentifierAssociation.child_type,
+                          models.DataIdentifierAssociation.bytes,
+                          models.DataIdentifierAssociation.adler32,
+                          models.DataIdentifierAssociation.md5,
+                          models.DataIdentifierAssociation.guid,
+                          models.DataIdentifierAssociation.events,
+                          models.DataIdentifierAssociation.rule_evaluation,
+                          models.DataIdentifierAssociation.created_at,
+                          models.DataIdentifierAssociation.updated_at).\
+        filter(or_(*content_clause))
+
+    for cont in query.all():
+        models.DataIdentifierAssociationHistory(
+            scope=cont.scope,
+            name=cont.name,
+            child_scope=cont.child_scope,
+            child_name=cont.child_name,
+            did_type=cont.did_type,
+            child_type=cont.child_type,
+            bytes=cont.bytes,
+            adler32=cont.adler32,
+            md5=cont.md5,
+            guid=cont.guid,
+            events=cont.events,
+            rule_evaluation=cont.rule_evaluation,
+            updated_at=cont.updated_at,
+            created_at=cont.created_at,
+            did_created_at=did_created_at,
+            deleted_at=datetime.utcnow()
+        ).save(session=session, flush=False)
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -37,9 +37,9 @@
 from datetime import datetime, timedelta
 from json import dumps
 from re import match
-from six import string_types
 from traceback import format_exc
 
+from six import string_types
 from sqlalchemy import func, and_, or_, exists, not_, update
 from sqlalchemy.exc import DatabaseError, IntegrityError
 from sqlalchemy.sql import label
@@ -1491,6 +1491,7 @@ def delete_replicas(rse_id, files, ignore_availability=True, session=None):
                                         models.DataIdentifier.did_type == parent_did_type))
 
             for chunk in chunks(child_did_condition, 10):
+                rucio.core.did.insert_content_history(content_clause=chunk, did_created_at=None, session=session)
                 rowcount = session.query(models.DataIdentifierAssociation).\
                     filter(or_(*chunk)).\
                     delete(synchronize_session=False)
diff --git a/lib/rucio/daemons/automatix/automatix.py b/lib/rucio/daemons/automatix/automatix.py
--- a/lib/rucio/daemons/automatix/automatix.py
+++ b/lib/rucio/daemons/automatix/automatix.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014
 # - Tomas Kouba <tomas.kouba@cern.ch>, 2015
@@ -23,6 +23,7 @@
 # PY3K COMPATIBLE
 
 from __future__ import division
+from __future__ import print_function
 
 import logging
 import socket
@@ -31,6 +32,7 @@
 import threading
 import traceback
 
+from datetime import datetime
 from json import load
 from math import exp
 from os import remove, rmdir, stat, getpid
@@ -38,13 +40,16 @@
 from time import sleep, time
 
 from rucio.client import Client
-from rucio.common.config import config_get, config_get_bool, config_get_int
+from rucio.common.config import config_get
 from rucio.common.utils import adler32
+from rucio.core.config import get
 from rucio.core import monitor, heartbeat
 from rucio.rse import rsemanager as rsemgr
+from rucio.core.scope import list_scopes
+from rucio.common.types import InternalScope
 
 from rucio.common.utils import execute, generate_uuid
-from rucio.common.exception import FileReplicaAlreadyExists
+from rucio.common.exception import FileReplicaAlreadyExists, ConfigNotFound
 
 
 logging.basicConfig(stream=stdout,
@@ -59,10 +64,10 @@
 FAILURE = 1
 
 
-graceful_stop = threading.Event()
+GRACEFUL_STOP = threading.Event()
 
 
-def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None, set_metadata=False):
+def upload(files, scope, metadata, rse, account, source_dir, worker_number, total_workers, dataset_lifetime, did=None, set_metadata=False, prepend_str=''):
     logging.debug('In upload')
     dsn = None
     if did:
@@ -72,17 +77,20 @@ def upload(files, scope, metadata, rse, account, source_dir, worker_number, tota
     list_files = []
     lfns = []
     prepend_str = 'Thread [%i/%i] : ' % (worker_number, total_workers)
-    logging.debug(prepend_str + 'Looping over the files')
+    logging.debug('%s Looping over the files', prepend_str)
     for filename in files:
-        fullpath = '%s/%s' % (source_dir, filename)
+        physical_fname = filename
+        if physical_fname.find('/') > -1:
+            physical_fname = "".join(filename.split('/'))
+        fullpath = '%s/%s' % (source_dir, physical_fname)
         size = stat(fullpath).st_size
         checksum = adler32(fullpath)
-        logging.info(prepend_str + 'File %s : Size %s , adler32 %s' % (fullpath, str(size), checksum))
+        logging.info('%s File %s : Size %s , adler32 %s', prepend_str, fullpath, str(size), checksum)
         list_files.append({'scope': scope, 'name': filename, 'bytes': size, 'adler32': checksum, 'meta': {'guid': generate_uuid()}})
-        lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum, 'filename': filename})
+        lfns.append({'name': filename, 'scope': scope, 'filesize': size, 'adler32': checksum, 'filename': physical_fname})
 
     # Physical upload
-    logging.info(prepend_str + 'Uploading physically the files %s on %s' % (str(lfns), rse))
+    logging.info('%s Uploading physically the files %s on %s', prepend_str, str(lfns), rse)
     rse_info = rsemgr.get_rse_info(rse)
     try:
         success_upload = True
@@ -90,28 +98,28 @@ def upload(files, scope, metadata, rse, account, source_dir, worker_number, tota
             rows = rsemgr.upload(rse_info, lfns=lfns, source_dir=source_dir)
             # temporary hack
             global_status, ret = rows['success'], rows[1]
-            logging.info(prepend_str + 'Returned global status : %s, Returned : %s' % (str(global_status), str(ret)))
+            logging.info('%s Returned global status : %s, Returned : %s', prepend_str, str(global_status), str(ret))
             if not global_status:
                 for item in ret:
                     if (not isinstance(ret[item], FileReplicaAlreadyExists)) and ret[item] is not True:
                         sleep(exp(cnt))
                         success_upload = False
-                        logging.error(prepend_str + 'Problem to upload file %s with error %s' % (item, str(ret[item])))
+                        logging.error('%s Problem to upload file %s with error %s', prepend_str, item, str(ret[item]))
                         break
             else:
                 break
         if not success_upload:
-            logging.error(prepend_str + 'Upload operation to %s failed, removing leftovers' % (rse))
+            logging.error('%s Upload operation to %s failed, removing leftovers', prepend_str, rse)
             rsemgr.delete(rse_info, lfns=lfns)
             return False
     except Exception as error:
         logging.debug(traceback.format_exc())
-        logging.error(prepend_str + '%s' % (str(error)))
+        logging.error('%s %s', prepend_str, str(error))
         return False
-    logging.info(prepend_str + 'Files successfully copied on %s' % (rse))
+    logging.info('%s Files successfully copied on %s', prepend_str, rse)
 
     # Registering DIDs and replicas in Rucio
-    logging.info(prepend_str + 'Registering DIDs and replicas in Rucio')
+    logging.info('%s Registering DIDs and replicas in Rucio', prepend_str)
     meta = metadata
     if not set_metadata:
         meta = None
@@ -119,25 +127,25 @@ def upload(files, scope, metadata, rse, account, source_dir, worker_number, tota
         try:
             client.add_dataset(scope=dsn['scope'], name=dsn['name'], rules=[{'account': account, 'copies': 1, 'rse_expression': rse, 'grouping': 'DATASET', 'activity': 'Functional Test'}], meta=meta, lifetime=dataset_lifetime)
             client.add_files_to_dataset(scope=dsn['scope'], name=dsn['name'], files=list_files, rse=rse)
-            logging.info(prepend_str + 'Upload operation for %s:%s done' % (dsn['scope'], dsn['name']))
+            logging.info('%s Upload operation for %s:%s done', prepend_str, dsn['scope'], dsn['name'])
         except Exception as error:
             logging.debug(traceback.format_exc())
-            logging.error(prepend_str + 'Failed to upload %(files)s' % locals())
-            logging.error(prepend_str + '%s' % (str(error)))
-            logging.error(prepend_str + 'Removing files from the Storage')
+            logging.error('%s Failed to upload %s', prepend_str, str(list_files))
+            logging.error('%s %s', prepend_str, str(error))
+            logging.error('%s Removing files from the Storage', prepend_str)
             rsemgr.delete(rse_info, lfns=lfns)
             return False
     else:
-        logging.warning(prepend_str + 'No dsn is specified')
+        logging.warning('%s No dsn is specified', prepend_str)
         try:
             client.add_replicas(files=list_files, rse=rse)
             client.add_replication_rule(list_files, copies=1, rse_expression=rse, activity='Functional Test')
-            logging.info(prepend_str + 'Upload operation for %s done' % (str(list_files)))
+            logging.info('%s Upload operation for %s done', prepend_str, str(list_files))
         except Exception as error:
             logging.debug(traceback.format_exc())
-            logging.error(prepend_str + 'Failed to upload %(files)s' % locals())
-            logging.error(prepend_str + '%s' % (str(error)))
-            logging.error(prepend_str + 'Removing files from the Storage')
+            logging.error('%s Failed to upload %s', prepend_str, str(list_files))
+            logging.error('%s %s', prepend_str, str(error))
+            logging.error('%s Removing files from the Storage', prepend_str)
             rsemgr.delete(rse_info, lfns=lfns)
             return False
     return True
@@ -167,14 +175,46 @@ def choose_element(probabilities, data):
 
 
 def generate_file(fname, size):
-    cmd = '/bin/dd if=/dev/urandom of=%(fname)s bs=%(size)s count=1' % locals()
+    cmd = '/bin/dd if=/dev/urandom of=%s bs=%s count=1' % (fname, size)
     exitcode, out, err = execute(cmd)
     logging.debug(out)
     logging.debug(err)
     return exitcode
 
 
-def automatix(sites, inputfile, sleep_time, account, worker_number=1, total_workers=1, once=False, dataset_lifetime=None, set_metadata=False):
+def generate_didname(metadata, dsn, did_type):
+    try:
+        did_prefix = get('automatix', 'did_prefix')
+    except ConfigNotFound:
+        did_prefix = ''
+    try:
+        pattern = get('automatix', '%s_pattern' % did_type)
+        separator = get('automatix', 'separator')
+    except ConfigNotFound:
+        return generate_uuid()
+    fields = pattern.split(separator)
+    file_name = ''
+    for field in fields:
+        if field == 'date':
+            field_str = str(datetime.now().date())
+        elif field == 'did_prefix':
+            field_str = did_prefix
+        elif field == 'dsn':
+            field_str = dsn
+        elif field == 'uuid':
+            field_str = generate_uuid()
+        elif field == 'randint':
+            field_str = str(random.randint(0, 100000))
+        else:
+            field_str = metadata.get(field, None)
+            if not field_str:
+                field_str = str(random.randint(0, 100000))
+        file_name = '%s%s%s' % (file_name, separator, field_str)
+    len_separator = len(separator)
+    return file_name[len_separator:]
+
+
+def automatix(sites, inputfile, sleep_time, account, worker_number=1, total_workers=1, scope='tests', once=False, dataset_lifetime=None, set_metadata=False):
     sleep(sleep_time * (total_workers - worker_number) / total_workers)
 
     executable = ' '.join(argv)
@@ -182,16 +222,14 @@ def automatix(sites, inputfile, sleep_time, account, worker_number=1, total_work
     pid = getpid()
     hb_thread = threading.current_thread()
     heartbeat.sanity_check(executable=executable, hostname=hostname)
-
-    while not graceful_stop.is_set():
+    prepend_str = 'Thread [%i/%i] : ' % (worker_number, total_workers)
+    while not GRACEFUL_STOP.is_set():
         heartbeat.live(executable, hostname, pid, hb_thread)
         starttime = time()
         prepend_str = 'Thread [%i/%i] : ' % (worker_number, total_workers)
-        logging.info(prepend_str + 'Getting data distribution')
+        logging.info('%s Getting data distribution', prepend_str)
         probabilities, data = get_data_distribution(inputfile)
-        logging.debug(prepend_str + 'Probabilities %s' % (probabilities))
-        account = 'root'
-        scope = 'tests'
+        logging.debug('%s Probabilities %s', prepend_str, probabilities)
         totretries = 3
         status = False
 
@@ -200,37 +238,36 @@ def automatix(sites, inputfile, sleep_time, account, worker_number=1, total_work
             for retry in range(0, totretries):
                 start_time = time()
                 tmpdir = tempfile.mkdtemp()
-                logging.info(prepend_str + 'Running on site %s' % (site))
+                logging.info('%s Running on site %s', prepend_str, site)
                 dic = choose_element(probabilities, data)
                 metadata = dic['metadata']
-                metadata['version'] = str(random.randint(0, 1000))
-                metadata['run_number'] = str(random.randint(0, 100000))
-                metadata['stream_name'] = 'automatix_stream'
-                metadata['campaign'] = 'automatix_campaign'
                 try:
                     nbfiles = dic['nbfiles']
                 except KeyError:
                     nbfiles = 2
-                    logging.warning(prepend_str + 'No nbfiles defined in the configuration, will use 2')
+                    logging.warning('%s No nbfiles defined in the configuration, will use 2', prepend_str)
                 try:
                     filesize = dic['filesize']
                 except KeyError:
                     filesize = 1000000
-                    logging.warning(prepend_str + 'No filesize defined in the configuration, will use 1M files')
-                dsn = 'tests:%s.%s.%s.%s.%s.%s' % (metadata['project'], metadata['run_number'], metadata['stream_name'], metadata['prod_step'], metadata['datatype'], metadata['version'])
+                    logging.warning('%s No filesize defined in the configuration, will use 1M files', prepend_str)
+                dsn = generate_didname(metadata, None, 'dataset')
                 fnames = []
                 lfns = []
-                for dummy_nbfile in range(nbfiles):
-                    fname = '%s.%s' % (metadata['datatype'], generate_uuid())
+                physical_fnames = []
+                for _ in range(nbfiles):
+                    fname = generate_didname(metadata=metadata, dsn=dsn, did_type='file')
                     lfns.append(fname)
-                    fname = '%s/%s' % (tmpdir, fname)
-                    logging.info(prepend_str + 'Generating file %(fname)s in dataset %(dsn)s' % locals())
-                    generate_file(fname, filesize)
+                    logging.info('%s Generating file %s in dataset %s', prepend_str, fname, dsn)
+                    physical_fname = '%s/%s' % (tmpdir, "".join(fname.split('/')))
+                    physical_fnames.append(physical_fname)
+                    generate_file(physical_fname, filesize)
                     fnames.append(fname)
-                logging.info(prepend_str + 'Upload %s to %s' % (dsn, site))
+                logging.info('%s Upload %s to %s', prepend_str, dsn, site)
+                dsn = '%s:%s' % (scope, dsn)
                 status = upload(files=lfns, scope=scope, metadata=metadata, rse=site, account=account, source_dir=tmpdir, worker_number=worker_number, total_workers=total_workers, dataset_lifetime=dataset_lifetime, did=dsn, set_metadata=set_metadata)
-                for fname in fnames:
-                    remove(fname)
+                for physical_fname in physical_fnames:
+                    remove(physical_fname)
                 rmdir(tmpdir)
                 if status:
                     monitor.record_counter(counters='automatix.addnewdataset.done', delta=1)
@@ -238,21 +275,21 @@ def automatix(sites, inputfile, sleep_time, account, worker_number=1, total_work
                     monitor.record_timer('automatix.datasetinjection', (time() - start_time) * 1000)
                     break
                 else:
-                    logging.info(prepend_str + 'Failed to upload files. Will retry another time (attempt %s/%s)' % (str(retry + 1), str(totretries)))
+                    logging.info('%s Failed to upload files. Will retry another time (attempt %s/%s)', prepend_str, str(retry + 1), str(totretries))
         if once is True:
-            logging.info(prepend_str + 'Run with once mode. Exiting')
+            logging.info('%s Run with once mode. Exiting', prepend_str)
             break
         tottime = time() - starttime
         if status:
-            logging.info(prepend_str + 'It took %s seconds to upload one dataset on %s' % (str(tottime), str(sites)))
+            logging.info('%s It took %s seconds to upload one dataset on %s', prepend_str, str(tottime), str(sites))
             if sleep_time > tottime:
-                logging.info(prepend_str + 'Will sleep for %s seconds' % (str(sleep_time - tottime)))
+                logging.info('%s Will sleep for %s seconds', prepend_str, str(sleep_time - tottime))
                 sleep(sleep_time - tottime)
         else:
-            logging.info(prepend_str + 'Retrying a new upload')
+            logging.info('%s Retrying a new upload', prepend_str)
     heartbeat.die(executable, hostname, pid, hb_thread)
-    logging.info(prepend_str + 'Graceful stop requested')
-    logging.info(prepend_str + 'Graceful stop done')
+    logging.info('%s Graceful stop requested', prepend_str)
+    logging.info('%s Graceful stop done', prepend_str)
 
 
 def run(total_workers=1, once=False, inputfile=None):
@@ -260,28 +297,36 @@ def run(total_workers=1, once=False, inputfile=None):
     Starts up the automatix threads.
     """
     try:
-        sites = [s.strip() for s in config_get('automatix', 'sites').split(',')]
+        sites = [s.strip() for s in get('automatix', 'sites').split(',')]
     except Exception:
         raise Exception('Could not load sites from configuration')
     if not inputfile:
         inputfile = '/opt/rucio/etc/automatix.json'
     try:
-        sleep_time = config_get_int('automatix', 'sleep_time')
+        sleep_time = get('automatix', 'sleep_time')
     except Exception:
-        sleep_time = 3600
+        sleep_time = 30
     try:
-        account = config_get_int('automatix', 'account')
+        account = get('automatix', 'account')
     except Exception:
         account = 'root'
     try:
-        dataset_lifetime = config_get_int('automatix', 'dataset_lifetime')
+        dataset_lifetime = get('automatix', 'dataset_lifetime')
     except Exception:
         dataset_lifetime = None
     try:
-        set_metadata = config_get_bool('automatix', 'set_metadata')
+        set_metadata = get('automatix', 'set_metadata')
     except Exception:
         set_metadata = False
 
+    try:
+        scope = get('automatix', 'scope')
+        if InternalScope(scope) not in list_scopes():
+            logging.error('Scope %s does not exist. Exiting', scope)
+            GRACEFUL_STOP.set()
+    except Exception:
+        scope = False
+
     threads = list()
     for worker_number in range(0, total_workers):
         kwargs = {'worker_number': worker_number,
@@ -292,11 +337,12 @@ def run(total_workers=1, once=False, inputfile=None):
                   'account': account,
                   'inputfile': inputfile,
                   'set_metadata': set_metadata,
+                  'scope': scope,
                   'dataset_lifetime': dataset_lifetime}
         threads.append(threading.Thread(target=automatix, kwargs=kwargs))
     [thread.start() for thread in threads]
     while threads[0].is_alive():
-        logging.debug('Still %i active threads' % len(threads))
+        logging.debug('Still %i active threads', len(threads))
         [thread.join(timeout=3.14) for thread in threads]
 
 
@@ -304,4 +350,4 @@ def stop(signum=None, frame=None):
     """
     Graceful exit.
     """
-    graceful_stop.set()
+    GRACEFUL_STOP.set()


Test patch: