Repository: haddocking/arctic3d

Issue:
fix logging
current logging system can be improved. will take inspiration from HADDOCK3 logging to improve it

Comments:

Patch:
diff --git a/src/arctic3d/__init__.py b/src/arctic3d/__init__.py
index e69de29..332735d 100644
--- a/src/arctic3d/__init__.py
+++ b/src/arctic3d/__init__.py
@@ -0,0 +1,12 @@
+"""arctic3d"""
+import logging
+import sys
+
+log = logging.getLogger(__name__)
+log.handlers.clear()
+log.setLevel(logging.DEBUG)
+handler = logging.StreamHandler(sys.stdout)
+handler.setFormatter(
+    logging.Formatter("[%(asctime)s %(module)s %(levelname)s] %(message)s")
+)
+log.addHandler(handler)
diff --git a/src/arctic3d/cli.py b/src/arctic3d/cli.py
index ea1b0a4..caf5b7a 100644
--- a/src/arctic3d/cli.py
+++ b/src/arctic3d/cli.py
@@ -1,12 +1,10 @@
 """Main CLI."""
 import argparse
-import logging
-import shutil
 import sys
 import time
-import os
 from pathlib import Path
 
+from arctic3d import log
 from arctic3d.modules.blast import run_blast
 from arctic3d.modules.cluster_interfaces import cluster_interfaces
 from arctic3d.modules.input import Input
@@ -14,21 +12,14 @@ from arctic3d.modules.interface import (
     get_interface_residues,
     read_interface_residues,
 )
-from arctic3d.modules.output import make_output, setup_output_folder
+from arctic3d.modules.output import (
+    make_output,
+    create_output_folder,
+    setup_output_folder,
+)
 from arctic3d.modules.pdb import get_best_pdb
 from arctic3d.modules.sequence import to_fasta
-
-# logging
-LOGNAME = f"arctic3d_{os.getpid()}.log"
-LOGNAME_FINAL = "arctic3d.log"
-logging.basicConfig(filename=LOGNAME)
-log = logging.getLogger(LOGNAME)
-ch = logging.StreamHandler()
-formatter = logging.Formatter(
-    " [%(asctime)s %(module)s:L%(lineno)d %(levelname)s] %(message)s"
-)
-ch.setFormatter(formatter)
-log.addHandler(ch)
+from arctic3d.modules.log import add_log_for_CLI
 
 
 argument_parser = argparse.ArgumentParser()
@@ -175,11 +166,10 @@ def main(
     ligand,
     linkage_strategy,
     threshold,
+    log_level="DEBUG",
 ):
     """Main function."""
     st_time = time.time()
-    log.setLevel("DEBUG")
-
     inp = Input(input_arg)
     input_files = {}
     # retrieve uniprot information
@@ -197,15 +187,23 @@ def main(
             input_files["interface_file"] = Path(interface_file)
             uniprot_id = None
 
+    # create output folder
+    run_dir_path = create_output_folder(run_dir, uniprot_id)
+    # configure logging
+    log_file = Path(run_dir_path, "arctic3d.log")
+    add_log_for_CLI(log, log_level, log_file)
+
+    log.info(f"Target UNIPROTID: {uniprot_id}")
+
     # save json files
     if interface_data:
         input_files["interface_data"] = Path(interface_data)
     if pdb_data:
         input_files["pdb_data"] = Path(pdb_data)
 
-    log.info(f"Target UNIPROTID: {uniprot_id}")
-
-    input_files = setup_output_folder(uniprot_id, input_files, run_dir)
+    input_files = setup_output_folder(
+        run_dir=run_dir_path, input_files=input_files
+    )
 
     # retrieve interfaces.
     if "interface_file" in input_files:
@@ -299,11 +297,6 @@ def main(
         f"arctic3d run completed in {(time.time() - st_time):.2f} seconds."
     )
 
-    # move log file to output folder
-    exp_log_path = Path(f"../{LOGNAME}")
-    if exp_log_path.exists():
-        shutil.move(exp_log_path, LOGNAME_FINAL)
-
 
 if __name__ == "__main__":
     sys.exit(maincli())
diff --git a/src/arctic3d/cli_localise.py b/src/arctic3d/cli_localise.py
index f89dc26..56b6066 100644
--- a/src/arctic3d/cli_localise.py
+++ b/src/arctic3d/cli_localise.py
@@ -31,34 +31,24 @@ and biological process (P)::
         --quickgo=F
 """
 import argparse
-import logging
 import os
-import shutil
 import sys
 import time
 from pathlib import Path
 
+from arctic3d import log
 
 from arctic3d.functions import make_request
 from arctic3d.modules.interface import parse_out_partner
+from arctic3d.modules.log import add_log_for_CLI
 from arctic3d.modules.output import (
     create_barplot,
+    create_output_folder,
     parse_clusters,
     setup_output_folder,
     write_dict,
 )
 
-LOGNAME = f"arctic3d_localise_{os.getpid()}.log"
-LOGNAME_FINAL = "arctic3d_localise.log"
-logging.basicConfig(filename=LOGNAME)
-log = logging.getLogger(LOGNAME)
-ch = logging.StreamHandler()
-formatter = logging.Formatter(
-    " [%(asctime)s %(module)s:L%(lineno)d %(levelname)s] %(message)s"
-)
-ch.setFormatter(formatter)
-log.addHandler(ch)
-
 UNIPROT_API_URL = "https://www.ebi.ac.uk/proteins/api/proteins"
 
 argument_parser = argparse.ArgumentParser()
@@ -302,10 +292,18 @@ def maincli():
     cli(argument_parser, main)
 
 
-def main(input_arg, run_dir, out_partner, quickgo, weight):
+def main(input_arg, run_dir, out_partner, quickgo, weight, log_level="DEBUG"):
     """Main function."""
-    log.setLevel("INFO")
+    log.setLevel(log_level)
     start_time = time.time()
+
+    # create output folder
+    run_dir_path = create_output_folder(run_dir)
+    # logging
+    log_file = Path(run_dir_path, "arctic3d_localise.log")
+    add_log_for_CLI(log, log_level, log_file)
+
+    # property name
     prop_name = "location"
     if quickgo:
         if quickgo == "F":
@@ -326,7 +324,7 @@ def main(input_arg, run_dir, out_partner, quickgo, weight):
     if not input_files["cl_filename"].exists():
         raise Exception("non existing input file")
 
-    input_files = setup_output_folder(None, input_files, run_dir)
+    input_files = setup_output_folder(run_dir_path, input_files)
 
     # parsing arctic3d clustering output
     clustering_dict = parse_clusters(input_files["cl_filename"])
@@ -425,12 +423,6 @@ def main(input_arg, run_dir, out_partner, quickgo, weight):
     elap_time = round((time.time() - start_time), 3)
     log.info(f"arctic3d_localise run took {elap_time} seconds")
 
-    # copying log file to the run folder (if possible)
-    try:
-        shutil.move(f"../{LOGNAME}", LOGNAME_FINAL)
-    except FileNotFoundError as e:
-        log.warning(f"Could not find log file: {e}")
-
 
 if __name__ == "__main__":
     sys.exit(maincli())
diff --git a/src/arctic3d/cli_resclust.py b/src/arctic3d/cli_resclust.py
index d7460f8..a994381 100644
--- a/src/arctic3d/cli_resclust.py
+++ b/src/arctic3d/cli_resclust.py
@@ -25,28 +25,18 @@ Input arguments:
     `criterion` : the criterion to extract the clusters.
 """
 import argparse
-import logging
 import sys
 
 import MDAnalysis as mda
 from scipy.spatial.distance import pdist
 
+from arctic3d import log
 from arctic3d.modules.clustering import (
     cluster_similarity_matrix,
     get_clustering_dict,
 )
 from arctic3d.modules.input import Input
 
-LOGNAME = "arctic3d_resclust.log"
-logging.basicConfig(filename=LOGNAME, filemode="w")
-log = logging.getLogger(LOGNAME)
-ch = logging.StreamHandler()
-formatter = logging.Formatter(
-    " [%(asctime)s %(module)s:L%(lineno)d %(levelname)s] %(message)s"
-)
-ch.setFormatter(formatter)
-log.addHandler(ch)
-
 
 argument_parser = argparse.ArgumentParser()
 argument_parser.add_argument(
@@ -177,9 +167,7 @@ def main(input_arg, residue_list, chain, threshold, linkage, criterion):
 
     n_chains = u.n_segments
     if n_chains != 1:
-        log.error(
-            f"Number of consistent segments ({n_chains}) != 1. Aborting."
-        )
+        log.error(f"Number of consistent segments ({n_chains}) != 1.Aborting.")
         sys.exit(1)
 
     # do the clustering
diff --git a/src/arctic3d/cli_restraints.py b/src/arctic3d/cli_restraints.py
index face33c..c2f2af6 100644
--- a/src/arctic3d/cli_restraints.py
+++ b/src/arctic3d/cli_restraints.py
@@ -33,26 +33,21 @@ This will consider only residues with a probability of being in the interface
 higher than 0.5 (for each cluster).
 """
 import argparse
-import logging
 import os
-import shutil
 import sys
 import time
 from pathlib import Path
 import tarfile
 
-from arctic3d.modules.output import read_residues_probs, setup_output_folder
+from arctic3d import log
 
-LOGNAME = f"arctic3d_restraints_{os.getpid()}.log"
-LOGNAME_FINAL = "arctic3d_restraints.log"
-logging.basicConfig(filename=LOGNAME)
-log = logging.getLogger(LOGNAME)
-ch = logging.StreamHandler()
-formatter = logging.Formatter(
-    " [%(asctime)s %(module)s:L%(lineno)d %(levelname)s] %(message)s"
+from arctic3d.modules.output import (
+    read_residues_probs,
+    create_output_folder,
+    setup_output_folder,
 )
-ch.setFormatter(formatter)
-log.addHandler(ch)
+from arctic3d.modules.log import add_log_for_CLI
+
 
 argument_parser = argparse.ArgumentParser()
 argument_parser.add_argument(
@@ -234,12 +229,14 @@ def maincli():
     cli(argument_parser, main)
 
 
-def main(r1, r2, ch1, ch2, run_dir, prob_threshold=0.5):
+def main(r1, r2, ch1, ch2, run_dir, prob_threshold=0.5, log_level="DEBUG"):
     """Main function."""
-    log.setLevel("INFO")
+    log.setLevel(log_level)
     start_time = time.time()
     log.info("Starting arctic3d_restraints")
-
+    run_dir_path = create_output_folder(run_dir)
+    log_file = Path(run_dir_path, "arctic3d_restraints.log")
+    add_log_for_CLI(log, log_level, log_file)
     # checking if r1 and r2 exists
     if not os.path.exists(r1):
         log.error(f"Could not find {r1}")
@@ -261,7 +258,7 @@ def main(r1, r2, ch1, ch2, run_dir, prob_threshold=0.5):
     # Setting up output folder
     input_files = {"r1_res_fname": r1_res_fname, "r2_res_fname": r2_res_fname}
     log.info(f"Input files are {input_files}")
-    input_files = setup_output_folder(None, input_files, run_dir)
+    input_files = setup_output_folder(run_dir_path, input_files)
 
     # read and filter probabilities
     r1_residues_probs = read_residues_probs(input_files["r1_res_fname"])
@@ -282,7 +279,7 @@ def main(r1, r2, ch1, ch2, run_dir, prob_threshold=0.5):
             ambig_fnames.append(ambig_fname)
             log.info(
                 f"Creating {ambig_fname} restraint file by"
-                "coupling {cl1} (r1) and {cl2} (r2)"
+                f" coupling {cl1} (r1) and {cl2} (r2)"
             )
             generate_restraints(residues1, residues2, ch1, ch2, ambig_fname)
             n_ambig += 1
@@ -291,9 +288,3 @@ def main(r1, r2, ch1, ch2, run_dir, prob_threshold=0.5):
 
     elap_time = round((time.time() - start_time), 3)
     log.info(f"arctic3d_restraints run took {elap_time} seconds")
-
-    # copying log file to the run folder (if possible)
-    try:
-        shutil.move(f"../{LOGNAME}", LOGNAME_FINAL)
-    except FileNotFoundError as e:
-        log.warning(f"Could not find log file: {e}")
diff --git a/src/arctic3d/modules/log.py b/src/arctic3d/modules/log.py
new file mode 100644
index 0000000..902aeca
--- /dev/null
+++ b/src/arctic3d/modules/log.py
@@ -0,0 +1,70 @@
+"""Manage logging."""
+import logging
+import sys
+from functools import partial
+from logging import FileHandler, StreamHandler
+
+
+log_file_name = "log"
+
+
+info_formatter = "[%(asctime)s %(module)s %(levelname)s] %(message)s"
+debug_formatter = (
+    "[%(asctime)s] "
+    "%(filename)s:%(name)s:%(funcName)s:%(lineno)d: "
+    "%(message)s"
+)
+
+log_formatters = {
+    "DEBUG": debug_formatter,
+    "INFO": info_formatter,
+    "WARNING": info_formatter,
+    "ERROR": info_formatter,
+    "CRITICAL": info_formatter,
+}
+
+log_levels = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+}
+
+
+def add_handler(
+    log_obj,
+    handler,
+    stream,
+    log_level="INFO",
+    formatter=info_formatter,
+):
+    """Add a logging Handler to the log object."""
+    ch = handler(stream)
+    ch.setLevel(log_levels[log_level.upper()])
+    ch.setFormatter(logging.Formatter(formatter))
+    log_obj.addHandler(ch)
+    return ch
+
+
+def add_log_for_CLI(log, log_level, logfile):
+    """Configure log for command-line clients."""
+    llu = log_level.upper()
+
+    params = {
+        "log_level": llu,
+        "formatter": log_formatters[llu],
+    }
+
+    log.handlers.clear()
+    add_sysout_handler(log, **params)
+    add_logfile_handler(log, stream=logfile, **params)
+    return
+
+
+add_sysout_handler = partial(
+    add_handler, handler=StreamHandler, stream=sys.stdout
+)  # noqa: E501
+add_logfile_handler = partial(
+    add_handler, handler=FileHandler, stream=log_file_name
+)  # noqa: E501
diff --git a/src/arctic3d/modules/output.py b/src/arctic3d/modules/output.py
index 1ea7821..06bb7b5 100644
--- a/src/arctic3d/modules/output.py
+++ b/src/arctic3d/modules/output.py
@@ -13,22 +13,20 @@ import plotly.graph_objects as go
 log = logging.getLogger("arctic3d.log")
 
 
-def setup_output_folder(uniprot_id, input_files, output_dir):
-    """Sets up output folder.
+def create_output_folder(output_dir, uniprot_id=None):
+    """Creates output folder.
 
     Parameters
     ----------
-    uniprot_id : string or None
-        uniprot_id of the run
-    input_files : dict of Paths
-        dict of input files
-    output_dir : str or None
+    output_dir : str
         user-defined name of the run
+    uniport_id : str or None
+        uniprot id of the target
 
     Returns
     -------
-    copied_input_files : dict of Paths
-        dict of copied input files
+    run_dir : Path
+        path to the run directory
     """
     run_dir = output_dir
     if run_dir is None:
@@ -41,10 +39,27 @@ def setup_output_folder(uniprot_id, input_files, output_dir):
 
     if os.path.exists(run_dir):
         raise Exception(f"{run_dir} already exists!")
-
-    # setting up the directory
-    log.info(f"Setting up output_directory {run_dir}")
+    log.info(f"Creating output_directory {run_dir}")
     os.mkdir(run_dir)
+    return run_dir
+
+
+def setup_output_folder(run_dir, input_files):
+    """Sets up output folder.
+
+    Parameters
+    ----------
+    run_dir : str or Path
+        name of the run directory
+    input_files : dict of Paths
+        dict of input files
+
+    Returns
+    -------
+    copied_input_files : dict of Paths
+        dict of copied input files
+    """
+    log.info(f"Setting up output folder {run_dir}")
     datadir = Path(run_dir, "input_data")
     os.mkdir(datadir)
 


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 5b143a0..996733c 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -27,5 +27,8 @@ def test_cli_empty():
     os.chdir(start_cwd)
     exp_dir = Path(f"arctic3d-{target_uniprot}")
     assert exp_dir.exists() is True
+    # Check that the log file has been created
+    assert Path(exp_dir, "arctic3d.log").exists()
+    # remove folder
     if exp_dir.exists():
         shutil.rmtree(exp_dir)
diff --git a/tests/test_cli_localise.py b/tests/test_cli_localise.py
index 8fd265b..a236a85 100644
--- a/tests/test_cli_localise.py
+++ b/tests/test_cli_localise.py
@@ -41,6 +41,7 @@ def example_uniprot_data():
 
 
 def test_localise_cli_empty(empty_cluster_filepath):
+    """Test localise cli with empty cluster file."""
     start_cwd = os.getcwd()
     run_dir = "arctic3d-localise"
     main(
@@ -51,6 +52,10 @@ def test_localise_cli_empty(empty_cluster_filepath):
         None,
     )
     os.chdir(start_cwd)
+    # Check that the output directory has been created
+    assert Path(run_dir).exists()
+    # check existence of log file
+    assert Path("arctic3d-localise", "arctic3d_localise.log").exists()
     shutil.rmtree(run_dir)
 
 
diff --git a/tests/test_cli_restraints.py b/tests/test_cli_restraints.py
index 413113c..4003a95 100644
--- a/tests/test_cli_restraints.py
+++ b/tests/test_cli_restraints.py
@@ -79,6 +79,8 @@ def test_main():
     main(r1, r2, None, None, run_dir=run_dir, prob_threshold=0.7)
     # check if the zipped tbl files exist
     assert Path("ambig.tbl.tgz").exists()
+    # check if log file exists
+    assert Path("arctic3d_restraints.log").exists()
     # check the correct number of tbl files exist
     ls_tbl = len(glob.glob("ambig*tbl"))
     assert ls_tbl == 4
diff --git a/tests/test_output.py b/tests/test_output.py
index cb18c6d..a4dfee9 100644
--- a/tests/test_output.py
+++ b/tests/test_output.py
@@ -4,6 +4,7 @@ from pathlib import Path
 import pytest
 
 from arctic3d.modules.output import (
+    create_output_folder,
     output_pdb,
     read_residues_probs,
     setup_output_folder,
@@ -154,16 +155,29 @@ def test_output_pdb(inp_pdb):
     os.unlink(output_files[0])
 
 
-def test_run_dir():
+def test_create_output_folder():
     """Test if the expected run_dir is effectively created."""
-    run_dir = "run_dir"
     uniprot_id = "fake_uniprot"
+    create_output_folder(output_dir=None, uniprot_id=uniprot_id)
+    exp_run_dir = Path(f"arctic3d-{uniprot_id}")
+    assert Path.exists(exp_run_dir)
+    os.rmdir(exp_run_dir)
+
+
+def test_setup_output_folder(inp_pdb):
+    """Test the correct setup of the output folder."""
+    run_dir = "dummy_output"
     start_cwd = os.getcwd()
-    setup_output_folder(uniprot_id, [], run_dir)
+    create_output_folder(run_dir)
+    input_files = {"pdb": inp_pdb}
+    setup_output_folder(run_dir, input_files)
     obs_cwd = Path(os.getcwd())
     exp_cwd = Path(start_cwd, run_dir)
     assert exp_cwd == obs_cwd
     os.chdir(start_cwd)
+    assert Path.exists(Path(run_dir, "input_data"))
+    assert Path.exists(Path(run_dir, "input_data", inp_pdb.name))
+    os.unlink(Path(run_dir, "input_data", inp_pdb.name))
     os.rmdir(Path(run_dir, "input_data"))
     os.rmdir(run_dir)