Repository: widdowquinn/pyani

Issue:
Add use of API key to `pyani download`
#### Summary:

I'm getting Entrez API rate limit errors when using `pyani` on a cluster. This may be due to multiple requests coming in from the same machine - even if not the same job.

The NCBI API key policy is described [https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/](here).

The Biopython Entrez interface allows specification of an API key via the `api_key` argument.

It seems sensible to add a command-line option to `pyani download` to use the API key in addition to the user email.

Discussion:

Patch:
diff --git a/docs/subcmd_download.rst b/docs/subcmd_download.rst
index 387a45f..449bae5 100644
--- a/docs/subcmd_download.rst
+++ b/docs/subcmd_download.rst
@@ -8,12 +8,13 @@ The ``download`` subcommand controls download of genome files from the `NCBI Ass
 
 .. code-block:: text
 
-    usage: pyani download [-h] [-l LOGFILE] [-v] [--disable_tqdm] -t TAXON
-                            --email EMAIL [--retries RETRIES]
-                            [--batchsize BATCHSIZE] [--timeout TIMEOUT] [-f]
-                            [--noclobber] [--labels LABELFNAME]
-                            [--classes CLASSFNAME] [--kraken] [--dry-run]
-                            outdir
+    usage: pyani.py download [-h] [-l LOGFILE] [-v] [--disable_tqdm] -t TAXON
+                         --email EMAIL [--api_key API_KEYPATH]
+                         [--retries RETRIES] [--batchsize BATCHSIZE]
+                         [--timeout TIMEOUT] [-f] [--noclobber]
+                         [--labels LABELFNAME] [--classes CLASSFNAME]
+                         [--kraken] [--dry-run]
+                         outdir
 
 --------------------
 Positional arguments
@@ -26,6 +27,9 @@ Positional arguments
 Flagged arguments
 -----------------
 
+``--api_key PATH_TO_API_KEY``
+    The program will attempt to use an NCBI API key located at ``PATH_TO_API_KEY``. Default: ``~/.ncbi/api_key``
+
 ``--batchsize BATCHSIZE``
     The download process will attempt to download assemblies in multiples of ``BATCHSIZE``. Default: 10000
 
diff --git a/pyani/download.py b/pyani/download.py
index 1c9c803..20985ed 100644
--- a/pyani/download.py
+++ b/pyani/download.py
@@ -1,44 +1,42 @@
 # -*- coding: utf-8 -*-
-"""Module providing functions useful for downloading genomes from NCBI.
-
-(c) The James Hutton Institute 2016-2019
-Author: Leighton Pritchard
-
-Contact:
-leighton.pritchard@hutton.ac.uk
-
-Leighton Pritchard,
-Information and Computing Sciences,
-James Hutton Institute,
-Errol Road,
-Invergowrie,
-Dundee,
-DD2 5DA,
-Scotland,
-UK
-
-The MIT License
-
-Copyright (c) 2016-2019 The James Hutton Institute
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-"""
+# (c) The James Hutton Institute 2016-2019
+# (c) University of Strathclyde 2019
+# Author: Leighton Pritchard
+#
+# Contact:
+# leighton.pritchard@strath.ac.uk
+#
+# Leighton Pritchard,
+# Strathclyde Institute for Pharmacy and Biomedical Sciences,
+# Cathedral Street,
+# Glasgow,
+# G1 1XQ
+# Scotland,
+# UK
+#
+# The MIT License
+#
+# Copyright (c) 2016-2019 The James Hutton Institute
+# Copyright (c) 2019 University of Strathclyde
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Module providing functions useful for downloading genomes from NCBI."""
 
 import hashlib
 import os
@@ -90,6 +88,17 @@ def set_ncbi_email(email):
     Entrez.tool = "pyani.py"
 
 
+def parse_api_key(api_path):
+    """Parse the contents of the file for NCBI API key.
+
+    This function expects that the indicated file contains a single line
+    (perhaps with a newline) containing no data other than an NCBI API key.
+    """
+    with api_path.open() as ifh:
+        api_key = ifh.readline().strip()
+    return api_key
+
+
 # Get results from NCBI web history, in batches
 def entrez_batch_webhistory(record, expected, batchsize, retries, *fnargs, **fnkwargs):
     """Recover the Entrez data from a prior NCBI webhistory search.
@@ -192,12 +201,12 @@ def extract_filestem(esummary):
 
 
 # Get eSummary data for a single assembly UID
-def get_ncbi_esummary(asm_uid, retries):
+def get_ncbi_esummary(asm_uid, retries, api_key=None):
     """Obtain full eSummary info for the passed assembly UID."""
     # Obtain full eSummary data for the assembly
     summary = Entrez.read(
         entrez_retry(
-            Entrez.esummary, retries, db="assembly", id=asm_uid, report="full"
+            Entrez.esummary, retries, db="assembly", id=asm_uid, report="full", api_key=api_key
         ),
         validate=False,
     )
@@ -253,7 +262,7 @@ def compile_url(filestem, suffix, ftpstem):
     """
     gcstem, acc, _ = tuple(filestem.split("_", 2))
     aaval = acc.split(".")[0]
-    subdirs = "/".join([acc[i : i + 3] for i in range(0, len(aaval), 3)])
+    subdirs = "/".join([acc[i:i + 3] for i in range(0, len(aaval), 3)])
 
     url = "{0}/{1}/{2}/{3}/{3}_{4}".format(ftpstem, gcstem, subdirs, filestem, suffix)
     hashurl = "{0}/{1}/{2}/{3}/{4}".format(
@@ -354,7 +363,7 @@ def extract_contigs(fname, ename):
 
 # Using a genomes UID, create class and label text files
 def create_labels(classification, filestem, genomehash):
-    """Return class and label text from UID classification.
+    r"""Return class and label text from UID classification.
 
     - classification  Classification named tuple (org, genus, species, strain)
     - filestem        filestem of input genome file
diff --git a/pyani/pyani_graphics.py b/pyani/pyani_graphics.py
index c782e52..08136c3 100644
--- a/pyani/pyani_graphics.py
+++ b/pyani/pyani_graphics.py
@@ -133,14 +133,14 @@ def get_seaborn_colorbar(dfr, classes):
 
 def add_seaborn_labels(fig, params):
     """Add genome ticklabels to Seaborn heatmap"""
-    xticklabels = [
-        params.labels[int(_._text)] for _ in fig.ax_heatmap.get_xticklabels()
-    ]
-    yticklabels = [
-        params.labels[int(_._text)] for _ in fig.ax_heatmap.get_yticklabels()
-    ]
-    fig.ax_heatmap.set_xticklabels(xticklabels, rotation=90)
-    fig.ax_heatmap.set_yticklabels(yticklabels, rotation=0)
+    # xticklabels = [
+    #     params.labels[int(_._text)] for _ in fig.ax_heatmap.get_xticklabels()
+    # ]
+    # yticklabels = [
+    #     params.labels[int(_._text)] for _ in fig.ax_heatmap.get_yticklabels()
+    # ]
+    fig.ax_heatmap.set_xticklabels(fig.ax_heatmap.get_xticklabels(), rotation=90)
+    fig.ax_heatmap.set_yticklabels(fig.ax_heatmap.get_yticklabels(), rotation=0)
     return fig
 
 
diff --git a/pyani/scripts/parsers/download_parser.py b/pyani/scripts/parsers/download_parser.py
index 72b1179..e794314 100644
--- a/pyani/scripts/parsers/download_parser.py
+++ b/pyani/scripts/parsers/download_parser.py
@@ -1,46 +1,42 @@
 # -*- coding: utf-8 -*-
-"""download_parser.py
-
-Provides parser for download subcommand
-
-(c) The James Hutton Institute 2016-2019
-Author: Leighton Pritchard
-
-Contact:
-leighton.pritchard@hutton.ac.uk
-
-Leighton Pritchard,
-Information and Computing Sciences,
-James Hutton Institute,
-Errol Road,
-Invergowrie,
-Dundee,
-DD2 5DA,
-Scotland,
-UK
-
-The MIT License
-
-Copyright (c) 2016-2019 The James Hutton Institute
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-"""
+# (c) The James Hutton Institute 2016-2019
+# (c) University of Strathclyde 2019
+# Author: Leighton Pritchard
+#
+# Contact:
+# leighton.pritchard@strath.ac.uk
+#
+# Leighton Pritchard,
+# Strathclyde Institute for Pharmacy and Biomedical Sciences,
+# Cathedral Street,
+# Glasgow,
+# G1 1XQ
+# Scotland,
+# UK
+#
+# The MIT License
+#
+# Copyright (c) 2016-2019 The James Hutton Institute
+# Copyright (c) 2019 University of Strathclyde
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Provides parser for download subcommand."""
 
 from argparse import ArgumentDefaultsHelpFormatter
 
@@ -58,6 +54,7 @@ def build(subps, parents=None):
 
     -t, --taxon (NCBI taxonomy IDs - comma-separated list, or one ID)
     --email     (email for providing to Entrez services)
+    --api_key   (path to file containing personal API key for Entrez)
     --retries   (number of Entrez retry attempts to make)
     --batchsize (number of Entrez records to download in a batch)
     --timeout   (how long to wait for Entrez query timeout)
@@ -91,6 +88,14 @@ def build(subps, parents=None):
         help="Email associated with NCBI queries (required)",
         required=True,
     )
+    # Optional argument for faster/more robust NCBI query
+    parser.add_argument(
+        "--api_key",
+        dest="api_keypath",
+        action="store",
+        default="~/.ncbi/api_key",
+        help="Path to NCBI Entrez API key file",
+    )
     # Arguments controlling connection to NCBI for download
     parser.add_argument(
         "--retries",
diff --git a/pyani/scripts/pyani_script.py b/pyani/scripts/pyani_script.py
index da91b1f..4ada3a0 100644
--- a/pyani/scripts/pyani_script.py
+++ b/pyani/scripts/pyani_script.py
@@ -1,43 +1,45 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+# (c) The James Hutton Institute 2016-2019
+# (c) University of Strathclyde 2019
+# Author: Leighton Pritchard
+#
+# Contact:
+# leighton.pritchard@strath.ac.uk
+#
+# Leighton Pritchard,
+# Strathclyde Institute for Pharmacy and Biomedical Sciences,
+# Cathedral Street,
+# Glasgow,
+# G1 1XQ
+# Scotland,
+# UK
+#
+# The MIT License
+#
+# Copyright (c) 2016-2019 The James Hutton Institute
+# Copyright (c) 2019 University of Strathclyde
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
 """pyani_script.py
 
 Implements the pyani script for classifying prokaryotic genomes.
-
-(c) The James Hutton Institute 2017-2018
-Author: Leighton Pritchard
-
-Contact: leighton.pritchard@hutton.ac.uk
-Leighton Pritchard,
-Information and Computing Sciences,
-James Hutton Institute,
-Errol Road,
-Invergowrie,
-Dundee,
-DD2 5DA,
-Scotland,
-UK
-
-The MIT License
-
-Copyright (c) 2017-2018 The James Hutton Institute
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
 """
 
 import sys
diff --git a/pyani/scripts/subcommands/subcmd_download.py b/pyani/scripts/subcommands/subcmd_download.py
index 2ded676..7684145 100644
--- a/pyani/scripts/subcommands/subcmd_download.py
+++ b/pyani/scripts/subcommands/subcmd_download.py
@@ -1,51 +1,48 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-
-"""subcmd_download.py
-
-Provides the download subcommand for pyani
-
-(c) The James Hutton Institute 2017-18
-
-Author: Leighton Pritchard
-Contact: leighton.pritchard@hutton.ac.uk
-
-Leighton Pritchard,
-Information and Computing Sciences,
-James Hutton Institute,
-Errol Road,
-Invergowrie,
-Dundee,
-DD6 9LH,
-Scotland,
-UK
-
-The MIT License
-
-Copyright (c) 2017-18 The James Hutton Institute
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-"""
+# (c) The James Hutton Institute 2016-2019
+# (c) University of Strathclyde 2019
+# Author: Leighton Pritchard
+#
+# Contact:
+# leighton.pritchard@strath.ac.uk
+#
+# Leighton Pritchard,
+# Strathclyde Institute for Pharmacy and Biomedical Sciences,
+# Cathedral Street,
+# Glasgow,
+# G1 1XQ
+# Scotland,
+# UK
+#
+# The MIT License
+#
+# Copyright (c) 2016-2019 The James Hutton Institute
+# Copyright (c) 2019 University of Strathclyde
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Provides the download subcommand for pyani."""
 
 import os
 
 from collections import namedtuple
+from pathlib import Path
 
 from Bio import SeqIO
 
@@ -66,20 +63,26 @@ def subcmd_download(args, logger):
 
     # Set Entrez email
     download.set_ncbi_email(args.email)
-    logger.info("Setting Entrez email address: %s", args.email)
+    logger.info(f"Setting Entrez email address: {args.email}")
+
+    # Parse Entrez API key, if provided
+    api_path = Path(os.path.expanduser(args.api_keypath))
+    if not api_path.is_file():
+        logger.warning(f"API path {api_path} not a valid file. Not using API key.")
+        api_key = None
+    else:
+        api_key = download.parse_api_key(api_path)
+        logger.info(f"API key recovered from {api_path}")
 
     # Get list of taxon IDs to download
     taxon_ids = download.split_taxa(args.taxon)
-    logger.info("Taxon IDs received: %s", taxon_ids)
+    logger.info(f"Taxon IDs received: {taxon_ids}")
 
     # Get assembly UIDs for each taxon
     asm_dict = tools.make_asm_dict(taxon_ids, args.retries)
     for tid, uids in asm_dict.items():
         logger.info(
-            "Taxon ID summary\n\tQuery: " + "%s\n\tasm count: %s\n\tUIDs: %s",
-            tid,
-            len(uids),
-            uids,
+            f"Taxon ID summary\n\tQuery: {tid}\n\tasm count: {len(uids)}\n\tUIDs: {uids}",
         )
 
     # Compile outputs to write class and label files, and a list of
@@ -89,7 +92,7 @@ def subcmd_download(args, logger):
     labels = []
     skippedlist = []
     Skipped = namedtuple(
-        "Skipped", "taxon_id accession organism strain " + "url dltype"
+        "Skipped", "taxon_id accession organism strain url dltype"
     )
 
     # Download contigs and hashes for each assembly UID in the list
@@ -97,31 +100,33 @@ def subcmd_download(args, logger):
     # downloaded or skipped (with skipped genome information preserved in
     # skippedlist), and class/label info will be collated, ready for writing
     # to file.
+    # Summary information is reported to the logger for each eSummary that
+    # can be recovered
     for tid, uids in asm_dict.items():
-        logger.info("Downloading contigs for Taxon ID %s", tid)
+        logger.info(f"Downloading contigs for Taxon ID {tid}")
         for uid in uids:
             # Obtain eSummary
-            logger.info("Get eSummary information for UID %s", uid)
-            esummary, filestem = download.get_ncbi_esummary(uid, args.retries)
+            logger.info(f"Get eSummary information for UID {uid}")
+            esummary, filestem = download.get_ncbi_esummary(uid, args.retries, api_key)
             uid_class = download.get_ncbi_classification(esummary)
 
             # Report summary
             outstr = "\n\t".join(
                 [
-                    "Species Taxid: %s" % esummary["SpeciesTaxid"],
-                    "TaxID: %s" % esummary["Taxid"],
-                    "Accession: %s" % esummary["AssemblyAccession"],
-                    "Name: %s" % esummary["AssemblyName"],
-                    "Organism: %s" % uid_class.organism,
-                    "Genus: %s" % uid_class.genus,
-                    "Species: %s" % uid_class.species,
-                    "Strain: %s" % uid_class.strain,
+                    f"Species Taxid: {esummary['SpeciesTaxid']}",
+                    f"TaxID: {esummary['Taxid']}",
+                    f"Accession: {esummary['AssemblyAccession']}",
+                    f"Name: {esummary['AssemblyName']}",
+                    f"Organism: {uid_class.organism}",
+                    f"Genus: {uid_class.genus}",
+                    f"Species: {uid_class.species}",
+                    f"Strain: {uid_class.strain}",
                 ]
             )
-            logger.info("eSummary information:\n\t%s", outstr)
+            logger.info(f"eSummary information:\n\t{outstr}")
             if args.dryrun:
                 logger.warning(
-                    "(dry-run) skipping download of %s", esummary["AssemblyAccession"]
+                    f"(dry-run) skipping download of {esummary['AssemblyAccession']}"
                 )
                 continue
 
@@ -130,7 +135,7 @@ def subcmd_download(args, logger):
             dlfiledata = DLFileData(
                 filestem, "ftp://ftp.ncbi.nlm.nih.gov/genomes/all", "genomic.fna.gz"
             )
-            logger.info("Retrieving URLs for %s", filestem)
+            logger.info(f"Retrieving URLs for {filestem}")
             # Try RefSeq first
             dlstatus = tools.download_genome_and_hash(
                 args,
@@ -153,7 +158,7 @@ def subcmd_download(args, logger):
                     )
                 )
                 logger.warning(
-                    "RefSeq failed. Trying GenBank alternative " + "assembly"
+                    "RefSeq failed. Trying GenBank alternative assembly"
                 )
                 # Try GenBank assembly
                 dlstatus = tools.download_genome_and_hash(
@@ -179,32 +184,32 @@ def subcmd_download(args, logger):
                     continue  # Move straight on to the next download
 
             # One of the downloads worked: report information
-            logger.info("Downloaded from URL: %s", dlstatus.url)
-            logger.info("Wrote assembly to: %s", dlstatus.outfname)
-            logger.info("Wrote MD5 hashes to: %s", dlstatus.outfhash)
+            logger.info(f"Downloaded from URL: {dlstatus.url}")
+            logger.info(f"Wrote assembly to: {dlstatus.outfname}")
+            logger.info(f"Wrote MD5 hashes to: {dlstatus.outfhash}")
 
             # Check hash for the download
             hashstatus = download.check_hash(dlstatus.outfname, dlstatus.outfhash)
-            logger.info("Local MD5 hash: %s", hashstatus.localhash)
-            logger.info("NCBI MD5 hash: %s", hashstatus.filehash)
+            logger.info(f"Local MD5 hash: {hashstatus.localhash}")
+            logger.info(f"NCBI MD5 hash: {hashstatus.filehash}")
             if hashstatus.passed:
                 logger.info("MD5 hash check passed")
             else:
-                logger.warning("MD5 hash check failed. " + "Please check and retry.")
+                logger.warning("MD5 hash check failed. Please check and retry.")
 
             # Extract downloaded files
             ename = os.path.splitext(dlstatus.outfname)[0]
             if os.path.exists(ename) and args.noclobber:
-                logger.warning("Output file %s exists, not extracting", ename)
+                logger.warning(f"Output file {ename} exists, not extracting")
             else:
-                logger.info("Extracting archive %s to %s", dlstatus.outfname, ename)
+                logger.info(f"Extracting archive {dlstatus.outfname} to {ename}")
                 download.extract_contigs(dlstatus.outfname, ename)
 
             # Modify sequence ID header if Kraken option active
             if args.kraken:
                 logger.warning("Modifying downloaded sequence for Kraken compatibility")
                 seqdata = list(SeqIO.parse(ename, "fasta"))
-                logger.info("Modifying %s", ename)
+                logger.info(f"Modifying {ename}")
                 for seq in seqdata:
                     seq.id = "|".join(
                         [seq.id, "kraken:taxid", esummary["SpeciesTaxid"]]
@@ -212,7 +217,7 @@ def subcmd_download(args, logger):
                 SeqIO.write(seqdata, ename, "fasta")
 
             # Create MD5 hash for the downloaded contigs
-            logger.info("Creating local MD5 hash for %s" % ename)
+            logger.info(f"Creating local MD5 hash for {ename}")
             hashfname = os.path.splitext(ename)[0] + ".md5"
             datahash = download.create_hash(ename)
             logger.info("Writing hash to %s" % hashfname)
@@ -223,38 +228,36 @@ def subcmd_download(args, logger):
             classes.append(classtxt)
             labels.append(labeltxt)
             logger.info(
-                "Label and class file entries\n" + "\tLabel: %s\n\tClass: %s",
-                labeltxt,
-                classtxt,
+                f"Label and class file entries\n\tLabel: {labeltxt}\n\tClass: {classtxt}"
             )
 
     # Write class and label files
     classfname = os.path.join(args.outdir, args.classfname)
-    logger.info("Writing classes file to %s", classfname)
+    logger.info(f"Writing classes file to {classfname}")
     if os.path.exists(classfname) and args.noclobber:
-        logger.warning("Class file %s exists, not overwriting", classfname)
+        logger.warning(f"Class file {classfname} exists, not overwriting")
     else:
         with open(classfname, "w") as ofh:
             ofh.write("\n".join(classes) + "\n")
 
     labelfname = os.path.join(args.outdir, args.labelfname)
-    logger.info("Writing labels file to %s", labelfname)
+    logger.info(f"Writing labels file to {labelfname}")
     if os.path.exists(labelfname) and args.noclobber:
-        logger.warning("Labels file %s exists, not overwriting", labelfname)
+        logger.warning(f"Labels file {labelfname} exists, not overwriting")
     else:
         with open(labelfname, "w") as ofh:
             ofh.write("\n".join(labels) + "\n")
 
     # Report skipped genome list
     if skippedlist:
-        logger.warning("%d genome downloads were skipped", len(skippedlist))
+        logger.warning(f"{len(skippedlist)} genome downloads were skipped")
         for skipped in skippedlist:
             outstr = "\n\t".join(
                 [
-                    "taxon id: %s" % skipped.taxon_id,
-                    "accession: %s" % skipped.accession,
-                    "URL: %s" % skipped.url,
-                    "source: %s" % skipped.dltype,
+                    f"taxon id: {skipped.taxon_id}",
+                    f"accession: {skipped.accession}",
+                    f"URL: {skipped.url}",
+                    f"source: {skipped.dltype}",
                 ]
             )
-            logger.warning("%s %s:\n\t%s", skipped.organism, skipped.strain, outstr)
+            logger.warning(f"{skipped.organism} {skipped.strain}:\n\t{outstr}")
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 17fc048..4a6c380 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -5,6 +5,7 @@ coverage
 doc8
 flake8
 jinja2
+pydocstyle
 pylint
 pytest
 pytest-cov


Test patch:
diff --git a/tests/test_graphics.py b/tests/test_graphics.py
index 2a311ae..c4b6f2a 100644
--- a/tests/test_graphics.py
+++ b/tests/test_graphics.py
@@ -1,8 +1,43 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-"""test_graphics.py
-
-Tests for pyani graphics
+# (c) The James Hutton Institute 2016-2019
+# (c) University of Strathclyde 2019
+# Author: Leighton Pritchard
+#
+# Contact:
+# leighton.pritchard@strath.ac.uk
+#
+# Leighton Pritchard,
+# Strathclyde Institute for Pharmacy and Biomedical Sciences,
+# Cathedral Street,
+# Glasgow,
+# G1 1XQ
+# Scotland,
+# UK
+#
+# The MIT License
+#
+# Copyright (c) 2016-2019 The James Hutton Institute
+# Copyright (c) 2019 University of Strathclyde
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Tests for pyani graphics.
 
 These tests are intended to be run from the repository root using:
 
@@ -10,44 +45,6 @@ pytest -v
 
 print() statements will be caught by nosetests unless there is an
 error. They can also be recovered with the -s option.
-
-(c) The James Hutton Institute 2017-2018
-Author: Leighton Pritchard
-
-Contact:
-leighton.pritchard@hutton.ac.uk
-
-Leighton Pritchard,
-Information and Computing Sciences,
-James Hutton Institute,
-Errol Road,
-Invergowrie,
-Dundee,
-DD2 5DA,
-Scotland,
-UK
-
-The MIT License
-
-Copyright (c) 2017-2018 The James Hutton Institute
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
 """
 
 import os
@@ -89,7 +86,7 @@ def draw_format_method(fmt, mth):
     method_params = pyani_graphics.Params(
         params[mth](df)[stem], inputs["labels"], inputs["classes"]
     )
-    fn[mth](df, outfilename, title="%s:%s test" % (mth, fmt), params=method_params)
+    fn[mth](df, outfilename, title=f"{mth}:{fmt} test", params=method_params)
 
 
 def test_png_mpl():
diff --git a/tests/test_legacy_scripts.py b/tests/test_legacy_scripts.py
index f546be9..4a57524 100644
--- a/tests/test_legacy_scripts.py
+++ b/tests/test_legacy_scripts.py
@@ -1,9 +1,43 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-"""test_legacy_scripts.py
-
-Test legacy average_nucleotide_identity.py and
-genbank_download_genomes_by_taxon.py scripts
+# (c) The James Hutton Institute 2016-2019
+# (c) University of Strathclyde 2019
+# Author: Leighton Pritchard
+#
+# Contact:
+# leighton.pritchard@strath.ac.uk
+#
+# Leighton Pritchard,
+# Strathclyde Institute for Pharmacy and Biomedical Sciences,
+# Cathedral Street,
+# Glasgow,
+# G1 1XQ
+# Scotland,
+# UK
+#
+# The MIT License
+#
+# Copyright (c) 2016-2019 The James Hutton Institute
+# Copyright (c) 2019 University of Strathclyde
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Test legacy scripts.
 
 The test suite is intended to be run from the repository root using:
 
@@ -12,43 +46,6 @@ pytest -v
 The two legacy scripts download genomes, then carry out ANI analysis. The
 pytest ordering plug-in is used to guarantee that the download script
 tests are conducted first.
-
-(c) The James Hutton Institute 2019
-
-Author: Leighton Pritchard
-Contact: leighton.pritchard@hutton.ac.uk
-
-Leighton Pritchard,
-Information and Computing Sciences,
-James Hutton Institute,
-Errol Road,
-Invergowrie,
-Dundee,
-DD6 9LH,
-Scotland,
-UK
-
-The MIT License
-
-Copyright (c) 2019 The James Hutton Institute
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
 """
 
 import logging
@@ -74,7 +71,7 @@ Executables = namedtuple(
 # Convenience struct for test directories
 TestDirs = namedtuple("TestDirs", "outdir tgtdir dldir")
 
-
+@pytest.mark.skip(reason="legacy scripts not ready for test")
 class TestLegacyScripts(PyaniTestCase):
     """Class defining tests of the pyani download subcommand."""
 
diff --git a/tests/test_subcmd_01_download.py b/tests/test_subcmd_01_download.py
index 8de373a..e3f0eb8 100644
--- a/tests/test_subcmd_01_download.py
+++ b/tests/test_subcmd_01_download.py
@@ -1,8 +1,43 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-"""test_subcmd_01_download.py
-
-Test download subcommand for pyani
+# (c) The James Hutton Institute 2016-2019
+# (c) University of Strathclyde 2019
+# Author: Leighton Pritchard
+#
+# Contact:
+# leighton.pritchard@strath.ac.uk
+#
+# Leighton Pritchard,
+# Strathclyde Institute for Pharmacy and Biomedical Sciences,
+# Cathedral Street,
+# Glasgow,
+# G1 1XQ
+# Scotland,
+# UK
+#
+# The MIT License
+#
+# Copyright (c) 2016-2019 The James Hutton Institute
+# Copyright (c) 2019 University of Strathclyde
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Test download subcommand for pyani.
 
 The test suite is intended to be run from the repository root using:
 
@@ -16,43 +51,6 @@ by test name, with values representing command-line options.
 
 For each test, command line options are defined in a Namespace and
 passed as the sole argument to the appropriate subcommand.
-
-(c) The James Hutton Institute 2018-2019
-
-Author: Leighton Pritchard
-Contact: leighton.pritchard@hutton.ac.uk
-
-Leighton Pritchard,
-Information and Computing Sciences,
-James Hutton Institute,
-Errol Road,
-Invergowrie,
-Dundee,
-DD6 9LH,
-Scotland,
-UK
-
-The MIT License
-
-Copyright (c) 2018-2019 The James Hutton Institute
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
 """
 
 import logging
@@ -94,6 +92,7 @@ class TestDownloadSubcommand(unittest.TestCase):
                 force=False,
                 kraken=False,
                 disable_tqdm=True,
+                api_keypath="~/.ncbi/api_key",
             ),
             "C_blochmannia": Namespace(
                 outdir=self.outdir,
@@ -109,6 +108,7 @@ class TestDownloadSubcommand(unittest.TestCase):
                 noclobber=False,
                 dryrun=False,
                 disable_tqdm=True,
+                api_keypath="~/.ncbi/api_key",
             ),
             "kraken": Namespace(
                 outdir=self.krakendir,
@@ -124,6 +124,7 @@ class TestDownloadSubcommand(unittest.TestCase):
                 noclobber=False,
                 dryrun=False,
                 disable_tqdm=True,
+                api_keypath="~/.ncbi/api_key",
             ),
         }
 
diff --git a/tests/test_subcmd_08_anib.py b/tests/test_subcmd_08_anib.py
index 3f9f6fc..0646481 100644
--- a/tests/test_subcmd_08_anib.py
+++ b/tests/test_subcmd_08_anib.py
@@ -58,10 +58,11 @@ import logging
 import os
 import unittest
 
+from argparse import Namespace
 from collections import namedtuple
 from pathlib import Path
 
-from argparse import Namespace
+import pytest
 
 from pyani.scripts import subcommands
 
@@ -76,6 +77,7 @@ DirPaths = namedtuple("DirPaths", "indir outdir")
 LabelPaths = namedtuple("LabelPaths", "classes labels")
 
 
+@pytest.mark.xfail(reason="ANIb is not currently fully implemented")
 class TestANIbsubcommand(unittest.TestCase):
     """Class defining tests of the pyani anib subcommand."""