Repository: virtool/virtool
Issue: Unhandled exception when fetching newer Genbank records without GIs

An example Genbank entry: https://www.ncbi.nlm.nih.gov/nuccore/JX513889.




Changed files:
setup.py, virtool/genbank.py, virtool/api/genbank.py, tests/api/test_otus.py, virtool/api/otus.py, tests/api/test_genbank.py

diff --git a/setup.py b/setup.py
index aa5d3e4bb..1c04318d7 100644
--- a/setup.py
+++ b/setup.py
@@ -3,13 +3,16 @@ from cx_Freeze import setup, Executable
 # Dependencies are automatically detected, but it might need
 # fine tuning.
 build_exe_options = {
-    "excludes": [
-        "numpy"
-    ],
     "bin_includes": [
         "libcrypto.so.1.0.0",
         "libssl.so.1.0.0"
     ],
+    "includes": [
+        "numpy",
+        "numpy.core._methods",
+        "numpy.lib",
+        "numpy.lib.format"
+    ],
     "packages": [
         "_cffi_backend",
         "appdirs",
diff --git a/tests/api/test_genbank.py b/tests/api/test_genbank.py
index 419e3221f..0fde98c46 100644
--- a/tests/api/test_genbank.py
+++ b/tests/api/test_genbank.py
@@ -1,25 +1,31 @@
+import aiohttp.client
 import pytest
 from aiohttp.test_utils import make_mocked_coro
 
 
-@pytest.mark.parametrize("not_found", [False, True])
-async def test_get(not_found, mocker, resp_is, spawn_client):
+@pytest.mark.parametrize("error", [False, True])
+async def test_get(error, mocker, resp_is, spawn_client):
     client = await spawn_client(authorize=True)
 
-    m_search = mocker.patch("virtool.genbank.search", new=make_mocked_coro(None if not_found else "foobar"))
-    m_fetch = mocker.patch("virtool.genbank.fetch", new=make_mocked_coro({"accession": "baz"}))
+    expected = {
+        "accession": "baz"
+    }
 
-    resp = await client.get("/api/genbank/NC_016574.1")
+    m_fetch = mocker.patch("virtool.genbank.fetch", make_mocked_coro(None if error else expected))
 
-    assert m_search.call_args[0][0] == client.app["settings"]
-    assert m_search.call_args[0][2] == "NC_016574.1"
+    resp = await client.get("/api/genbank/NC_016574.1")
 
-    if not_found:
+    if error:
         await resp_is.not_found(resp)
+        return
+
+    m_fetch.assert_called_with(
+        client.app["settings"],
+        mocker.ANY,
+        "NC_016574.1"
+    )
 
-    else:
-        assert m_fetch.call_args[0][0] == client.app["settings"]
-        assert m_fetch.call_args[0][2] == "foobar"
+    assert isinstance(m_fetch.call_args[0][1], aiohttp.client.ClientSession)
 
-        assert resp.status == 200
-        assert await resp.json() == {"accession": "baz"}
+    assert resp.status == 200
+    assert await resp.json() == expected
diff --git a/tests/api/test_otus.py b/tests/api/test_otus.py
index 337852002..7de73deb3 100644
--- a/tests/api/test_otus.py
+++ b/tests/api/test_otus.py
@@ -1605,6 +1605,9 @@ class TestCreateSequence:
             "otu_id": "6116cba1",
             "isolate_id": "cab8b360",
             "host": "Plant",
+            "reference": {
+                "id": "hxn167"
+            },
             "sequence": "ATGCGTGTACTG",
             "segment": None,
 
@@ -1643,6 +1646,9 @@ class TestCreateSequence:
             "otu_id": "6116cba1",
             "isolate_id": "cab8b360",
             "host": "Plant",
+            "reference": {
+                "id": "hxn167"
+            },
             "sequence": "ATGCGTGTACTG",
             "segment": None
         }]
@@ -1652,7 +1658,8 @@ class TestCreateSequence:
             "version": 1
         })
 
-        assert test_add_history.call_args[0][1:] == (
+        test_add_history.assert_called_with(
+            client.db,
             "create_sequence",
             old,
             new,
diff --git a/virtool/api/genbank.py b/virtool/api/genbank.py
index 57e0585c2..d06e4df6d 100644
--- a/virtool/api/genbank.py
+++ b/virtool/api/genbank.py
@@ -23,11 +23,9 @@ async def get(req):
     settings = req.app["settings"]
 
     async with aiohttp.ClientSession() as session:
-        gi = await virtool.genbank.search(settings, session, accession)
+        data = await virtool.genbank.fetch(settings, session, accession)
 
-        if not gi:
+        if data is None:
             return not_found()
 
-        data = await virtool.genbank.fetch(settings, session, gi)
-
         return json_response(data)
diff --git a/virtool/api/otus.py b/virtool/api/otus.py
index 4ec6cb99e..8a0745b03 100644
--- a/virtool/api/otus.py
+++ b/virtool/api/otus.py
@@ -676,7 +676,9 @@ async def create_sequence(req):
     if not document:
         return not_found()
 
-    if not await virtool.db.references.check_right(req, document["reference"]["id"], "modify_otu"):
+    ref_id = document["reference"]["id"]
+
+    if not await virtool.db.references.check_right(req, ref_id, "modify_otu"):
         return insufficient_rights()
 
     segment = data.get("segment", None)
@@ -689,6 +691,9 @@ async def create_sequence(req):
         "otu_id": otu_id,
         "isolate_id": isolate_id,
         "host": data.get("host", ""),
+        "reference": {
+            "id": ref_id
+        },
         "segment": segment
     })
 
diff --git a/virtool/genbank.py b/virtool/genbank.py
index e8d98e335..7035b74b8 100644
--- a/virtool/genbank.py
+++ b/virtool/genbank.py
@@ -1,20 +1,19 @@
-import re
+import logging
 import string
 
 import virtool.http.proxy
 
+logger = logging.getLogger(__name__)
+
 EMAIL = "dev@virtool.ca"
 TOOL = "virtool"
 
 FETCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
 
-SEARCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
-SEARCH_REGEX = re.compile("<Id>([0-9]+)</Id>")
-
 
-async def fetch(settings, session, gi):
+async def fetch(settings, session, accession):
     """
-    Fetch the Genbank record for the passed `gi`.
+    Fetch the Genbank record for the passed `accession`.
 
     :param settings: the application settings object
     :type settings: :class:`virtool.app_settings.Settings`
@@ -22,8 +21,8 @@ async def fetch(settings, session, gi):
     :param session: an aiohttp client session
     :type session: :class:`aiohttp.ClientSession`
 
-    :param gi: the GI to fetch
-    :type gi: Union[int,str]
+    :param accession: the accession to fetch
+    :type accession: Union[int,str]
 
     :return: parsed Genbank data
     :rtype: dict
@@ -32,7 +31,7 @@ async def fetch(settings, session, gi):
     params = {
         "db": "nuccore",
         "email": EMAIL,
-        "id": gi,
+        "id": accession,
         "retmode": "text",
         "rettype": "gb",
         "tool": TOOL
@@ -42,6 +41,12 @@ async def fetch(settings, session, gi):
 
         body = await resp.text()
 
+        if resp.status != 200:
+            if "Failed to retrieve sequence" not in body:
+                logger.warning("Unexpected Genbank error: {}".format(body))
+
+            return None
+
         data = {
             "host": ""
         }
@@ -66,42 +71,3 @@ async def fetch(settings, session, gi):
             data["sequence"] = sequence_field.upper()
 
         return data
-
-
-async def search(settings, session, accession):
-    """
-    Search for and return the GI associated with a given Genbank accession. Returns `None` if no GI can be found for
-    the `accession`.
-
-    :param settings: the application settings object
-    :type settings: :class:`virtool.app_settings.Settings`
-
-    :param session: an aiohttp client session
-    :type session: :class:`aiohttp.ClientSession`
-
-
-    :param accession: the accession to find a GI for
-    :type accession: str
-
-    :return: a GI
-    :rtype: Union[None, str]
-
-    """
-    params = {
-        "db": "nucleotide",
-        "term": "{}[accn]".format(accession),
-        "tool": TOOL,
-        "email": EMAIL
-    }
-
-    gi = None
-
-    async with virtool.http.proxy.ProxyRequest(settings, session.get, SEARCH_URL, params=params) as resp:
-        data = await resp.text()
-
-        match = SEARCH_REGEX.search(data)
-
-        if match:
-            gi = match.group(1)
-
-    return gi