Repository: biopython/biopython

Issue:
ADJUSTED_FOR_SHORT_QUERY
### Setup

I am reporting a problem with Biopython version, Python version, and operating
system as follows:

3.6.3 (v3.6.3:2c5fed86e0, Oct  3 2017, 00:32:08) 
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
CPython
Darwin-17.7.0-x86_64-i386-64bit
1.70

### Expected behaviour

I am running a search for a peptide sequence using blastp on the PDB database. I would like to set the parameter to adjust for short queries, but that parameter is not available.  As a result, the output returned by the python query does not match the output produced by the website when that parameter is activated

### Steps to reproduce

result_handle=NCBIWWW.qblast('blastp', 'pdb_v5', 'QRFSQPTFKLPQGRLTLSRKF')

run above query and notice that no results are returned with the is query, while a large number or results are returned when run with the Adjust for short query parameter set to on (located in algorithm parameters section) in the website.



Discussion:
Have you seen any NCBI documentation on their QBLAST API indicating if it supports this, and how to do this? Or are you willing and able to try investigating this on your own?
I haven't seen any documentation, but by looking at the service calls I've
been able to include the parameter into the call used by the qblast method.



On Sun, May 5, 2019 at 12:29 PM Peter Cock <notifications@github.com> wrote:

> Have you seen any NCBI documentation on their QBLAST API indicating if it
> supports this, and how to do this? Or are you willing and able to try
> investigating this on your own?
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/biopython/biopython/issues/2049#issuecomment-489441673>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AB6AUZLDF2OYFPNT5UORKVLPT4DPNANCNFSM4HK3PAXA>
> .
>


Resolving patch:
diff --git a/Bio/Blast/NCBIWWW.py b/Bio/Blast/NCBIWWW.py
--- a/Bio/Blast/NCBIWWW.py
+++ b/Bio/Blast/NCBIWWW.py
@@ -16,12 +16,16 @@
 
 from __future__ import print_function
 
+import warnings
+
 from Bio._py3k import StringIO
 from Bio._py3k import _as_string, _as_bytes
 from Bio._py3k import urlopen as _urlopen
 from Bio._py3k import urlencode as _urlencode
 from Bio._py3k import Request as _Request
 
+from Bio import BiopythonWarning
+
 
 NCBI_BLAST_URL = "https://blast.ncbi.nlm.nih.gov/Blast.cgi"
 
@@ -35,7 +39,7 @@ def qblast(program, database, sequence, url_base=NCBI_BLAST_URL,
            other_advanced=None, perc_ident=None, phi_pattern=None,
            query_file=None, query_believe_defline=None, query_from=None,
            query_to=None, searchsp_eff=None, service=None, threshold=None,
-           ungapped_alignment=None, word_size=None,
+           ungapped_alignment=None, word_size=None, short_query=None,
            alignments=500, alignment_view=None, descriptions=500,
            entrez_links_new_window=None, expect_low=None, expect_high=None,
            format_entrez_query=None, format_object=None, format_type='XML',
@@ -44,13 +48,18 @@ def qblast(program, database, sequence, url_base=NCBI_BLAST_URL,
            ):
     """BLAST search using NCBI's QBLAST server or a cloud service provider.
 
-    Supports all parameters of the qblast API for Put and Get.
+    Supports all parameters of the old qblast API for Put and Get.
 
-    Please note that BLAST on the cloud supports the NCBI-BLAST Common
-    URL API (http://ncbi.github.io/blast-cloud/dev/api.html). To
-    use this feature, please set url_base to
-    'http://host.my.cloud.service.provider.com/cgi-bin/blast.cgi' and
-    format_object='Alignment'. For more details, please see
+    Please note that NCBI uses the new Common URL API for BLAST searches
+    on the internet (http://ncbi.github.io/blast-cloud/dev/api.html). Thus,
+    some of the parameters used by this function are not (or are no longer)
+    officially supported by NCBI. Although they are still functioning, this
+    may change in the future.
+
+    The Common URL API (http://ncbi.github.io/blast-cloud/dev/api.html) allows
+    doing BLAST searches on cloud servers. To use this feature, please set
+    ``url_base='http://host.my.cloud.service.provider.com/cgi-bin/blast.cgi'``
+    and ``format_object='Alignment'``. For more details, please see
     https://blast.ncbi.nlm.nih.gov/Blast.cgi?PAGE_TYPE=BlastDocs&DOC_TYPE=CloudBlast
 
     Some useful parameters:
@@ -68,6 +77,10 @@ def qblast(program, database, sequence, url_base=NCBI_BLAST_URL,
      - entrez_query   Entrez query to limit Blast search
      - hitlist_size   Number of hits to return. Default 50
      - megablast      TRUE/FALSE whether to use MEga BLAST algorithm (blastn only)
+     - short_query    TRUE/FALSE whether to adjust the search parameters for a
+                      short query sequence. Note that this will override
+                      manually set parameters like word size and e value. Turns
+                      off when sequence length is > 30 residues. Default: None.
      - service        plain, psi, phi, rpsblast, megablast (lower case)
 
     This function does no checking of the validity of the parameters
@@ -82,6 +95,24 @@ def qblast(program, database, sequence, url_base=NCBI_BLAST_URL,
         raise ValueError("Program specified is %s. Expected one of %s"
                          % (program, ", ".join(programs)))
 
+    # SHORT_QUERY_ADJUST throws an error when using blastn (wrong parameter
+    # assignment from NCBIs side).
+    # Thus we set the (known) parameters directly:
+    if short_query and program == 'blastn':
+        short_query = None
+        # We only use the 'short-query' parameters for short sequences:
+        if len(sequence) < 31:
+            expect = 1000
+            word_size = 7
+            nucl_reward = 1
+            filter = None
+            lcase_mask = None
+            warnings.warn('"SHORT_QUERY_ADJUST" is incorrectly implemented '
+                          '(by NCBI) for blastn. We bypass the problem by '
+                          'manually adjusting the search parameters. Thus, '
+                          'results may slightly differ from web page '
+                          'searches.', BiopythonWarning)
+
     # Format the "Put" command, which sends search requests to qblast.
     # Parameters taken from http://www.ncbi.nlm.nih.gov/BLAST/Doc/node5.html on 9 July 2007
     # Additional parameters are taken from http://www.ncbi.nlm.nih.gov/BLAST/Doc/node9.html on 8 Oct 2010
@@ -119,6 +150,7 @@ def qblast(program, database, sequence, url_base=NCBI_BLAST_URL,
         # ('RESULTS_FILE',...), - Can we use this parameter?
         ('SEARCHSP_EFF', searchsp_eff),
         ('SERVICE', service),
+        ('SHORT_QUERY_ADJUST', short_query),
         ('TEMPLATE_TYPE', template_type),
         ('TEMPLATE_LENGTH', template_length),
         ('THRESHOLD', threshold),


Test patch:
diff --git a/Tests/test_NCBI_qblast.py b/Tests/test_NCBI_qblast.py
--- a/Tests/test_NCBI_qblast.py
+++ b/Tests/test_NCBI_qblast.py
@@ -14,11 +14,13 @@
 
 """
 import unittest
+import warnings
 
 from Bio._py3k import HTTPError
 from Bio._py3k import StringIO
 
 from Bio import MissingExternalDependencyError
+from Bio import BiopythonWarning
 
 # We want to test these:
 from Bio.Blast import NCBIWWW
@@ -46,7 +48,9 @@ def test_blastp_nr_actin(self):
         # the actin related protein 2/3 complex, subunit 1B [Mus musculus]
         self.run_qblast("blastp", "nr", "NP_075631.2", 0.001,
                         "rat [ORGN]", {'megablast': 'FALSE'},
-                        ['9506405', '13592137', '37589612', '149064087', '56912225'])
+                        ["NP_112408.1", "AAH59131.1", "EDM14357.1",
+                         "NP_001008766.1", "NP_001102411.1", "EDL80109.1",
+                         "EDL80106.1", "NP_001100434.1", "AAI67084.1"])
 
     def test_pcr_primers(self):
         # This next example finds PCR primer matches in Chimpanzees, e.g. BRCA1:
@@ -71,7 +75,9 @@ def test_orchid_est(self):
                         GCAGAGTGGTGGCTCCTTGTCCCAGCAGCAGTAATTACTTTTTTTTCTCTTTTTGTTTCCAAATTAAGAA
                         ACATTAGTATCATATGGCTATTTGCTCAATTGCAGATTTCTTTCTTTTGTGAATG""",
                         0.0000001, None, {'megablast': 'FALSE'},
-                        ["21554275", "18409071", "296087288", "566183510"])
+                        ["XP_021665344.1", "XP_021615158.1", "XP_017223689.1",
+                         "OMP06800.1", "XP_021634873.1", "XP_021299673.1",
+                         "XP_002311451.2", "XP_021976565.1", "OMO90244.1"])
 
     def test_discomegablast(self):
         self.run_qblast("blastn", "nr",
@@ -122,6 +128,7 @@ def test_discomegablast(self):
                         }, ['XM_635681.1', 'XM_008496783.1'])
 
     def run_qblast(self, program, database, query, e_value, entrez_filter, additional_args, expected_hits):
+        """Do qblast searches with given parameters and analyze results."""
         try:
             if program == "blastn":
                 # Check the megablast parameter is accepted
@@ -141,6 +148,7 @@ def run_qblast(self, program, database, query, e_value, entrez_filter, additiona
         except HTTPError:
             # e.g. a proxy error
             raise MissingExternalDependencyError("internet connection failed")
+
         record = NCBIXML.read(handle)
 
         if record.query == "No definition line":
@@ -200,6 +208,40 @@ def test_parse_qblast_ref_page(self):
             handle = StringIO(f.read())
         self.assertRaises(ValueError, NCBIWWW._parse_qblast_ref_page, handle)
 
+    def test_short_query(self):
+        """Test SHORT_QUERY_ADJUST parameter."""
+        # Should give no hits:
+        my_search = NCBIWWW.qblast('blastp', 'nr', 'ICWENRM', hitlist_size=5)
+        my_hits = NCBIXML.read(my_search)
+        my_search.close()
+        self.assertEqual(len(my_hits.alignments), 0)
+
+        # Should give hits:
+        my_search = NCBIWWW.qblast('blastp', 'nr', 'ICWENRM', hitlist_size=5,
+                                   short_query=True)
+        my_hits = NCBIXML.read(my_search)
+        my_search.close()
+        self.assertEqual(len(my_hits.alignments), 5)
+
+        with warnings.catch_warnings(record=True) as w:
+            # Cause all warnings to always be triggered.
+            warnings.simplefilter('always')
+            # Trigger a warning.
+            my_search = NCBIWWW.qblast('blastn', 'nt', 'ATGTCAACTTCAGAA',
+                                       hitlist_size=5, short_query=True)
+            # Verify some things
+            self.assertEqual(len(w), 1)
+            self.assertEqual(w[-1].category, BiopythonWarning)
+            self.assertIn('blastn', str(w[-1].message))
+            my_hits = NCBIXML.read(my_search)
+            my_search.close()
+            self.assertEqual(len(my_hits.alignments), 5)
+
+    def test_error_conditions(self):
+        """Test if exceptions were properly handled."""
+        self.assertRaises(ValueError, NCBIWWW.qblast, 'megablast', 'nt',
+                          'ATGCGTACGCAGCTAAAGTAAACCTATCGCGTCTCCT')
+
 
 if __name__ == "__main__":
     runner = unittest.TextTestRunner(verbosity=2)