Repository: biopython/biopython

Issue:
Entrez.read of batch gene esummary throws RuntimeError: cannot get document summary if any IDs have errors 
Hi, thanks for BioPython! I am having some trouble with Entrez.read throwing an exception if any result from a batch request has an error in it

Say I query 1000 gene IDs, and 1 fails, then I can't get the data out for the 999 that work

### Setup

I am reporting a problem with Biopython version, Python version, and operating
system as follows:

```python
import sys; print(sys.version)
import platform; print(platform.python_implementation()); print(platform.platform())
import Bio; print(Bio.__version__)
```

3.10.6 (main, Nov  2 2022, 18:53:38) [GCC 11.3.0]
CPython
Linux-5.15.0-52-generic-x86_64-with-glibc2.35
1.79

### Expected behaviour

Entrez.read returns a document, where the records with errors have some kind of error state

This will allow you to use any good results that were returned, and determine which records caused the error

### Actual behaviour

```
RuntimeError                              Traceback (most recent call last)
Cell In [1], line 14
     12 query_key = result["QueryKey"]
     13 data = Entrez.esummary(db="gene", webenv=web_env, query_key=query_key)
---> 14 document = Entrez.read(data, validate=False)

File /usr/local/lib/python3.10/dist-packages/Bio/Entrez/__init__.py:508, in read(handle, validate, escape)
    505 from .Parser import DataHandler
    507 handler = DataHandler(validate, escape)
--> 508 record = handler.read(handle)
    509 return record

File /usr/local/lib/python3.10/dist-packages/Bio/Entrez/Parser.py:335, in DataHandler.read(self, handle)
    333     raise TypeError("file should be opened in binary mode")
    334 try:
--> 335     self.parser.ParseFile(handle)
    336 except expat.ExpatError as e:
    337     if self.parser.StartElementHandler:
    338         # We saw the initial <!xml declaration, so we can be sure that
    339         # we are parsing XML data. Most likely, the XML file is
    340         # corrupted.

File ../Modules/pyexpat.c:468, in EndElement()

File /usr/local/lib/python3.10/dist-packages/Bio/Entrez/Parser.py:704, in DataHandler.endErrorElementHandler(self, name)
    701 if self.data:
    702     # error found:
    703     value = "".join(self.data)
--> 704     raise RuntimeError(value)
    705 # no error found:
    706 if self.element is not None:

RuntimeError: cannot get document summary
```


### Steps to reproduce

```
BAD_ID = True

from Bio import Entrez
gene_ids = ["107403241","107436001"]  # These work
if BAD_ID:
    gene_ids.append("127566430")  # This has error = cannot get document summary

request = Entrez.epost("gene", id=",".join(gene_ids))
result = Entrez.read(request)
web_env = result["WebEnv"]
query_key = result["QueryKey"]
data = Entrez.esummary(db="gene", webenv=web_env, query_key=query_key)
document = Entrez.read(data, validate=False)
```

Running with BAD_ID=False works as the bad record wasn't added


The bad record looks like:

```
<DocumentSummary uid="127566430">
<error>cannot get document summary</error>
</DocumentSummary>
```

The geneIDs are from https://ftp.ncbi.nlm.nih.gov/refseq/H_sapiens/Homo_sapiens.gene_info.gz



Discussion:
What do you think the code should do in this case?
Hi, expected behaviour is:

Entrez.read returns a document, where the records with errors have some kind of error state

This will allow you to use any good results that were returned, and determine which records caused the error
I think that is possible, but I guess that when using `Entrez.parse` instead of `Entrez.read`, you would still want a `RuntimeError`.
Cool. Happy for it to be behind a default parameter eg ```allow_errors=False``` to keep backwards compatibility if you need it

If anyone else has this error, a workaround (warning: deletes bad entries) is:

```
    from lxml import etree
    from io import BytesIO

    root = etree.parse(data)
    dss = root.find("DocumentSummarySet")
    num_removed = 0
    for ds in dss.findall("DocumentSummary"):
        if ds.find("error") is not None:
            dss.remove(ds)
            num_removed += 1
    if num_removed:
        print(f"Warning: Removed {num_removed} gene records with 'error: cannot get document summary'")
    bytes_io = BytesIO()
    root.write(bytes_io, xml_declaration=True)
    bytes_io.seek(0)
    document = Entrez.read(bytes_io)
```
Looks like the problem was already solved for your particular set of IDs.

Resolving patch:
diff --git a/Bio/Entrez/Parser.py b/Bio/Entrez/Parser.py
--- a/Bio/Entrez/Parser.py
+++ b/Bio/Entrez/Parser.py
@@ -159,6 +159,7 @@ def store(self, value):
         key = value.key
         if self.allowed_tags is not None and key not in self.allowed_tags:
             raise ValueError("Unexpected item '%s' in list" % key)
+        del value.key
         self.append(value)
 
 
@@ -193,6 +194,7 @@ def store(self, value):
         tag = value.tag
         if self.allowed_tags is not None and tag not in self.allowed_tags:
             raise ValueError("Unexpected item '%s' in dictionary" % key)
+        del value.key
         if self.repeated_tags and key in self.repeated_tags:
             self[key].append(value)
         else:
@@ -236,6 +238,24 @@ def store(self, value):
         self[-1].append(value)
 
 
+class ErrorElement(str):
+    """NCBI Entrez XML element containing an error message."""
+
+    def __new__(cls, value, *args, **kwargs):
+        """Create an ErrorElement."""
+        return str.__new__(cls, value)
+
+    def __init__(self, value, tag):
+        """Initialize an ErrorElement."""
+        self.tag = tag
+        self.key = tag
+
+    def __repr__(self):
+        """Return the error message as a string."""
+        text = str.__repr__(self)
+        return f"ErrorElement({text})"
+
+
 class NotXMLError(ValueError):
     """Failed to parse file as XML."""
 
@@ -337,7 +357,7 @@ class DataHandler(metaclass=DataHandlerMeta):
 
     del Entrez
 
-    def __init__(self, validate, escape):
+    def __init__(self, validate, escape, ignore_errors):
         """Create a DataHandler object."""
         self.dtd_urls = []
         self.element = None
@@ -350,6 +370,7 @@ def __init__(self, validate, escape):
         self.items = set()
         self.errors = set()
         self.validating = validate
+        self.ignore_errors = ignore_errors
         self.parser = expat.ParserCreate(namespace_separator=" ")
         self.parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
         self.parser.XmlDeclHandler = self.xmlDeclHandler
@@ -380,7 +401,7 @@ def read(self, handle):
                 # the input data is not in XML format.
                 raise NotXMLError(e) from None
         try:
-            return self.record
+            record = self.record
         except AttributeError:
             if self.parser.StartElementHandler:
                 # We saw the initial <!xml declaration, and expat didn't notice
@@ -395,6 +416,9 @@ def read(self, handle):
                 # We did not see the initial <!xml declaration, so probably
                 # the input data is not in XML format.
                 raise NotXMLError("XML declaration not found") from None
+        else:
+            del record.key
+            return record
 
     def parse(self, handle):
         """Parse the XML in the given file handle."""
@@ -733,7 +757,7 @@ def endStringElementHandler(self, tag):
         self.allowed_tags = None
 
     def endRawElementHandler(self, name):
-        """Handle start of an XML raw element."""
+        """Handle end of an XML raw element."""
         self.level -= 1
         if self.level == 0:
             self.parser.EndElementHandler = self.endStringElementHandler
@@ -746,22 +770,30 @@ def endRawElementHandler(self, name):
         self.data.append(tag)
 
     def endSkipElementHandler(self, name):
-        """Handle start of an XML skip element."""
+        """Handle end of an XML skip element."""
         self.level -= 1
         if self.level == 0:
             self.parser.StartElementHandler = self.startElementHandler
             self.parser.EndElementHandler = self.endElementHandler
 
-    def endErrorElementHandler(self, name):
-        """Handle start of an XML error element."""
-        if self.data:
-            # error found:
-            value = "".join(self.data)
-            raise RuntimeError(value)
-        # no error found:
-        if self.element is not None:
+    def endErrorElementHandler(self, tag):
+        """Handle end of an XML error element."""
+        element = self.element
+        if element is not None:
+            self.parser.StartElementHandler = self.startElementHandler
             self.parser.EndElementHandler = self.endElementHandler
             self.parser.CharacterDataHandler = self.skipCharacterDataHandler
+        data = "".join(self.data)
+        if data == "":
+            return
+        if self.ignore_errors is False:
+            raise RuntimeError(data)
+        self.data = []
+        value = ErrorElement(data, tag)
+        if element is None:
+            self.record = element
+        else:
+            element.store(value)
 
     def endElementHandler(self, name):
         """Handle end of an XML element."""
@@ -928,9 +960,9 @@ def elementDecl(self, name, model):
         # only once, and which can occur multiple times.
         single = []
         multiple = []
+        errors = []
         # The 'count' function is called recursively to make sure all the
-        # children in this model are counted. Error keys are ignored;
-        # they raise an exception in Python.
+        # children in this model are counted.
 
         def count(model):
             quantifier, key, children = model[1:]
@@ -944,7 +976,9 @@ def count(model):
                 else:
                     for child in children:
                         count(child)
-            elif key.upper() != "ERROR":
+            elif key.upper() == "ERROR":
+                errors.append(key)
+            else:
                 if quantifier in (
                     expat.model.XML_CQUANT_NONE,
                     expat.model.XML_CQUANT_OPT,
@@ -958,10 +992,10 @@ def count(model):
 
         count(model)
         if len(single) == 0 and len(multiple) == 1:
-            allowed_tags = frozenset(multiple)
+            allowed_tags = frozenset(multiple + errors)
             self.constructors[name] = (ListElement, (allowed_tags,))
         else:
-            allowed_tags = frozenset(single + multiple)
+            allowed_tags = frozenset(single + multiple + errors)
             repeated_tags = frozenset(multiple)
             args = (allowed_tags, repeated_tags)
             self.constructors[name] = (DictionaryElement, args)
diff --git a/Bio/Entrez/__init__.py b/Bio/Entrez/__init__.py
--- a/Bio/Entrez/__init__.py
+++ b/Bio/Entrez/__init__.py
@@ -455,7 +455,7 @@ def ecitmatch(**keywds):
     return _open(request)
 
 
-def read(handle, validate=True, escape=False):
+def read(handle, validate=True, escape=False, ignore_errors=False):
     """Parse an XML file from the NCBI Entrez Utilities into python objects.
 
     This function parses an XML file created by NCBI's Entrez Utilities,
@@ -495,12 +495,12 @@ def read(handle, validate=True, escape=False):
     """
     from .Parser import DataHandler
 
-    handler = DataHandler(validate, escape)
+    handler = DataHandler(validate, escape, ignore_errors)
     record = handler.read(handle)
     return record
 
 
-def parse(handle, validate=True, escape=False):
+def parse(handle, validate=True, escape=False, ignore_errors=False):
     """Parse an XML file from the NCBI Entrez Utilities into python objects.
 
     This function parses an XML file created by NCBI's Entrez Utilities,
@@ -549,7 +549,7 @@ def parse(handle, validate=True, escape=False):
     """
     from .Parser import DataHandler
 
-    handler = DataHandler(validate, escape)
+    handler = DataHandler(validate, escape, ignore_errors)
     records = handler.parse(handle)
     return records
 


Test patch:
diff --git a/Tests/test_Entrez.py b/Tests/test_Entrez.py
--- a/Tests/test_Entrez.py
+++ b/Tests/test_Entrez.py
@@ -429,7 +429,7 @@ def test_custom_directory(self):
         import os
         import shutil
 
-        handler = Parser.DataHandler(validate=False, escape=False)
+        handler = Parser.DataHandler(validate=False, escape=False, ignore_errors=False)
 
         # Create a temporary directory
         tmpdir = tempfile.mkdtemp()
diff --git a/Tests/test_Entrez_parser.py b/Tests/test_Entrez_parser.py
--- a/Tests/test_Entrez_parser.py
+++ b/Tests/test_Entrez_parser.py
@@ -77,14 +77,16 @@ def test_pickle(self):
                 "biosample.xml",  # DTD not specified in XML file
                 "einfo3.xml",  # DTD incomplete
                 "einfo4.xml",  # XML corrupted
-                "epost2.xml",  # XML returned by EPost with incorrect arguments
-                "esummary8.xml",  # XML returned by ESummary with incorrect arguments
                 "journals.xml",  # Missing XML declaration
             ):
                 continue
             path = os.path.join(directory, filename)
             with open(path, "rb") as stream:
-                record = Entrez.read(stream)
+                if filename in ("epost2.xml", "esummary8.xml", "esummary10.xml"):
+                    # these include an ErrorElement
+                    record = Entrez.read(stream, ignore_errors=True)
+                else:
+                    record = Entrez.read(stream)
             with BytesIO() as stream:
                 pickle.dump(record, stream)
                 stream.seek(0)
@@ -1770,6 +1772,12 @@ def test_wrong(self):
         # >>> Bio.Entrez.epost(db="nothing")
         with open("Entrez/epost2.xml", "rb") as handle:
             self.assertRaises(RuntimeError, Entrez.read, handle)
+        with open("Entrez/epost2.xml", "rb") as handle:
+            record = Entrez.read(handle, ignore_errors=True)
+        self.assertEqual(len(record), 1)
+        self.assertEqual(len(record.attributes), 0)
+        self.assertEqual(record["ERROR"], "Wrong DB name")
+        self.assertEqual(record["ERROR"].tag, "ERROR")
 
     def test_invalid(self):
         """Test parsing XML returned by EPost with invalid id (overflow tag)."""
@@ -2180,6 +2188,12 @@ def test_wrong(self):
         # >>> Bio.Entrez.esummary()
         with open("Entrez/esummary8.xml", "rb") as handle:
             self.assertRaises(RuntimeError, Entrez.read, handle)
+        with open("Entrez/esummary8.xml", "rb") as handle:
+            record = Entrez.read(handle, ignore_errors=True)
+        self.assertEqual(len(record), 1)
+        self.assertEqual(len(record.attributes), 0)
+        self.assertEqual(record[0], "Neither query_key nor id specified")
+        self.assertEqual(record[0].tag, "ERROR")
 
     def test_integer_none(self):
         """Test parsing ESummary XML where an Integer is not defined."""