Repository: biopython/biopython

Issue:
PDBList does not have the option to retrieve the 'biological assembly file'
PDBList does not have the option to retrieve the 'biological assembly file'.

The file_format="bundle" option usually returns 
"Desired structure doesn't exists"

Can anyone explain how to overcome this?
As an example, try downloading the 'biological assembly file' for PDB '2exs'. At the moment I can only achieve this by using the PDB website. 


Discussion:
Best as I can tell, PDB only has the 'bundle' files for structures that are too big to be represented in the PDBx/mmCIF file format, see [documentation here](http://www.wwpdb.org/documentation/file-formats-and-the-pdb). I'm able to find a bundle for [4UG0](https://www.rcsb.org/structure/4ug0) but not [2exs](https://www.rcsb.org/structure/2exs).

I think that the 'biological assembly file' is something different from the 'bundle'. From what I can tell, these are in the ftp site [here](http://ftp.wwpdb.org/pub/pdb/data/biounit/).

This feature doesn't seem too hard to add, I think there would just need to be small tweaks to the `retrieve_pdb_file`, `download_pdb_files`, and `download_entire_pdb` functions.

However, I don't frequently use PDB myself - so I'm not sure what the best name for this format should be (`biounit`?) or if it makes sense to add to this code. Anyone have any feedback?
@dadoskawina or @jgreener64 might have some advice here?
I second the idea to add an option to download biological assemblies. Is the functionality added already?
Agreed, this would be very nice indeed. 
Just a note that mmCIF assemblies will be available from March 2022 onwards, so this will simplify this issue quite a bit: https://pdbj.org/news/Assembly_files_mmCIF_format

Resolving patch:
diff --git a/Bio/PDB/PDBList.py b/Bio/PDB/PDBList.py
--- a/Bio/PDB/PDBList.py
+++ b/Bio/PDB/PDBList.py
@@ -37,10 +37,11 @@
 
 
 import contextlib
+import ftplib
 import gzip
 import os
-import shutil
 import re
+import shutil
 import sys
 
 from urllib.request import urlopen
@@ -234,7 +235,7 @@ def retrieve_pdb_file(
             * "pdb" (format PDB),
             * "xml" (PDBML/XML format),
             * "mmtf" (highly compressed),
-            * "bundle" (PDB formatted archive for large structure}
+            * "bundle" (PDB formatted archive for large structure)
 
         :type file_format: string
 
@@ -330,14 +331,14 @@ def retrieve_pdb_file(
                     print(f"Structure exists: '{final_file}' ")
                 return final_file
 
-        # Retrieve the file
+        # Retrieve the file(s)
         if self._verbose:
             print(f"Downloading PDB structure '{pdb_code}'...")
         try:
             urlcleanup()
             urlretrieve(url, filename)
         except OSError:
-            print("Desired structure doesn't exists")
+            print("Desired structure doesn't exist")
         else:
             with gzip.open(filename, "rb") as gz:
                 with open(final_file, "wb") as out:
@@ -345,7 +346,7 @@ def retrieve_pdb_file(
             os.remove(filename)
         return final_file
 
-    def update_pdb(self, file_format=None):
+    def update_pdb(self, file_format=None, with_assemblies=False):
         """Update your local copy of the PDB files.
 
         I guess this is the 'most wanted' function from this module.
@@ -364,12 +365,27 @@ def update_pdb(self, file_format=None):
         for pdb_code in new + modified:
             try:
                 self.retrieve_pdb_file(pdb_code, file_format=file_format)
-            except Exception:
-                print(f"error {pdb_code}\n")
+                if with_assemblies:
+                    assemblies = self.get_all_assemblies(file_format)
+                    for a_pdb_code, assembly_num in assemblies:
+                        if a_pdb_code == pdb_code:
+                            pl.retrieve_assembly_file(
+                                pdb_code,
+                                assembly_num,
+                                file_format=file_format,
+                                overwrite=True,
+                            )
+
+            except Exception as err:
+                print(f"error {pdb_code}: {err}\n")
                 # you can insert here some more log notes that
                 # something has gone wrong.
 
         # Move the obsolete files to a special folder
+        # NOTE: This should be updated to handle multiple file types and
+        # assemblies. As of now, it only looks for PDB-formatted files.
+        # Using pathlib will be probably the best approach here, to build
+        # and index of which files we have efficiently (or glob them).
         for pdb_code in obsolete:
             if self.flat_tree:
                 old_file = os.path.join(self.local_pdb, f"pdb{pdb_code}.ent")
@@ -412,7 +428,7 @@ def download_pdb_files(
             * "pdb" (format PDB),
             * "xml" (PMDML/XML format),
             * "mmtf" (highly compressed),
-            * "bundle" (PDB formatted archive for large structure}
+            * "bundle" (PDB formatted archive for large structure)
 
         :param overwrite: if set to True, existing structure files will be overwritten. Default: False
         :type overwrite: bool
@@ -443,6 +459,159 @@ def download_pdb_files(
                 overwrite=overwrite,
             )
 
+    def get_all_assemblies(self, file_format="mmCif"):
+        """Retrieve the list of PDB entries with an associated bio assembly.
+
+        The requested list will be cached to avoid multiple calls to the FTP
+        server.
+
+        :type  file_format: str, optional
+        :param file_format: format in which to download the entries. Available
+            options are "mmCif" or "pdb". Defaults to mmCif.
+        """
+        if hasattr(self, "assemblies") and self.assemblies:
+            if self._verbose:
+                print("Retrieving cached list of assemblies.")
+            return self.assemblies  # cache
+
+        if self._verbose:
+            print("Retrieving list of assemblies. This might take a while.")
+
+        # FTPLib is much faster than urlopen
+        idx = self.pdb_server.find("://")
+        if idx >= 0:
+            ftp = ftplib.FTP(self.pdb_server[idx + 3 :])
+        else:
+            ftp = ftplib.FTP(self.pdb_server)
+        ftp.login()  # anonymous
+
+        if file_format.lower() == "mmcif":
+            ftp.cwd("/pub/pdb/data/assemblies/mmCIF/all/")
+            re_name = re.compile(r"(\d[0-9a-z]{3})-assembly(\d+).cif.gz")
+        elif file_format.lower() == "pdb":
+            ftp.cwd("/pub/pdb/data/biounit/PDB/all/")
+            re_name = re.compile(r"(\d[0-9a-z]{3}).pdb(\d+).gz")
+        else:
+            msg = "file_format for assemblies must be 'pdb' or 'mmCif'"
+            raise ValueError(msg)
+
+        response = []
+        ftp.retrlines("NLST", callback=response.append)
+        # PDB entries can have more than one assembly:
+        #  e.g.
+        #    104l-assembly1.cif.gz           2022-01-15 19:38   36K
+        #    104l-assembly2.cif.gz           2022-01-15 19:38   36K
+        #    ...
+        #    104l.pdb1.gz               2010-01-15 08:00   28K
+        #    104l.pdb2.gz               2010-01-15 08:00   28K
+        all_assemblies = []
+        for line in response:
+            if line.endswith(".gz"):
+                match = re_name.findall(line)
+                try:
+                    if len(match):
+                        entry, assembly = match[0]
+                except ValueError:
+                    pass
+                else:
+                    all_assemblies.append((entry, assembly))
+        self.assemblies = all_assemblies  # cache
+        return all_assemblies
+
+    def retrieve_assembly_file(
+        self, pdb_code, assembly_num, pdir=None, file_format=None, overwrite=False
+    ):
+        """Fetch one or more assembly structures associated with a PDB entry.
+
+        Unless noted below, parameters are described in ``retrieve_pdb_file``.
+
+        :type  assembly_num: int
+        :param assembly_num: assembly number to download.
+
+        :rtype : str
+        :return: file name of the downloaded assembly file.
+        """
+        archive = {
+            "pdb": "%s.pdb%s.gz",
+            "mmcif": "%s-assembly%s.cif.gz",
+        }
+
+        file_format = self._print_default_format_warning(file_format)
+        file_format = file_format.lower()  # we should standardize this.
+        if file_format not in archive:
+            raise (
+                "Specified file_format '%s' is not supported. Use one of the "
+                "following: 'mmcif' or 'pdb'." % file_format
+            )
+
+        # Get the compressed assembly structure name
+        archive_fn = archive[file_format] % (pdb_code.lower(), int(assembly_num))
+
+        if file_format == "mmcif":
+            url = self.pdb_server + "/pub/pdb/data/assemblies/mmCIF/all/%s" % archive_fn
+        elif file_format == "pdb":
+            url = self.pdb_server + "/pub/pdb/data/biounit/PDB/all/%s" % archive_fn
+        else:  # better safe than sorry
+            raise ValueError("file_format '%s' not supported: %s" % file_format)
+
+        # Where will the file be saved?
+        if pdir is None:
+            path = self.local_pdb
+            if not self.flat_tree:  # Put in PDB-style directory tree
+                path = os.path.join(path, pdb_code[1:3])
+        else:  # Put in specified directory
+            path = pdir
+        if not os.access(path, os.F_OK):
+            os.makedirs(path)
+
+        assembly_gz_file = os.path.join(path, archive_fn)
+        assembly_final_file = os.path.join(path, archive_fn[:-3])  # no .gz
+
+        # Skip download if the file already exists
+        if not overwrite:
+            if os.path.exists(assembly_final_file):
+                if self._verbose:
+                    print(f"Structure exists: '{assembly_final_file}' ")
+                return assembly_final_file
+
+        # Otherwise,retrieve the file(s)
+        if self._verbose:
+            print(
+                f"Downloading assembly ({assembly_num}) for PDB entry "
+                f"'{pdb_code}'..."
+            )
+        try:
+            urlcleanup()
+            urlretrieve(url, assembly_gz_file)
+        except OSError as err:
+            print(f"Download failed! Maybe the desired assembly does not exist: {err}")
+        else:
+            with gzip.open(assembly_gz_file, "rb") as gz:
+                with open(assembly_final_file, "wb") as out:
+                    out.writelines(gz)
+            os.remove(assembly_gz_file)
+        return assembly_final_file
+
+    def download_all_assemblies(self, listfile=None, file_format=None):
+        """Retrieve all biological assemblies not in the local PDB copy.
+
+        :type  listfile: str, optional
+        :param listfile: file name to which all assembly codes will be written
+
+        :type  file_format: str, optional
+        :param file_format: format in which to download the entries. Available
+            options are "mmCif" or "pdb". Defaults to mmCif.
+        """
+        # Deprecation warning
+        file_format = self._print_default_format_warning(file_format)
+        assemblies = self.get_all_assemblies(file_format)
+        for pdb_code, assembly_num in assemblies:
+            self.retrieve_assembly_file(pdb_code, assembly_num, file_format=file_format)
+        # Write the list
+        if listfile:
+            with open(listfile, "w") as outfile:
+                outfile.writelines(f"{pdb_code}.{assembly_num}\n" for x in assemblies)
+
     def download_entire_pdb(self, listfile=None, file_format=None):
         """Retrieve all PDB entries not present in the local PDB copy.
 
@@ -455,7 +624,7 @@ def download_entire_pdb(self, listfile=None, file_format=None):
             * "pdb" (format PDB),
             * "xml" (PMDML/XML format),
             * "mmtf" (highly compressed),
-            * "bundle" (PDB formatted archive for large structure}
+            * "bundle" (PDB formatted archive for large structure)
 
         NOTE. The default download format has changed from PDB to PDBx/mmCif
         """
@@ -514,6 +683,8 @@ def get_seqres_file(self, savefile="pdb_seqres.txt"):
                                                    local pdb tree.
         PDBList.py obsol  <pdb_path> [options]   - write all obsolete PDB
                                                    entries to local pdb tree.
+        PDBList.py assemb <pdb_path> [options]   - write all assemblies for each
+                                                   PDB entry to local pdb tree.
         PDBList.py <PDB-ID> <pdb_path> [options] - retrieve single structure
         PDBList.py (<PDB-ID1>,<PDB-ID2>,...) <pdb_path> [options] - retrieve a set
                                                    of structures
@@ -524,6 +695,7 @@ def get_seqres_file(self, savefile="pdb_seqres.txt"):
      -pdb     Downloads structures in PDB format
      -xml     Downloads structures in PDBML (XML) format
      -mmtf    Downloads structures in mmtf format
+     -with-assemblies    Downloads assemblies along with regular entries.
 
     Maximum one format can be specified simultaneously (if more selected, only
     the last will be considered). By default (no format specified) structures are
@@ -533,6 +705,7 @@ def get_seqres_file(self, savefile="pdb_seqres.txt"):
 
     file_format = "mmCif"
     overwrite = False
+    with_assemblies = False
 
     if len(sys.argv) > 2:
         pdb_path = sys.argv[2]
@@ -545,6 +718,10 @@ def get_seqres_file(self, savefile="pdb_seqres.txt"):
                     overwrite = True
                 elif option in ("-pdb", "-xml", "-mmtf"):
                     file_format = option[1:]
+                # Allow for download of assemblies alongside ASU
+                elif option == "-with-assemblies":
+                    with_assemblies = True
+
     else:
         pdb_path = os.getcwd()
         pl = PDBList()
@@ -554,21 +731,41 @@ def get_seqres_file(self, savefile="pdb_seqres.txt"):
         if sys.argv[1] == "update":
             # update PDB
             print("updating local PDB at " + pdb_path)
-            pl.update_pdb(file_format=file_format)
+            pl.update_pdb(file_format=file_format, with_assemblies=with_assemblies)
 
         elif sys.argv[1] == "all":
             # get the entire PDB
             pl.download_entire_pdb(file_format=file_format)
+            if with_assemblies:
+                # get all assembly structures
+                pl.download_all_assemblies(file_format=file_format)
 
         elif sys.argv[1] == "obsol":
             # get all obsolete entries
             pl.download_obsolete_entries(pdb_path, file_format=file_format)
 
+        elif sys.argv[1] == "assemb":
+            # get all assembly structures
+            pl.download_all_assemblies(file_format=file_format)
+
         elif len(sys.argv[1]) == 4 and sys.argv[1][0].isdigit():
+            pdb_code = sys.argv[1]
             # get single PDB entry
             pl.retrieve_pdb_file(
-                sys.argv[1], pdir=pdb_path, file_format=file_format, overwrite=overwrite
+                pdb_code, pdir=pdb_path, file_format=file_format, overwrite=overwrite
             )
+            if with_assemblies:
+                # PDB Code might have more than one assembly.
+                assemblies = pl.get_all_assemblies(file_format)
+                for a_pdb_code, assembly_num in assemblies:
+                    if a_pdb_code == pdb_code:
+                        pl.retrieve_assembly_file(
+                            pdb_code,
+                            assembly_num,
+                            pdir=pdb_path,
+                            file_format=file_format,
+                            overwrite=overwrite,
+                        )
 
         elif sys.argv[1][0] == "(":
             # get a set of PDB entries
@@ -577,3 +774,15 @@ def get_seqres_file(self, savefile="pdb_seqres.txt"):
                 pl.retrieve_pdb_file(
                     pdb_id, pdir=pdb_path, file_format=file_format, overwrite=overwrite
                 )
+                if with_assemblies:
+                    # PDB Code might have more than one assembly.
+                    assemblies = pl.get_all_assemblies(file_format)
+                    for a_pdb_code, assembly_num in assemblies:
+                        if a_pdb_code == pdb_id:
+                            pl.retrieve_assembly_file(
+                                pdb_id,
+                                assembly_num,
+                                pdir=pdb_path,
+                                file_format=file_format,
+                                overwrite=overwrite,
+                            )


Test patch:
diff --git a/Tests/test_PDBList.py b/Tests/test_PDBList.py
--- a/Tests/test_PDBList.py
+++ b/Tests/test_PDBList.py
@@ -49,6 +49,15 @@ def test_get_all_obsolete(self):
         # was exceeded
         self.assertGreater(len(entries), 3000)
 
+    def test_get_all_assemblies(self):
+        """Tests the Bio.PDB.PDBList.get_all_assemblies method."""
+        # obsolete_pdb declared to prevent from creating the "obsolete" directory
+        pdblist = PDBList(obsolete_pdb="unimportant")
+        entries = pdblist.get_all_assemblies()
+        # As number of obsolete entries constantly grow, test checks if a certain number
+        # was exceeded
+        self.assertGreater(len(entries), 100000)
+
 
 class TestPDBListGetStructure(unittest.TestCase):
     """Test methods responsible for getting structures."""
@@ -63,7 +72,7 @@ def make_temp_directory(self, directory):
 
     def check(self, structure, filename, file_format, obsolete=False, pdir=None):
         with self.make_temp_directory(os.getcwd()) as tmp:
-            pdblist = PDBList(pdb=tmp, obsolete_pdb=os.path.join(tmp, "obsolete"))
+            pdblist = PDBList(pdb=tmp)
             path = os.path.join(tmp, filename)
             if pdir:
                 pdir = os.path.join(tmp, pdir)
@@ -71,7 +80,6 @@ def check(self, structure, filename, file_format, obsolete=False, pdir=None):
                 structure, obsolete=obsolete, pdir=pdir, file_format=file_format
             )
             self.assertTrue(os.path.isfile(path))
-            os.remove(path)
 
     def test_retrieve_pdb_file_small_pdb(self):
         """Tests retrieving the small molecule in pdb format."""
@@ -134,13 +142,77 @@ def test_retrieve_pdb_file_mmtf(self):
         structure = "127d"
         self.check(structure, os.path.join(structure[1:3], f"{structure}.mmtf"), "mmtf")
 
-    def test_double_retrieve(self):
+    def test_double_retrieve_structure(self):
         """Tests retrieving the same file to different directories."""
         structure = "127d"
         self.check(structure, os.path.join("a", f"{structure}.cif"), "mmCif", pdir="a")
         self.check(structure, os.path.join("b", f"{structure}.cif"), "mmCif", pdir="b")
 
 
+class TestPDBListGetAssembly(unittest.TestCase):
+    """Test methods responsible for getting assemblies."""
+
+    @contextlib.contextmanager
+    def make_temp_directory(self, directory):
+        temp_dir = tempfile.mkdtemp(dir=directory)
+        try:
+            yield temp_dir
+        finally:
+            shutil.rmtree(temp_dir)
+
+    def check(self, structure, assembly_num, filename, file_format, pdir=None):
+        with self.make_temp_directory(os.getcwd()) as tmp:
+            pdblist = PDBList(pdb=tmp)
+            path = os.path.join(tmp, filename)
+            if pdir:
+                pdir = os.path.join(tmp, pdir)
+            pdblist.retrieve_assembly_file(
+                structure, assembly_num, pdir=pdir, file_format=file_format
+            )
+            self.assertTrue(os.path.isfile(path))
+
+    def test_retrieve_assembly_file_mmcif(self):
+        """Tests retrieving a small assembly in mmCif format."""
+        structure = "127d"
+        assembly_num = "1"
+        self.check(
+            structure,
+            assembly_num,
+            os.path.join(structure[1:3], f"{structure}-assembly{assembly_num}.cif"),
+            "mmCif",
+        )
+
+    def test_retrieve_assembly_file_pdb(self):
+        """Tests retrieving a small assembly in pdb format."""
+        structure = "127d"
+        assembly_num = "1"
+        self.check(
+            structure,
+            assembly_num,
+            os.path.join(structure[1:3], f"{structure}.pdb{assembly_num}"),
+            "pdb",
+        )
+
+    def test_double_retrieve_assembly(self):
+        """Tests retrieving the same file to different directories."""
+        structure = "127d"
+        assembly_num = "1"
+        self.check(
+            structure,
+            assembly_num,
+            os.path.join("a", f"{structure}-assembly{assembly_num}.cif"),
+            "mmCif",
+            pdir="a",
+        )
+        self.check(
+            structure,
+            assembly_num,
+            os.path.join("b", f"{structure}-assembly{assembly_num}.cif"),
+            "mmCif",
+            pdir="b",
+        )
+
+
 if __name__ == "__main__":
     runner = unittest.TextTestRunner(verbosity=2)
     unittest.main(testRunner=runner)