Repository: biopython/biopython

Issue:
Per-column annotations for multiple-sequence alignments
This is needed for #357 where we currently discard per-column annotation when parsing Stockholm files.

Like the existing `SeqRecord` per-letter annotations dictionary this needs to be sliced in `__getitem__` to pull out the right columns, and similarly be supported when adding alignments.

This replaces https://redmine.open-bio.org/issues/3388 on the old issue tracker (currently offline).



Discussion:

Resolving patch:
diff --git a/Bio/Align/__init__.py b/Bio/Align/__init__.py
--- a/Bio/Align/__init__.py
+++ b/Bio/Align/__init__.py
@@ -12,7 +12,7 @@
 from __future__ import print_function
 
 from Bio.Seq import Seq
-from Bio.SeqRecord import SeqRecord
+from Bio.SeqRecord import SeqRecord, _RestrictedDict
 from Bio import Alphabet
 
 
@@ -103,7 +103,7 @@ class MultipleSeqAlignment(object):
     """
 
     def __init__(self, records, alphabet=None,
-                 annotations=None):
+                 annotations=None, column_annotations=None):
         """Initialize a new MultipleSeqAlignment object.
 
         Arguments:
@@ -115,6 +115,10 @@ def __init__(self, records, alphabet=None,
                       record alphabets.  If omitted, a consensus alphabet is
                       used.
          - annotations - Information about the whole alignment (dictionary).
+         - column_annotations - Per column annotation (restricted dictionary).
+                      This holds Python sequences (lists, strings, tuples)
+                      whose length matches the number of columns. A typical
+                      use would be a secondary structure consensus string.
 
         You would normally load a MSA from a file using Bio.AlignIO, but you
         can do this from a list of SeqRecord objects too:
@@ -126,7 +130,9 @@ def __init__(self, records, alphabet=None,
         >>> a = SeqRecord(Seq("AAAACGT", generic_dna), id="Alpha")
         >>> b = SeqRecord(Seq("AAA-CGT", generic_dna), id="Beta")
         >>> c = SeqRecord(Seq("AAAAGGT", generic_dna), id="Gamma")
-        >>> align = MultipleSeqAlignment([a, b, c], annotations={"tool": "demo"})
+        >>> align = MultipleSeqAlignment([a, b, c],
+        ...                              annotations={"tool": "demo"},
+        ...                              column_annotations={"stats": "CCCXCCC"})
         >>> print(align)
         DNAAlphabet() alignment with 3 rows and 7 columns
         AAAACGT Alpha
@@ -134,6 +140,8 @@ def __init__(self, records, alphabet=None,
         AAAAGGT Gamma
         >>> align.annotations
         {'tool': 'demo'}
+        >>> align.column_annotations
+        {'stats': 'CCCXCCC'}
 
         NOTE - The older Bio.Align.Generic.Alignment class only accepted a
         single argument, an alphabet.  This is still supported via a backwards
@@ -180,6 +188,45 @@ def __init__(self, records, alphabet=None,
             raise TypeError("annotations argument should be a dict")
         self.annotations = annotations
 
+        # Annotations about each colum of the alignment
+        if column_annotations is None:
+            column_annotations = {}
+        # Handle this via the property set function which will validate it
+        self.column_annotations = column_annotations
+
+    def _set_per_column_annotations(self, value):
+        if not isinstance(value, dict):
+            raise TypeError("The per-column-annotations should be a "
+                            "(restricted) dictionary.")
+        # Turn this into a restricted-dictionary (and check the entries)
+        if len(self):
+            # Use the standard method to get the length
+            expected_length = self.get_alignment_length()
+            self._per_col_annotations = _RestrictedDict(length=expected_length)
+            self._per_col_annotations.update(value)
+        else:
+            # Bit of a problem case... number of columns is undefined
+            self._per_col_annotations = None
+            if value:
+                raise ValueError("Can't set per-column-annotations without an alignment")
+
+    def _get_per_column_annotations(self):
+        if self._per_col_annotations is None:
+            # This happens if empty at initialisation
+            if len(self):
+                # Use the standard method to get the length
+                expected_length = self.get_alignment_length()
+            else:
+                # Should this raise an exception? Compare SeqRecord behaviour...
+                expected_length = 0
+            self._per_col_annotations = _RestrictedDict(length=expected_length)
+        return self._per_col_annotations
+
+    column_annotations = property(
+        fget=_get_per_column_annotations,
+        fset=_set_per_column_annotations,
+        doc="""Dictionary of per-letter-annotation for the sequence.""")
+
     def _str_line(self, record, length=50):
         """Return a truncated string representation of a SeqRecord (PRIVATE).
 
@@ -481,6 +528,9 @@ def extend(self, records):
                 return
             expected_length = len(rec)
             self._append(rec, expected_length)
+            # Can now setup the per-column-annotations as well, set to None
+            # while missing the length:
+            self.column_annotations = {}
             # Now continue to the rest of the records as usual
 
         for rec in records:
@@ -573,9 +623,11 @@ def __add__(self, other):
         >>> b2 = SeqRecord(Seq("GT", generic_dna), id="Beta")
         >>> c2 = SeqRecord(Seq("GT", generic_dna), id="Gamma")
         >>> left = MultipleSeqAlignment([a1, b1, c1],
-        ...                             annotations={"tool": "demo", "name": "start"})
+        ...                             annotations={"tool": "demo", "name": "start"},
+        ...                             column_annotations={"stats": "CCCXC"})
         >>> right = MultipleSeqAlignment([a2, b2, c2],
-        ...                             annotations={"tool": "demo", "name": "end"})
+        ...                             annotations={"tool": "demo", "name": "end"},
+        ...                             column_annotations={"stats": "CC"})
 
         Now, let's look at these two alignments:
 
@@ -621,6 +673,11 @@ def __add__(self, other):
         >>> combined.annotations
         {'tool': 'demo'}
 
+        Similarly any common per-column-annotations are combined:
+
+        >>> combined.column_annotations
+        {'stats': 'CCCXCCC'}
+
         """
         if not isinstance(other, MultipleSeqAlignment):
             raise NotImplementedError
@@ -634,7 +691,11 @@ def __add__(self, other):
         for k, v in self.annotations.items():
             if k in other.annotations and other.annotations[k] == v:
                 annotations[k] = v
-        return MultipleSeqAlignment(merged, alpha, annotations)
+        column_annotations = dict()
+        for k, v in self.column_annotations.items():
+            if k in other.column_annotations:
+                column_annotations[k] = v + other.column_annotations[k]
+        return MultipleSeqAlignment(merged, alpha, annotations, column_annotations)
 
     def __getitem__(self, index):
         """Access part of the alignment.
@@ -754,7 +815,13 @@ def __getitem__(self, index):
             return self._records[index]
         elif isinstance(index, slice):
             # e.g. sub_align = align[i:j:k]
-            return MultipleSeqAlignment(self._records[index], self._alphabet)
+            new = MultipleSeqAlignment(self._records[index], self._alphabet)
+            if self.column_annotations and len(new) == len(self):
+                # All rows kept (although could have been reversed)
+                # Perserve the column annotations too,
+                for k, v in self.column_annotations.items():
+                    new.column_annotations[k] = v
+            return new
         elif len(index) != 2:
             raise TypeError("Invalid index type.")
 
@@ -768,8 +835,14 @@ def __getitem__(self, index):
             return "".join(rec[col_index] for rec in self._records[row_index])
         else:
             # e.g. sub_align = align[1:4, 5:7], gives another alignment
-            return MultipleSeqAlignment((rec[col_index] for rec in self._records[row_index]),
-                                        self._alphabet)
+            new = MultipleSeqAlignment((rec[col_index] for rec in self._records[row_index]),
+                                       self._alphabet)
+            if self.column_annotations and len(new) == len(self):
+                # All rows kept (although could have been reversed)
+                # Perserve the column annotations too,
+                for k, v in self.column_annotations.items():
+                    new.column_annotations[k] = v[col_index]
+            return new
 
     def sort(self, key=None, reverse=False):
         """Sort the rows (SeqRecord objects) of the alignment in place.
diff --git a/Bio/AlignIO/ClustalIO.py b/Bio/AlignIO/ClustalIO.py
--- a/Bio/AlignIO/ClustalIO.py
+++ b/Bio/AlignIO/ClustalIO.py
@@ -49,6 +49,14 @@ def write_alignment(self, alignment):
         if max_length <= 0:
             raise ValueError("Non-empty sequences are required")
 
+        if "clustal_consensus" in alignment.column_annotations:
+            star_info = alignment.column_annotations["clustal_consensus"]
+        elif hasattr(alignment, "_star_info"):
+            # This was originally stored by Bio.Clustalw as ._star_info
+            star_info = alignment._star_info
+        else:
+            star_info = None
+
         # keep displaying sequences until we reach the end
         while cur_char != max_length:
             # calculate the number of sequences to show, which will
@@ -70,10 +78,8 @@ def write_alignment(self, alignment):
                 output += line + "\n"
 
             # now we need to print out the star info, if we've got it
-            # This was stored by Bio.Clustalw using a ._star_info property.
-            if hasattr(alignment, "_star_info") and alignment._star_info != '':
-                output += (" " * 36) + \
-                    alignment._star_info[cur_char:(cur_char + show_num)] + "\n"
+            if star_info:
+                output += (" " * 36) + star_info[cur_char:(cur_char + show_num)] + "\n"
 
             output += "\n"
             cur_char += show_num
@@ -284,5 +290,7 @@ def __next__(self):
             assert len(consensus) == alignment_length, \
                 "Alignment length is %i, consensus length is %i, '%s'" \
                 % (alignment_length, len(consensus), consensus)
+            alignment.column_annotations["clustal_consensus"] = consensus
+            # For backward compatibility prior to .column_annotations:
             alignment._star_info = consensus
         return alignment
diff --git a/Bio/AlignIO/StockholmIO.py b/Bio/AlignIO/StockholmIO.py
--- a/Bio/AlignIO/StockholmIO.py
+++ b/Bio/AlignIO/StockholmIO.py
@@ -70,7 +70,16 @@
     -----------------<<<<<<<<-----<<.<<-------->>.>>----------.<<<<<--------->>>>>.-->>>>>>>>---------------
 
 Any general annotation for each row is recorded in the SeqRecord's annotations
-dictionary.  You can output this alignment in many different file formats
+dictionary.  Any per-column annotation for the entire alignment in in the
+alignment's column annotations dictionary, such as the secondary structure
+consensus in this example:
+
+    >>> sorted(align.column_annotations.keys())
+    ['secondary_structure']
+    >>> align.column_annotations["secondary_structure"]
+    '.................<<<<<<<<...<<<<<<<........>>>>>>>........<<<<<<<.......>>>>>>>..>>>>>>>>...............'
+
+You can output this alignment in many different file formats
 using Bio.AlignIO.write(), or the MultipleSeqAlignment object's format method:
 
     >>> print(align.format("fasta"))
@@ -96,6 +105,7 @@
     #=GS AE007476.1 AC AE007476.1
     #=GS AE007476.1 DE AE007476.1
     #=GR AE007476.1 SS -----------------<<<<<<<<-----<<.<<-------->>.>>----------.<<<<<--------->>>>>.-->>>>>>>>---------------
+    #=GC SS_cons .................<<<<<<<<...<<<<<<<........>>>>>>>........<<<<<<<.......>>>>>>>..>>>>>>>>...............
     //
     <BLANKLINE>
 
@@ -131,6 +141,32 @@
     >>> print(sub_record.letter_annotations['secondary_structure'])
     -------<<<
 
+Likewise with the alignment object, as long as you are not dropping any rows,
+slicing specific columns of an alignment will slice any per-column-annotations:
+
+    >>> align.column_annotations["secondary_structure"]
+    '.................<<<<<<<<...<<<<<<<........>>>>>>>........<<<<<<<.......>>>>>>>..>>>>>>>>...............'
+    >>> part_align = align[:,10:20]
+    >>> part_align.column_annotations["secondary_structure"]
+    '.......<<<'
+
+You can also see this in the Stockholm output of this partial-alignment:
+
+    >>> print(part_align.format("stockholm"))
+    # STOCKHOLM 1.0
+    #=GF SQ 2
+    AP001509.1 UCAACACUCU
+    #=GS AP001509.1 AC AP001509.1
+    #=GS AP001509.1 DE AP001509.1
+    #=GR AP001509.1 SS -------<<<
+    AE007476.1 AUCGUUUUAC
+    #=GS AE007476.1 AC AE007476.1
+    #=GS AE007476.1 DE AE007476.1
+    #=GR AE007476.1 SS -------<<<
+    #=GC SS_cons .......<<<
+    //
+    <BLANKLINE>
+
 """
 from __future__ import print_function
 
@@ -154,6 +190,9 @@ class StockholmWriter(SequentialAlignmentWriter):
                        "ligand_binding": "LI",
                        "active_site": "AS",
                        "intron": "IN"}
+    # These GC mappings are in addition to *_cons in GR mapping:
+    pfam_gc_mapping = {"reference_annotation": "RF",
+                       "model_mask": "MM"}
     # Following dictionary deliberately does not cover AC, DE or DR
     pfam_gs_mapping = {"organism": "OS",
                        "organism_classification": "OC",
@@ -171,9 +210,6 @@ def write_alignment(self, alignment):
         self._length_of_sequences = alignment.get_alignment_length()
         self._ids_written = []
 
-        # NOTE - For now, the alignment object does not hold any per column
-        # or per alignment annotation - only per sequence.
-
         if count == 0:
             raise ValueError("Must have at least one sequence")
         if self._length_of_sequences == 0:
@@ -183,6 +219,17 @@ def write_alignment(self, alignment):
         self.handle.write("#=GF SQ %i\n" % count)
         for record in alignment:
             self._write_record(record)
+        # This shouldn't be None... but just in case,
+        if alignment.column_annotations:
+            for k, v in sorted(alignment.column_annotations.items()):
+                if k in self.pfam_gc_mapping:
+                    self.handle.write("#=GC %s %s\n" % (self.pfam_gc_mapping[k], v))
+                elif k in self.pfam_gr_mapping:
+                    self.handle.write("#=GC %s %s\n" % (self.pfam_gr_mapping[k] + "_cons", v))
+                else:
+                    # It doesn't follow the PFAM standards, but should we record
+                    # this data anyway?
+                    pass
         self.handle.write("//\n")
 
     def _write_record(self, record):
@@ -313,6 +360,9 @@ class StockholmIterator(AlignmentIterator):
                        "LI": "ligand_binding",
                        "AS": "active_site",
                        "IN": "intron"}
+    # These GC mappings are in addition to *_cons in GR mapping:
+    pfam_gc_mapping = {"RF": "reference_annotation",
+                       "MM": "model_mask"}
     # Following dictionary deliberately does not cover AC, DE or DR
     pfam_gs_mapping = {"OS": "organism",
                        "OC": "organism_classification",
@@ -348,6 +398,7 @@ def __next__(self):
         gs = {}
         gr = {}
         gf = {}
+        gc = {}
         passed_end_alignment = False
         while True:
             line = handle.readline()
@@ -394,7 +445,11 @@ def __next__(self):
                 elif line[:5] == '#=GC ':
                     # Generic per-Column annotation, exactly 1 char per column
                     # Format: "#=GC <feature> <exactly 1 char per column>"
-                    pass
+                    feature, text = line[5:].strip().split(None, 2)
+                    if feature not in gc:
+                        gc[feature] = ""
+                    gc[feature] += text.strip()  # append to any previous entry
+                    # Might be interleaved blocks, so can't check length yet
                 elif line[:5] == '#=GS ':
                     # Generic per-Sequence annotation, free text
                     # Format: "#=GS <seqname> <feature> <free text>"
@@ -418,9 +473,7 @@ def __next__(self):
                     if feature not in gr[seq_id]:
                         gr[seq_id][feature] = ""
                     gr[seq_id][feature] += text.strip()  # append to any previous entry
-                    # TODO - Should we check the length matches the alignment length?
-                    #       For iterlaced sequences the GR data can be split over
-                    #       multiple lines
+                    # Might be interleaved blocks, so can't check length yet
             # Next line...
 
         assert len(seqs) <= len(ids)
@@ -460,8 +513,21 @@ def __next__(self):
 
                 self._populate_meta_data(seq_id, record)
                 records.append(record)
+            for k, v in gc.items():
+                if len(v) != alignment_length:
+                    raise ValueError("%s length %i, expected %i"
+                                     % (k, len(v), alignment_length))
             alignment = MultipleSeqAlignment(records, self.alphabet)
 
+            for k, v in sorted(gc.items()):
+                if k in self.pfam_gc_mapping:
+                    alignment.column_annotations[self.pfam_gc_mapping[k]] = v
+                elif k.endswith("_cons") and k[:-5] in self.pfam_gr_mapping:
+                    alignment.column_annotations[self.pfam_gr_mapping[k[:-5]]] = v
+                else:
+                    # Ignore it?
+                    alignment.column_annotations["GC:" + k] = v
+
             # TODO - Introduce an annotated alignment class?
             # For now, store the annotation a new private property:
             alignment._annotations = gr


Test patch:
diff --git a/Tests/output/test_AlignIO b/Tests/output/test_AlignIO
--- a/Tests/output/test_AlignIO
+++ b/Tests/output/test_AlignIO
@@ -3,6 +3,7 @@ Testing reading clustal format file Clustalw/cw02.aln with 1 alignments
  Alignment 0, with 2 sequences of length 601
   MENSDSNDKGSDQSAAQRRSQMDRLDREEAFYQFVN...SVV gi|4959044|gb|AAD34209.1|AF069
   ---------MSPQTETKASVGFKAGVKEYKLTYYTP...--- gi|671626|emb|CAA85685.1|
+            * *: ::    :.   :*  :  :. ...    clustal_consensus
  Checking can write/read as 'clustal' format
  Checking can write/read as 'maf' format
  Checking can write/read as 'mauve' format
@@ -58,6 +59,7 @@ Testing reading clustal format file Clustalw/odd_consensus.aln with 1 alignments
  Alignment 0, with 2 sequences of length 687
   ------------------------------------...TAG AT3G20900.1-CDS
   ATGAACAAAGTAGCGAGGAAGAACAAAACATCAGGT...TAG AT3G20900.1-SEQ
+                                      ...*** clustal_consensus
  Checking can write/read as 'clustal' format
  Checking can write/read as 'maf' format
  Checking can write/read as 'mauve' format
@@ -169,6 +171,7 @@ Testing reading stockholm format file Stockholm/simple.sth with 1 alignments
  Alignment 0, with 2 sequences of length 104
   UUAAUCGAGCUCAACACUCUUCGUAUAUCCUC-UCA...UGU AP001509.1
   AAAAUUGAAUAUCGUUUUACUUGUUUAU-GUCGUGA...GAU AE007476.1
+  .................<<<<<<<<...<<<<<<<....... secondary_structure
  Checking can write/read as 'clustal' format
  Checking can write/read as 'maf' format
  Checking can write/read as 'mauve' format
diff --git a/Tests/test_AlignIO.py b/Tests/test_AlignIO.py
--- a/Tests/test_AlignIO.py
+++ b/Tests/test_AlignIO.py
@@ -93,6 +93,10 @@ def alignment_summary(alignment, index="  ", vertical_threshold=5):
         for record in alignment:
             answer.append("%s%s %s" % (
                 index, str_summary(str(record.seq)), record.id))
+        for key, value in alignment.column_annotations.items():
+            if isinstance(value, str):
+                answer.append("%s%s %s" %
+                              (index, str_summary(str(value)), key))
     else:
         # Show each sequence row vertically
         for i in range(min(5, alignment_len)):