utils/msa_utils.py

import numpy as np
import pandas as pd
from collections import defaultdict
import random
import os
import torch
from Bio.Align.Applications import ClustalOmegaCommandline

def filter_msa(msa_data, num_sequences_kept=3):
    """
    Helper function to filter an input MSA msa_data (obtained via process_msa_data) and keep only num_sequences_kept aligned sequences.
    If the MSA already has fewer sequences than num_sequences_kept, we keep the MSA as is.
    If filtering, we always keep the first sequence of the MSA (ie. the wild type) by default.
    Sampling is done without replacement.
    """
    if len(list(msa_data.keys())) <= num_sequences_kept:
        return  msa_data
    filtered_msa = {}
    wt_name = next(iter(msa_data)) 
    filtered_msa[wt_name] = msa_data[wt_name]
    del msa_data[wt_name]
    sequence_names = list(msa_data.keys())
    sequence_names_sampled  = random.sample(sequence_names,k=num_sequences_kept-1)
    for seq in sequence_names_sampled:
        filtered_msa[seq] = msa_data[seq]
    return filtered_msa

def process_msa_data(MSA_data_file):
    """
    Helper function that takes as input a path to a MSA file (expects a2m format) and returns a dict mapping sequence ID to the corresponding AA sequence.
    """
    msa_data = defaultdict(str)
    sequence_name = ""
    with open(MSA_data_file, "r") as msa_file:
        for i, line in enumerate(msa_file):
            line = line.rstrip()
            if line.startswith(">"):
                sequence_name = line
            else:
                msa_data[sequence_name] += line.upper()
    return msa_data

def get_one_hot_sequences_dict(msa_data,MSA_start,MSA_end,vocab):
    vocab_size = len(vocab.keys())
    num_sequences_msa = len(msa_data.keys())
    one_hots = np.zeros((num_sequences_msa,MSA_end-MSA_start,vocab_size))
    for i,seq_name in enumerate(msa_data.keys()):
        sequence = msa_data[seq_name]
        for j,letter in enumerate(sequence):
            if letter in vocab: 
                k = vocab[letter]
                one_hots[i,j,k] = 1.0
    return one_hots

def one_hot(sequence_string,vocab):
    one_hots = np.zeros((len(sequence_string),len(vocab.keys())))
    for j,letter in enumerate(sequence_string):
        if letter in vocab: 
            k = vocab[letter]
            one_hots[j,k] = 1.0
    return one_hots.flatten()

def get_msa_prior(MSA_data_file, MSA_weight_file_name, MSA_start, MSA_end, len_target_seq, vocab, retrieval_aggregation_mode="aggregate_substitution", filter_MSA=True, verbose=False):
    """
    Function to enable retrieval inference mode, via computation of (weighted) pseudocounts of AAs at each position of the retrieved MSA.
    MSA_data_file: (string) path to MSA file (expects a2m format).
    MSA_weight_file_name: (string) path to sequence weights in MSA.
    MSA_start: (int) Sequence position that the MSA starts at (1-indexing).
    MSA_end: (int) Sequence position that the MSA ends at (1-indexing).
    len_target_seq: (int) Full length of sequence to be scored.
    vocab: (dict) Vocabulary of the tokenizer.
    retrieval_aggregation_mode: (string) Mode for retrieval inference (aggregate_substitution Vs aggregate_indel). If None, places a uniform prior over each token.
    filter_MSA: (bool) Whether to filter out sequences with very low hamming similarity (< 0.2) to the reference sequence in the MSA (first sequence).
    verbose: (bool) Whether to print to the console processing details along the way.
    """
    msa_data = process_msa_data(MSA_data_file)
    vocab_size = len(vocab.keys())
    if verbose: print("Target seq len is {}, MSA length is {}, start position is {}, end position is {} and vocab size is {}".format(len_target_seq,MSA_end-MSA_start,MSA_start,MSA_end,vocab_size))

    if filter_MSA:
        if verbose: print("Num sequences in MSA pre filtering: {}".format(len(msa_data.keys())))
        list_sequence_names = list(msa_data.keys())
        focus_sequence_name = list(msa_data.keys())[0]
        ref_sequence_hot = one_hot(msa_data[focus_sequence_name],vocab)
        for sequence_name in list_sequence_names:
            seq_hot = one_hot(msa_data[sequence_name],vocab)
            hamming_similarity_seq_ref = np.dot(ref_sequence_hot,seq_hot) / np.dot(ref_sequence_hot,ref_sequence_hot)
            if hamming_similarity_seq_ref < 0.2:
                del msa_data[sequence_name]
        if verbose: print("Num sequences in MSA post filtering: {}".format(len(msa_data.keys())))

    if MSA_weight_file_name is not None:
        if verbose: print("Using weights in {} for sequences in MSA.".format(MSA_weight_file_name))
        assert os.path.exists(MSA_weight_file_name), "Weights file not located on disk."
        MSA_EVE = MSA_processing(
                MSA_location=MSA_data_file,
                use_weights=True,
                weights_location=MSA_weight_file_name
        )
        #We scan through all sequences to see if we have a weight for them as per EVE pre-processing. We drop them otherwise.
        dropped_sequences=0
        list_sequence_names = list(msa_data.keys())
        MSA_weight=[]
        for sequence_name in list_sequence_names:
            if sequence_name not in MSA_EVE.seq_name_to_sequence:
                dropped_sequences +=1
                del msa_data[sequence_name]
            else:
                MSA_weight.append(MSA_EVE.seq_name_to_weight[sequence_name])
        if verbose: print("Dropped {} sequences from MSA due to absent sequence weights".format(dropped_sequences))
    else:
        MSA_weight = [1] *  len(list(msa_data.keys()))

    if retrieval_aggregation_mode=="aggregate_substitution" or retrieval_aggregation_mode=="aggregate_indel":
        one_hots = get_one_hot_sequences_dict(msa_data,MSA_start,MSA_end,vocab)
        MSA_weight = np.expand_dims(np.array(MSA_weight),axis=(1,2))
        base_rate = 1e-5
        base_rates = np.ones_like(one_hots) * base_rate
        weighted_one_hots = (one_hots + base_rates) * MSA_weight
        MSA_weight_norm_counts = weighted_one_hots.sum(axis=-1).sum(axis=0)
        MSA_weight_norm_counts = np.tile(MSA_weight_norm_counts.reshape(-1,1), (1,vocab_size))
        one_hots_avg = weighted_one_hots.sum(axis=0) / MSA_weight_norm_counts
        msa_prior = np.zeros((len_target_seq,vocab_size))
        msa_prior[MSA_start:MSA_end,:]=one_hots_avg
    else:
        msa_prior = np.ones((len_target_seq,vocab_size)) / vocab_size
    
    if verbose:
        for idx, position in enumerate(msa_prior):
            if len(position)!=25:
                print("Size error")
            if not round(position.sum(),2)==1.0:
                print("Position at index {} does not add up to 1: {}".format(idx, position.sum()))
    
    return msa_prior


def update_retrieved_MSA_log_prior_indel(model, MSA_log_prior, MSA_start, MSA_end, full_raw_sequence):
    """
    Function to process MSA when scoring indels.
    To identify positions to add / remove in the retrieved MSA, we append and align the sequence to be scored to the original MSA for that protein family with Clustal Omega.
    If the original MSA is relatively deep (over 100k sequences), we sample (by default) 100k rows at random from that MSA to speed computations.
    MSA sampling is performed only once (for the first sequence to be scored). Subsequent scoring use the same MSA sample.
    """
    if not os.path.isdir(model.MSA_folder + os.sep + "Sampled"):
        os.mkdir(model.MSA_folder + os.sep + "Sampled")
    sampled_MSA_location = model.MSA_folder + os.sep + "Sampled" + os.sep + "Sampled_" + model.MSA_filename.split(os.sep)[-1]
    
    if not os.path.exists(sampled_MSA_location):
        msa_data = process_msa_data(model.MSA_filename)
        msa_data_sampled = filter_msa(msa_data, num_sequences_kept=100000) #If MSA has less than 100k sequences, the sample is identical to original MSA
        with open(sampled_MSA_location, 'w') as sampled_write_location:
            for index, key in enumerate(msa_data_sampled):
                key_name = ">REFERENCE_SEQUENCE" if index==0 else key
                msa_data_sampled[key] = msa_data_sampled[key].upper()
                msa_data_sampled[key] = msa_data_sampled[key].replace(".","-")
                sampled_write_location.write(key_name+"\n"+"\n".join([msa_data_sampled[key][i:i+80] for i in range(0, len(msa_data_sampled[key]), 80)])+"\n")
    
    seq_to_align_location = model.MSA_folder + os.sep + "Sampled" + os.sep + "Seq_to_align_" + model.MSA_filename.split(os.sep)[-1]
    sequence_text_split = [full_raw_sequence[i:i+80] for i in range(0, len(full_raw_sequence), 80)]
    sequence_text_split_split_join = "\n".join([">SEQ_TO_SCORE"]+sequence_text_split)
    os.system("echo '"+sequence_text_split_split_join+"' > "+seq_to_align_location)
    
    expanded_MSA_location = model.MSA_folder + os.sep + "Sampled" + os.sep + "Expanded_" + model.MSA_filename.split(os.sep)[-1]
    clustalw_cline = ClustalOmegaCommandline(cmd=model.config.clustal_omega_location,
                                            profile1=sampled_MSA_location,
                                            profile2=seq_to_align_location,
                                            outfile=expanded_MSA_location,
                                            force=True)                                        
    stdout, stderr = clustalw_cline()
    msa_data = process_msa_data(expanded_MSA_location)
    aligned_seqA, aligned_seqB = msa_data[">SEQ_TO_SCORE"], msa_data[">REFERENCE_SEQUENCE"]
    try:
        keep_column=[]
        for column_index_pairwise_alignment in range(len(aligned_seqA)):
            if aligned_seqA[column_index_pairwise_alignment]=="-" and aligned_seqB[column_index_pairwise_alignment]=="-":
                continue
            elif aligned_seqA[column_index_pairwise_alignment]=="-":
                keep_column.append(False)
            elif aligned_seqB[column_index_pairwise_alignment]=="-":
                MSA_log_prior=torch.cat((MSA_log_prior[:column_index_pairwise_alignment], torch.zeros(MSA_log_prior.shape[1]).view(1,-1).cuda(), MSA_log_prior[column_index_pairwise_alignment:]),dim=0)
                keep_column.append(True) #keep the zero column we just added
            else:
                keep_column.append(True)
        MSA_log_prior = MSA_log_prior[keep_column]
        MSA_end = MSA_start + len(MSA_log_prior)
    except: 
        print("Error when processing the following alignment: {}".format(expanded_MSA_location))
    return MSA_log_prior, MSA_start, MSA_end

class MSA_processing:
    def __init__(self,
        MSA_location="",
        theta=0.2,
        use_weights=True,
        weights_location="./data/weights",
        preprocess_MSA=True,
        threshold_sequence_frac_gaps=0.5,
        threshold_focus_cols_frac_gaps=0.3,
        remove_sequences_with_indeterminate_AA_in_focus_cols=True
        ):
        
        """
        This MSA_processing class is directly borrowed from the EVE codebase: https://github.com/OATML-Markslab/EVE
        
        Parameters:
        - msa_location: (path) Location of the MSA data. Constraints on input MSA format: 
            - focus_sequence is the first one in the MSA data
            - first line is structured as follows: ">focus_seq_name/start_pos-end_pos" (e.g., >SPIKE_SARS2/310-550)
            - corespondding sequence data located on following line(s)
            - then all other sequences follow with ">name" on first line, corresponding data on subsequent lines
        - theta: (float) Sequence weighting hyperparameter. Generally: Prokaryotic and eukaryotic families =  0.2; Viruses = 0.01
        - use_weights: (bool) If False, sets all sequence weights to 1. If True, checks weights_location -- if non empty uses that; 
            otherwise compute weights from scratch and store them at weights_location
        - weights_location: (path) Location to load from/save to the sequence weights
        - preprocess_MSA: (bool) performs pre-processing of MSA to remove short fragments and positions that are not well covered.
        - threshold_sequence_frac_gaps: (float, between 0 and 1) Threshold value to define fragments
            - sequences with a fraction of gap characters above threshold_sequence_frac_gaps are removed
            - default is set to 0.5 (i.e., fragments with 50% or more gaps are removed)
        - threshold_focus_cols_frac_gaps: (float, between 0 and 1) Threshold value to define focus columns
            - positions with a fraction of gap characters above threshold_focus_cols_pct_gaps will be set to lower case (and not included in the focus_cols)
            - default is set to 0.3 (i.e., focus positions are the ones with 30% of gaps or less, i.e., 70% or more residue occupancy)
        - remove_sequences_with_indeterminate_AA_in_focus_cols: (bool) Remove all sequences that have indeterminate AA (e.g., B, J, X, Z) at focus positions of the wild type
        """
        np.random.seed(2021)
        self.MSA_location = MSA_location
        self.weights_location = weights_location
        self.theta = theta
        self.alphabet = "ACDEFGHIKLMNPQRSTVWY"
        self.use_weights = use_weights
        self.preprocess_MSA = preprocess_MSA
        self.threshold_sequence_frac_gaps = threshold_sequence_frac_gaps
        self.threshold_focus_cols_frac_gaps = threshold_focus_cols_frac_gaps
        self.remove_sequences_with_indeterminate_AA_in_focus_cols = remove_sequences_with_indeterminate_AA_in_focus_cols

        self.gen_alignment()
        
    def gen_alignment(self, verbose=False):
        """ Read training alignment and store basics in class instance """
        self.aa_dict = {}
        for i,aa in enumerate(self.alphabet):
            self.aa_dict[aa] = i

        self.seq_name_to_sequence = defaultdict(str)
        name = ""
        with open(self.MSA_location, "r") as msa_data:
            for i, line in enumerate(msa_data):
                line = line.rstrip()
                if line.startswith(">"):
                    name = line
                    if i==0:
                        self.focus_seq_name = name
                else:
                    self.seq_name_to_sequence[name] += line

        
        ## MSA pre-processing to remove inadequate columns and sequences
        if self.preprocess_MSA:
            msa_df = pd.DataFrame.from_dict(self.seq_name_to_sequence, orient='index', columns=['sequence'])
            # Data clean up
            msa_df.sequence = msa_df.sequence.apply(lambda x: x.replace(".","-")).apply(lambda x: ''.join([aa.upper() for aa in x]))
            # Remove columns that would be gaps in the wild type
            non_gap_wt_cols = [aa!='-' for aa in msa_df.sequence[self.focus_seq_name]]
            msa_df['sequence'] = msa_df['sequence'].apply(lambda x: ''.join([aa for aa,non_gap_ind in zip(x, non_gap_wt_cols) if non_gap_ind]))
            assert 0.0 <= self.threshold_sequence_frac_gaps <= 1.0,"Invalid fragment filtering parameter"
            assert 0.0 <= self.threshold_focus_cols_frac_gaps <= 1.0,"Invalid focus position filtering parameter"
            msa_array = np.array([list(seq) for seq in msa_df.sequence])
            gaps_array = np.array(list(map(lambda seq: [aa=='-' for aa in seq], msa_array)))
            # Identify fragments with too many gaps
            seq_gaps_frac = gaps_array.mean(axis=1)
            seq_below_threshold = seq_gaps_frac <= self.threshold_sequence_frac_gaps
            if verbose: print("Proportion of sequences dropped due to fraction of gaps: "+str(round(float(1 - seq_below_threshold.sum()/seq_below_threshold.shape)*100,2))+"%")
            # Identify focus columns
            columns_gaps_frac = gaps_array[seq_below_threshold].mean(axis=0)
            index_cols_below_threshold = columns_gaps_frac <= self.threshold_focus_cols_frac_gaps
            if verbose: print("Proportion of non-focus columns removed: "+str(round(float(1 - index_cols_below_threshold.sum()/index_cols_below_threshold.shape)*100,2))+"%")
            # Lower case non focus cols and filter fragment sequences
            msa_df['sequence'] = msa_df['sequence'].apply(lambda x: ''.join([aa.upper() if upper_case_ind else aa.lower() for aa, upper_case_ind in zip(x, index_cols_below_threshold)]))
            msa_df = msa_df[seq_below_threshold]
            # Overwrite seq_name_to_sequence with clean version
            self.seq_name_to_sequence = defaultdict(str)
            for seq_idx in range(len(msa_df['sequence'])):
                self.seq_name_to_sequence[msa_df.index[seq_idx]] = msa_df.sequence[seq_idx]

        self.focus_seq = self.seq_name_to_sequence[self.focus_seq_name]
        self.focus_cols = [ix for ix, s in enumerate(self.focus_seq) if s == s.upper() and s!='-'] 
        self.focus_seq_trimmed = [self.focus_seq[ix] for ix in self.focus_cols]
        self.seq_len = len(self.focus_cols)
        self.alphabet_size = len(self.alphabet)

        # Connect local sequence index with uniprot index (index shift inferred from 1st row of MSA)
        focus_loc = self.focus_seq_name.split("/")[-1]
        start,stop = focus_loc.split("-")
        self.focus_start_loc = int(start)
        self.focus_stop_loc = int(stop)
        self.uniprot_focus_col_to_wt_aa_dict \
            = {idx_col+int(start):self.focus_seq[idx_col] for idx_col in self.focus_cols} 
        self.uniprot_focus_col_to_focus_idx \
            = {idx_col+int(start):idx_col for idx_col in self.focus_cols} 

        # Move all letters to CAPS; keeps focus columns only
        self.raw_seq_name_to_sequence = self.seq_name_to_sequence.copy()
        for seq_name,sequence in self.seq_name_to_sequence.items():
            sequence = sequence.replace(".","-")
            self.seq_name_to_sequence[seq_name] = [sequence[ix].upper() for ix in self.focus_cols]

        # Remove sequences that have indeterminate AA (e.g., B, J, X, Z) in the focus columns
        if self.remove_sequences_with_indeterminate_AA_in_focus_cols:
            alphabet_set = set(list(self.alphabet))
            seq_names_to_remove = []
            for seq_name,sequence in self.seq_name_to_sequence.items():
                for letter in sequence:
                    if letter not in alphabet_set and letter != "-":
                        seq_names_to_remove.append(seq_name)
                        continue
            seq_names_to_remove = list(set(seq_names_to_remove))
            for seq_name in seq_names_to_remove:
                del self.seq_name_to_sequence[seq_name]

        # Encode the sequences
        self.one_hot_encoding = np.zeros((len(self.seq_name_to_sequence.keys()),len(self.focus_cols),len(self.alphabet)))
        if verbose: print("One-hot encoded sequences shape:" + str(self.one_hot_encoding.shape))
        for i,seq_name in enumerate(self.seq_name_to_sequence.keys()):
            sequence = self.seq_name_to_sequence[seq_name]
            for j,letter in enumerate(sequence):
                if letter in self.aa_dict: 
                    k = self.aa_dict[letter]
                    self.one_hot_encoding[i,j,k] = 1.0

        if self.use_weights:
            try:
                self.weights = np.load(file=self.weights_location)
                if verbose: print("Loaded sequence weights from disk")
            except:
                if verbose: print ("Computing sequence weights")
                list_seq = self.one_hot_encoding
                list_seq = list_seq.reshape((list_seq.shape[0], list_seq.shape[1] * list_seq.shape[2]))
                def compute_weight(seq):
                    number_non_empty_positions = np.dot(seq,seq)
                    if number_non_empty_positions>0:
                        denom = np.dot(list_seq,seq) / np.dot(seq,seq) 
                        denom = np.sum(denom > 1 - self.theta) 
                        return 1/denom
                    else:
                        return 0.0 #return 0 weight if sequence is fully empty
                self.weights = np.array(list(map(compute_weight,list_seq)))
                np.save(file=self.weights_location, arr=self.weights)
        else:
            # If not using weights, use an isotropic weight matrix
            if verbose: print("Not weighting sequence data")
            self.weights = np.ones(self.one_hot_encoding.shape[0])

        self.Neff = np.sum(self.weights)
        self.num_sequences = self.one_hot_encoding.shape[0]
        self.seq_name_to_weight={}
        for i,seq_name in enumerate(self.seq_name_to_sequence.keys()):
            self.seq_name_to_weight[seq_name]=self.weights[i]

        if verbose:
            print ("Neff =",str(self.Neff))
            print ("Data Shape =",self.one_hot_encoding.shape)