import argparse import os import gzip import pickle import numpy as np import pandas as pd import tensorflow as tf from Bio import SeqIO # column names ID_COL = 'Transcript ID' SEQ_COL = 'Transcript Sequence' TARGET_COL = 'Target Sequence' GUIDE_COL = 'Guide Sequence' MM_COL = 'Number of Mismatches' SCORE_COL = 'Guide Score' # nucleotide tokens NUCLEOTIDE_TOKENS = dict(zip(['A', 'C', 'G', 'T', 'N'], [0, 1, 2, 3, 255])) NUCLEOTIDE_COMPLEMENT = dict(zip(['A', 'C', 'G', 'T'], ['T', 'G', 'C', 'A'])) # model hyper-parameters GUIDE_LEN = 23 CONTEXT_5P = 3 CONTEXT_3P = 0 TARGET_LEN = CONTEXT_5P + GUIDE_LEN + CONTEXT_3P UNIT_INTERVAL_MAP = 'exp-lin-exp' # reference transcript files REFERENCE_TRANSCRIPTS = ('gencode.v19.pc_transcripts.fa.gz', 'gencode.v19.lncRNA_transcripts.fa.gz') # application configuration BATCH_SIZE_COMPUTE = 500 BATCH_SIZE_SCAN = 20 BATCH_SIZE_TRANSCRIPTS = 50 NUM_TOP_GUIDES = 10 NUM_MISMATCHES = 3 RUN_MODES = dict( all='All on-target guides per transcript', top_guides='Top {:d} guides per transcript'.format(NUM_TOP_GUIDES), titration='Top {:d} guides per transcript & their titration candidates'.format(NUM_TOP_GUIDES) ) # configure GPUs for gpu in tf.config.list_physical_devices('GPU'): tf.config.experimental.set_memory_growth(gpu, enable=True) if len(tf.config.list_physical_devices('GPU')) > 0: tf.config.experimental.set_visible_devices(tf.config.list_physical_devices('GPU')[0], 'GPU') def load_transcripts(fasta_files: list, enforce_unique_ids: bool = True): # load all transcripts from fasta files into a DataFrame transcripts = pd.DataFrame() for file in fasta_files: try: if os.path.splitext(file)[1] == '.gz': with gzip.open(file, 'rt') as f: df = pd.DataFrame([(t.id, str(t.seq)) for t in SeqIO.parse(f, 'fasta')], columns=[ID_COL, SEQ_COL]) else: df = pd.DataFrame([(t.id, str(t.seq)) for t in SeqIO.parse(file, 'fasta')], columns=[ID_COL, SEQ_COL]) except Exception as e: print(e, 'while loading', file) continue transcripts = pd.concat([transcripts, df]) # set index transcripts[ID_COL] = transcripts[ID_COL].apply(lambda s: s.split('|')[0]) transcripts.set_index(ID_COL, inplace=True) if enforce_unique_ids: assert not transcripts.index.has_duplicates, "duplicate transcript ID's detected in fasta file" return transcripts def sequence_complement(sequence: list): return [''.join([NUCLEOTIDE_COMPLEMENT[nt] for nt in list(seq)]) for seq in sequence] def one_hot_encode_sequence(sequence: list, add_context_padding: bool = False): # stack list of sequences into a tensor sequence = tf.ragged.stack([tf.constant(list(seq)) for seq in sequence], axis=0) # tokenize sequence nucleotide_table = tf.lookup.StaticVocabularyTable( initializer=tf.lookup.KeyValueTensorInitializer( keys=tf.constant(list(NUCLEOTIDE_TOKENS.keys()), dtype=tf.string), values=tf.constant(list(NUCLEOTIDE_TOKENS.values()), dtype=tf.int64)), num_oov_buckets=1) sequence = tf.RaggedTensor.from_row_splits(values=nucleotide_table.lookup(sequence.values), row_splits=sequence.row_splits).to_tensor(255) # add context padding if requested if add_context_padding: pad_5p = 255 * tf.ones([sequence.shape[0], CONTEXT_5P], dtype=sequence.dtype) pad_3p = 255 * tf.ones([sequence.shape[0], CONTEXT_3P], dtype=sequence.dtype) sequence = tf.concat([pad_5p, sequence, pad_3p], axis=1) # one-hot encode sequence = tf.one_hot(sequence, depth=4, dtype=tf.float16) return sequence def process_data(transcript_seq: str): # convert to upper case transcript_seq = transcript_seq.upper() # get all target sites target_seq = [transcript_seq[i: i + TARGET_LEN] for i in range(len(transcript_seq) - TARGET_LEN + 1)] # prepare guide sequences guide_seq = sequence_complement([seq[CONTEXT_5P:len(seq) - CONTEXT_3P] for seq in target_seq]) # model inputs model_inputs = tf.concat([ tf.reshape(one_hot_encode_sequence(target_seq, add_context_padding=False), [len(target_seq), -1]), tf.reshape(one_hot_encode_sequence(guide_seq, add_context_padding=True), [len(guide_seq), -1]), ], axis=-1) return target_seq, guide_seq, model_inputs def prediction_transform(predictions: np.array, **params): if len(params) == 0: with open('transform_params.pkl', 'rb') as f: params = pickle.load(f) if UNIT_INTERVAL_MAP == 'sigmoid': return 1 - 1 / (1 + np.exp(params['a'] * predictions + params['b'])) elif UNIT_INTERVAL_MAP == 'min-max': return 1 - (predictions - params['a']) / (params['b'] - params['a']) elif UNIT_INTERVAL_MAP == 'exp-lin-exp': # regime indices active_saturation = predictions < params['a'] linear_regime = (params['a'] <= predictions) & (predictions <= params['c']) inactive_saturation = params['c'] < predictions # linear regime slope = (params['d'] - params['b']) / (params['c'] - params['a']) intercept = -params['a'] * slope + params['b'] predictions[linear_regime] = slope * predictions[linear_regime] + intercept # active saturation regime alpha = slope / params['b'] beta = alpha * params['a'] - np.log(params['b']) predictions[active_saturation] = np.exp(alpha * predictions[active_saturation] - beta) # inactive saturation regime alpha = slope / (1 - params['d']) beta = -alpha * params['c'] - np.log(1 - params['d']) predictions[inactive_saturation] = 1 - np.exp(-alpha * predictions[inactive_saturation] - beta) return 1 - predictions else: raise NotImplementedError def get_on_target_predictions(transcripts: pd.DataFrame, model: tf.keras.Model, status_update_fn=None): # loop over transcripts predictions = pd.DataFrame() for i, (index, row) in enumerate(transcripts.iterrows()): # parse transcript sequence target_seq, guide_seq, model_inputs = process_data(row[SEQ_COL]) # get predictions lfc_estimate = model.predict(model_inputs, batch_size=BATCH_SIZE_COMPUTE, verbose=False) scores = prediction_transform(tf.squeeze(lfc_estimate).numpy()) predictions = pd.concat([predictions, pd.DataFrame({ ID_COL: [index] * len(scores), TARGET_COL: target_seq, GUIDE_COL: guide_seq, SCORE_COL: scores})]) # progress update percent_complete = 100 * min((i + 1) / len(transcripts), 1) update_text = 'Evaluating on-target guides for each transcript: {:.2f}%'.format(percent_complete) print('\r' + update_text, end='') if status_update_fn is not None: status_update_fn(update_text, percent_complete) print('') return predictions def top_guides_per_transcript(predictions: pd.DataFrame): # select and sort top guides for each transcript top_guides = pd.DataFrame() for transcript in predictions[ID_COL].unique(): df = predictions.loc[predictions[ID_COL] == transcript] df = df.sort_values(SCORE_COL, ascending=False).reset_index(drop=True).iloc[:NUM_TOP_GUIDES] top_guides = pd.concat([top_guides, df]) return top_guides.reset_index(drop=True) def get_titration_candidates(top_guide_predictions: pd.DataFrame): # generate a table of all titration candidates titration_candidates = pd.DataFrame() for _, row in top_guide_predictions.iterrows(): for i in range(len(row[GUIDE_COL])): nt = row[GUIDE_COL][i] for mutation in set(NUCLEOTIDE_TOKENS.keys()) - {nt, 'N'}: sm_guide = list(row[GUIDE_COL]) sm_guide[i] = mutation sm_guide = ''.join(sm_guide) assert row[GUIDE_COL] != sm_guide titration_candidates = pd.concat([titration_candidates, pd.DataFrame({ ID_COL: [row[ID_COL]], TARGET_COL: [row[TARGET_COL]], GUIDE_COL: [sm_guide] })]) return titration_candidates def find_off_targets(top_guides: pd.DataFrame, status_update_fn=None): # load reference transcripts reference_transcripts = load_transcripts([os.path.join('transcripts', f) for f in REFERENCE_TRANSCRIPTS]) # one-hot encode guides to form a filter guide_filter = one_hot_encode_sequence(sequence_complement(top_guides[GUIDE_COL]), add_context_padding=False) guide_filter = tf.transpose(guide_filter, [1, 2, 0]) # loop over transcripts in batches i = 0 off_targets = pd.DataFrame() while i < len(reference_transcripts): # select batch df_batch = reference_transcripts.iloc[i:min(i + BATCH_SIZE_SCAN, len(reference_transcripts))] i += BATCH_SIZE_SCAN # find locations of off-targets transcripts = one_hot_encode_sequence(df_batch[SEQ_COL].values.tolist(), add_context_padding=False) num_mismatches = GUIDE_LEN - tf.nn.conv1d(transcripts, guide_filter, stride=1, padding='SAME') loc_off_targets = tf.where(tf.round(num_mismatches) <= NUM_MISMATCHES).numpy() # off-targets discovered if len(loc_off_targets) > 0: # log off-targets dict_off_targets = pd.DataFrame({ 'On-target ' + ID_COL: top_guides.iloc[loc_off_targets[:, 2]][ID_COL], GUIDE_COL: top_guides.iloc[loc_off_targets[:, 2]][GUIDE_COL], 'Off-target ' + ID_COL: df_batch.index.values[loc_off_targets[:, 0]], 'Guide Midpoint': loc_off_targets[:, 1], SEQ_COL: df_batch[SEQ_COL].values[loc_off_targets[:, 0]], MM_COL: tf.gather_nd(num_mismatches, loc_off_targets).numpy().astype(int), }).to_dict('records') # trim transcripts to targets for row in dict_off_targets: start_location = row['Guide Midpoint'] - (GUIDE_LEN // 2) del row['Guide Midpoint'] target = row[SEQ_COL] del row[SEQ_COL] if start_location < CONTEXT_5P: target = target[0:GUIDE_LEN + CONTEXT_3P] target = 'N' * (TARGET_LEN - len(target)) + target elif start_location + GUIDE_LEN + CONTEXT_3P > len(target): target = target[start_location - CONTEXT_5P:] target = target + 'N' * (TARGET_LEN - len(target)) else: target = target[start_location - CONTEXT_5P:start_location + GUIDE_LEN + CONTEXT_3P] if row[MM_COL] == 0 and 'N' not in target: assert row[GUIDE_COL] == sequence_complement([target[CONTEXT_5P:TARGET_LEN - CONTEXT_3P]])[0] row[TARGET_COL] = target # append new off-targets off_targets = pd.concat([off_targets, pd.DataFrame(dict_off_targets)]) # progress update percent_complete = 100 * min((i + 1) / len(reference_transcripts), 1) update_text = 'Scanning for off-targets: {:.2f}%'.format(percent_complete) print('\r' + update_text, end='') if status_update_fn is not None: status_update_fn(update_text, percent_complete) print('') return off_targets def predict_off_target(off_targets: pd.DataFrame, model: tf.keras.Model): if len(off_targets) == 0: return pd.DataFrame() # compute off-target predictions model_inputs = tf.concat([ tf.reshape(one_hot_encode_sequence(off_targets[TARGET_COL], add_context_padding=False), [len(off_targets), -1]), tf.reshape(one_hot_encode_sequence(off_targets[GUIDE_COL], add_context_padding=True), [len(off_targets), -1]), ], axis=-1) lfc_estimate = model.predict(model_inputs, batch_size=BATCH_SIZE_COMPUTE, verbose=False) off_targets[SCORE_COL] = prediction_transform(tf.squeeze(lfc_estimate).numpy()) return off_targets.reset_index(drop=True) def tiger_exhibit(transcripts: pd.DataFrame, mode: str, check_off_targets: bool, status_update_fn=None): # load model if os.path.exists('model'): tiger = tf.keras.models.load_model('model') else: print('no saved model!') exit() # evaluate all on-target guides per transcript on_target_predictions = get_on_target_predictions(transcripts, tiger, status_update_fn) # initialize other outputs titration_predictions = off_target_predictions = None if mode == 'all' and not check_off_targets: off_target_candidates = None elif mode == 'top_guides': on_target_predictions = top_guides_per_transcript(on_target_predictions) off_target_candidates = on_target_predictions elif mode == 'titration': on_target_predictions = top_guides_per_transcript(on_target_predictions) titration_candidates = get_titration_candidates(on_target_predictions) titration_predictions = predict_off_target(titration_candidates, model=tiger) off_target_candidates = pd.concat([on_target_predictions, titration_predictions]) else: raise NotImplementedError # check off-target effects for top guides if check_off_targets and off_target_candidates is not None: off_target_candidates = find_off_targets(off_target_candidates, status_update_fn) off_target_predictions = predict_off_target(off_target_candidates, model=tiger) off_target_predictions = off_target_predictions.sort_values(SCORE_COL, ascending=False).reset_index(drop=True) # finalize tables for df in [on_target_predictions, titration_predictions, off_target_predictions]: if df is not None: df[GUIDE_COL] = df[GUIDE_COL].apply(lambda s: s[::-1]) # reverse guide sequences df[TARGET_COL] = df[TARGET_COL].apply(lambda seq: seq[CONTEXT_5P:len(seq) - CONTEXT_3P]) # remove context return on_target_predictions, titration_predictions, off_target_predictions if __name__ == '__main__': # common arguments parser = argparse.ArgumentParser() parser.add_argument('--mode', type=str, default='titration') parser.add_argument('--check_off_targets', action='store_true', default=False) parser.add_argument('--fasta_path', type=str, default=None) args = parser.parse_args() # check for any existing results if os.path.exists('on_target.csv') or os.path.exists('titration.csv') or os.path.exists('off_target.csv'): raise FileExistsError('please rename or delete existing results') # load transcripts from a directory of fasta files if args.fasta_path is not None and os.path.exists(args.fasta_path): df_transcripts = load_transcripts([os.path.join(args.fasta_path, f) for f in os.listdir(args.fasta_path)]) # otherwise consider simple test case with first 50 nucleotides from EIF3B-003's CDS else: df_transcripts = pd.DataFrame({ ID_COL: ['ManualEntry'], SEQ_COL: ['ATGCAGGACGCGGAGAACGTGGCGGTGCCCGAGGCGGCCGAGGAGCGCGC']}) df_transcripts.set_index(ID_COL, inplace=True) # process in batches batch = 0 num_batches = len(df_transcripts) // BATCH_SIZE_TRANSCRIPTS num_batches += (len(df_transcripts) % BATCH_SIZE_TRANSCRIPTS > 0) for idx in range(0, len(df_transcripts), BATCH_SIZE_TRANSCRIPTS): batch += 1 print('Batch {:d} of {:d}'.format(batch, num_batches)) # run batch idx_stop = min(idx + BATCH_SIZE_TRANSCRIPTS, len(df_transcripts)) df_on_target, df_titration, df_off_target = tiger_exhibit( transcripts=df_transcripts[idx:idx_stop], mode=args.mode, check_off_targets=args.check_off_targets ) # save batch results df_on_target.to_csv('on_target.csv', header=batch == 1, index=False, mode='a') if df_titration is not None: df_titration.to_csv('titration.csv', header=batch == 1, index=False, mode='a') if df_off_target is not None: df_off_target.to_csv('off_target.csv', header=batch == 1, index=False, mode='a') # clear session to prevent memory blow up tf.keras.backend.clear_session()