Kaggle competition: Mechanisms of Action (MoA) Prediction
def get_feats(doc):
token_length = len(doc.split())
 i1 = len(re.findall(r'(?=\bI\b|\bwe\b|\bme\b|\bus\b|\bmy\b|\bour\b|\bmine\b|\bours\b)', doc)) /token_length
i2 = len(re.findall(r'(?=\byou\b|\byour\b|\byours\b)', doc)) /token_length
i3 = len(re.findall(r'(?=\bhe\b|\bshe\b|\bit\b|\bhim\b|\bher\b|\bhis\b|\bhers\b|\bits\b)', doc)) /token_length
i4 = len(re.findall(r'(?=\bthey\b|\bthem\b|\btheir\b|\btheirs\b)', doc)) /token_length
i5 = len(re.findall(r'(?=\bwhat\b|\bwhatever\b|\bwhich\b|\bwhichever\b|\bwho\b|\bwhoever\b|\bwhom\b|\bwhomever\b|\bwhoses\b)', doc))
i6 = len(re.findall(r'(?=\bmyself\b|\byourself\b|\bherself\b|\bhimself\b|\bitself\b|\bourselves\b|\byourselves\b|\bthemselves\b)', doc))
i7 = len(re.findall(r'(?=\bfor\b|\band\b|\bnor\b|\bbut\b|\byet\b|\bso\b|\bbefore\b|\bonce\b|\bsince\b|\bthough\b|\bwhile\b|\bas\b|\bbecause\b|\bafter\b)', doc)) /token_length
i8 =(len(doc)- len(re.findall('[a-zA-Z]', doc)) - doc.count(' ')- len(re.findall('[0-9]', doc)) - doc.count(',')- doc.count('?')- doc.count('.'))
i9 = doc.count(',')
 i10 = doc.count('?')
 i11 = token_length
i12 = len(re.findall(r'(?=\bcan't\b|\bcouldn't\b|\bshan't\b|\bshouldn't\b|\bwouldn't\b|\bhaven't\b|\bdidn't\b|\bnot\b|\bnever\b|\bwon't\b|\bdon't\b|\bhadn't\b|\bcant\b|\bcouldnt\b|\bshant\b|\bshouldnt\b|\bwouldnt\b|\bhavent\b|\bdidnt\b|\bwont\b|\bdont\b|\bhadnt\b)', doc))
temp_doc =(" " ).join(re.findall(r"[a-zA-Z0-9']+", doc))
i13 = len(set(temp_doc.split())-final_vocab)
 return(i1,i2,i3,i4,i5,i6,i7,i8,i9,i10,i11,i12,i13 )<feature_engineering>
train=pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train
df['fp'] = df['feat_vect'].apply(lambda x: x[0])
 df['sp'] = df['feat_vect'].apply(lambda x: x[1])
 df['tps'] = df['feat_vect'].apply(lambda x: x[2])
 df['tpp'] = df['feat_vect'].apply(lambda x: x[3])
 df['conjunction'] = df['feat_vect'].apply(lambda x: x[6])
 df['special_chars'] = df['feat_vect'].apply(lambda x: x[7])
 df['commas'] = df['feat_vect'].apply(lambda x: x[8])
 df['qm'] = df['feat_vect'].apply(lambda x: x[9])
 df['len'] = df['feat_vect'].apply(lambda x: x[10])
 df['sm'] = df['feat_vect'].apply(lambda x: x[12])
 sincere = df[df['target']==0]
insincere = df[df['target']==1]<load_from_csv>
train.isnull().sum(axis=0 ).sum() , train.isnull().sum(axis=1 ).sum()
test = pd.read_csv('.. /input/test.csv')
 test['clean'] = test['question_text'].apply(cleaner )<feature_engineering>
train.nunique(dropna=False ).sort_values()
test['clean_2'] = test['question_text'].apply(cleaner_2 )<feature_engineering>
train_target = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv")
 limit = 0
for col in tqdm_notebook(train_target.columns):
if col != "sig_id":
print(train_target[col].value_counts())
 limit+=1
if limit >= 15:
break
tqdm.pandas()
test['feat_vect'] = test['question_text'].progress_apply(get_feats)
 test['fp'] = test['feat_vect'].apply(lambda x: x[0])
 test['sp'] = test['feat_vect'].apply(lambda x: x[1])
 test['tps'] = test['feat_vect'].apply(lambda x: x[2])
 test['tpp'] = test['feat_vect'].apply(lambda x: x[3])
 test['conjunction'] = test['feat_vect'].apply(lambda x: x[6])
 test['special_chars'] = test['feat_vect'].apply(lambda x: x[7])
 test['commas'] = test['feat_vect'].apply(lambda x: x[8])
 test['qm'] = test['feat_vect'].apply(lambda x: x[9])
 test['len'] = test['feat_vect'].apply(lambda x: x[10])
 test['sm'] = test['feat_vect'].apply(lambda x: x[12])
 test.head()<set_options>
ctlVehicle_idx = train["cp_type"] != "ctl_vehicle"
train = train.loc[ctlVehicle_idx].reset_index(drop=True)
 train = train.drop("cp_type", axis=1)
 train_target = train_target.loc[ctlVehicle_idx].reset_index(drop=True )
del final_vocab
gc.collect()<feature_engineering>
gcols = [g for g in train.columns if "g-" in g]
ccols = [c for c in train.columns if "c-" in c]
cpcols = [cp for cp in train.columns if "cp_" in cp]
def my_tweet_tokenizer(doc):
t1 = TweetTokenizer()
return t1.tokenize(doc)
 c2 = CountVectorizer(tokenizer=my_tweet_tokenizer, ngram_range=(2,3), min_df=6)
 tick = time.time()
c2.fit(insincere['clean'])
 bow2 = c2.transform(insincere['clean'])
 print('Time taken to fit CV: ' + str(time.time() -tick)+ ' seconds.')
 lda2 = LatentDirichletAllocation(n_components=8, learning_method='batch')
 tick = time.time()
lda2.fit(bow2)
 print('Time taken to fit lda: ' + str(time.time() -tick)+ ' seconds.')
<split>
test = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 ctlVehicle_test = test["cp_type"] == "ctl_vehicle"
test = test.drop("cp_type", axis=1)
 enc = LabelEncoder()
for col in train[cpcols]:
train[col] = enc.fit_transform(train[col])
 enc = LabelEncoder()
for col in test[cpcols]:
test[col] = enc.fit_transform(test[col] )
tick = time.time()
new_train_df = pd.concat([sincere.sample(frac=0.8, random_state=42),insincere])
 new_train_df = new_train_df.sample(frac=1, random_state=113)
 X_train, X_dev, y_train, y_dev = train_test_split(new_train_df, new_train_df['target'], test_size=0.2, random_state=42, stratify=new_train_df['target'])
 t1 = Tokenizer()
t1.fit_on_texts(pd.concat([df['clean_2'], test['clean_2']]))
t2 = Tokenizer(filters=None, char_level=True)
 t2.fit_on_texts(pd.concat([df['clean_2'], test['clean_2']]))
print('Time taken to fit keras tokenizers : ' + str(time.time() -tick)+ ' seconds.' )<string_transform>
g_pca = PCA(n_components=0.95)
 c_pca = PCA(n_components=0.95)
 train_test_g_concat = pd.concat([train[gcols], test[gcols]], axis=0)
 train_test_c_concat = pd.concat([train[ccols], test[ccols]], axis=0)
 g_pca.fit(train_test_g_concat)
 c_pca.fit(train_test_c_concat)
 train_gtrans = pd.DataFrame(g_pca.transform(train[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=train.index)
 test_gtrans = pd.DataFrame(g_pca.transform(test[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=test.index)
 train_ctrans = pd.DataFrame(c_pca.transform(train[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=train.index)
 test_ctrans = pd.DataFrame(c_pca.transform(test[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=test.index)
 g_pca.n_components_, c_pca.n_components_
tick = time.time()
X_train_tts = t1.texts_to_sequences(X_train['clean_2'])
 X_dev_tts = t1.texts_to_sequences(X_dev['clean_2'])
 test_tts = t1.texts_to_sequences(test['clean_2'])
 X_train_tts = pad_sequences(X_train_tts, maxlen=90, padding='post', truncating='post')
 X_dev_tts = pad_sequences(X_dev_tts, maxlen=90, padding='post', truncating='post')
 test_tts = pad_sequences(test_tts, maxlen=90, padding='post', truncating='post')
 X_train_cts = t2.texts_to_sequences(X_train['clean_2'])
 X_dev_cts = t2.texts_to_sequences(X_dev['clean_2'])
 test_cts = t2.texts_to_sequences(test['clean_2'])
 X_train_cts = pad_sequences(X_train_cts, maxlen=180, padding='post', truncating='post')
 X_dev_cts = pad_sequences(X_dev_cts, maxlen=180, padding='post', truncating='post')
 test_cts = pad_sequences(test_cts, maxlen=180, padding='post', truncating='post')
 print('Time taken for tts conversion : ' + str(time.time() -tick)+ ' seconds.' )<categorify>
train = pd.concat([train_gtrans, train_ctrans, train[cpcols]], axis=1)
 test = pd.concat([test_gtrans, test_ctrans, test[cpcols]], axis=1)
 train
tick = time.time()
bow_train = c2.transform(X_train['clean'])
 ldavec_train = lda2.transform(bow_train)
 bow_dev = c2.transform(X_dev['clean'])
 ldavec_dev = lda2.transform(bow_dev)
 bow_test = c2.transform(test['clean'])
 ldavec_test = lda2.transform(bow_test)
 print('Time taken to generate lda vectors for train, dev & test : ' + str(time.time() -tick)+ ' seconds.' )<drop_column>
train_target = train_target.drop("sig_id", axis=1)
 train_target
feat_train = X_train[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm']].values
feat_dev = X_dev[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm']].values
feat_test = test[['fp', 'sp', 'tps', 'tpp', 'conjunction', 'special_chars',
'commas', 'qm', 'len', 'sm']].values<split>
def create_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(1024, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(1024, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 1e-5)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-15), optimizer=lookahead_radam)
 return model
tick = time.time()
embedding_dim = 300
vocab_size = len(t1.word_index)+1
EMBEDDING_FILE = ".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_glove = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8'))
embedding_matrix = np.random.randn(vocab_size,embedding_dim ).astype(np.float32)* np.sqrt(2.0/vocab_size)
 for word, i in t1.word_index.items() :
embedding_vector = embeddings_glove.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
del embeddings_glove, EMBEDDING_FILE
gc.collect()
print("Embedding Matrix generated in {} seconds.".format(str(time.time() -tick)) )<feature_engineering>
predictions = []
kf = KFold(shuffle=True, random_state=42)
 for fold_id,(train_idx, valid_idx)in enumerate(kf.split(train)) :
model = create_model()
history = model.fit(train.iloc[train_idx], train_target.iloc[train_idx],
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=50,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history.history["loss"]), min(history.history["val_loss"])))
 model.load_weights("model_fold" + str(fold_id)+ ".h5")
 predictions.append(model.predict(test))
tick = time.time()
EMBEDDING_FILE = ".. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec"
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_wiki = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding='utf-8'))
embedding_matrix_wiki = np.random.randn(vocab_size,embedding_dim ).astype(np.float32)* np.sqrt(2.0/vocab_size)
 for word, i in t1.word_index.items() :
embedding_vector = embeddings_wiki.get(word)
 if embedding_vector is not None:
embedding_matrix_wiki[i] = embedding_vector
del embeddings_wiki, EMBEDDING_FILE
gc.collect()
print("Embedding Matrix WIKI generated in {} seconds.".format(str(time.time() -tick)) )<load_pretrained>
pred = np.average(predictions, axis=0)
 pred = np.clip(pred, 0.001, 0.999)
 pred.shape
tick = time.time()
embeddings_google = KeyedVectors.load_word2vec_format('.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)
 print("Load time {} seconds.".format(str(time.time() -tick)) )<data_type_conversions>
sub = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 sub.loc[:, 1:] = pred
sub.loc[ctlVehicle_test, sub.columns != "sig_id"] = 0
sub
<feature_engineering><EOS> 
sub.to_csv("submission.csv", index=False )