Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<groupby>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
from sklearn.model_selection import train_test_split
ages_mean_train = mytrainset['Age'].mean()
ages_mean_train = round(ages_mean_train)
 ages_mean_test = mytestset['Age'].mean()
ages_mean_test = round(ages_mean_test)
<categorify>
class MyModel(nn.Module):
def __init__(self, in_features, out_features):
super().__init__()
self.in_features = in_features
self.out_features = out_features
self.model = nn.Sequential(
 nn.BatchNorm1d(in_features),
nn.Dropout(0.5),
nn.Linear(in_features, 2048),
nn.ReLU() ,
nn.BatchNorm1d(2048),
nn.Dropout(0.5),
nn.Linear(2048, 1024),
nn.ReLU() ,
nn.BatchNorm1d(1024),
nn.Dropout(0.5),
nn.Linear(1024, out_features),
nn.Sigmoid() 
)
 def forward(self, x):
return self.model(x )
mytrainset['Age'] = mytrainset['Age'].replace(np.nan, ages_mean_train)
 mytrainset['Age'] = mytrainset['Age'].replace(np.nan, ages_mean_train)
 mytrainset = mytrainset.drop(["Cabin"], axis=1)
 mytrainset = mytrainset.drop(["Name"], axis=1)
 mytrainset = mytrainset.drop(["Ticket"], axis=1)
 mytrainset['Embarked'].replace({'C':1, 'Q':2, 'S':3},inplace=True)
 mytrainset['Embarked'] = mytrainset['Embarked'].fillna(0)
 mytrainset['Sex'].replace({'male':0, 'female':1},inplace=True)
 mytestset['Age'] = mytestset['Age'].replace(np.nan, ages_mean_test)
 mytestset = mytestset.drop(["Cabin"], axis=1)
 mytestset = mytestset.drop(["Name"], axis=1)
 mytestset = mytestset.drop(["Ticket"], axis=1)
 mytestset['Embarked'].replace({'C':1, 'Q':2, 'S':3},inplace=True)
 mytestset['Sex'].replace({'male':0, 'female':1},inplace=True)
 mytestset['Fare'] = mytestset['Fare'].fillna(mytestset['Fare'].mean())
<split>
def train_model(model, train_dataloader, test_dataloader=None, optimizer=None, scheduler=None, max_epoch=10, criterion=None):
if optimizer is None:
optimizer = torch.optim.Adam(model.parameters())
 if scheduler is None:
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 0.1, 2)
 if criterion is None:
criterion = nn.BCELoss()
for epoch in range(max_epoch):
model.train()
for i,(x_batch, y_batch)in enumerate(train_dataloader):
preds = model(x_batch)
 optimizer.zero_grad()
loss = criterion(preds, y_batch)
 loss.backward()
optimizer.step()
scheduler.step()
if i % 20 == 0:
print(f'Epoch: {epoch}, train loss: {loss.item() :12.5f}')
 model.eval()
with torch.no_grad() :
train, y_train = train_dataloader.dataset.tensors
train_preds = model(train)
 train_loss = criterion(train_preds, y_train ).item()
if test_dataloader is not None:
test, y_test = test_dataloader.dataset.tensors
test_preds = model(test)
 test_loss = criterion(test_preds, y_test ).item()
print(f'Epoch: {epoch} final', f'test loss: {test_loss}', f'train loss: {train_loss}')
 else:
print(f'Epoch: {epoch} final: train loss: {train_loss}' )
y = mytrainset['Survived']
features = ["Pclass", "Age", "Sex", "SibSp", "Parch", "Embarked"]
X = mytrainset[features]
X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.3, random_state=1 )<train_model>
def train_ensemble(X, y, base_model=MyModel, n_models=5, max_epoch=50):
criterion = nn.BCELoss()
kf = KFold(n_splits=n_models, shuffle=True, random_state=48)
 models = []
for i,(train_ind, val_ind)in enumerate(kf.split(X, y)) :
print(f'Training model {i+1}/{kf.n_splits}')
 x_train = torch.tensor(X.iloc[train_ind,:].values, dtype=torch.float32)
 y_train = torch.tensor(y.iloc[train_ind,:].values, dtype=torch.float32)
 x_val = torch.tensor(X.iloc[val_ind,:].values, dtype=torch.float32)
 y_val = torch.tensor(y.iloc[val_ind,:].values, dtype=torch.float32)
 train_dataloader = DataLoader(TensorDataset(x_train, y_train), batch_size=128, shuffle=True)
 val_dataloader = DataLoader(TensorDataset(x_val, y_val), batch_size=128)
 best_loss = np.inf
best_model = None
cur_model = base_model(x_train.shape[1], y_train.shape[1] ).to('cuda')
 optimizer = torch.optim.Adam(cur_model.parameters() ,weight_decay=1e-5)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4)
 for epoch in range(max_epoch):
cur_model.train()
for i,(x_batch, y_batch)in enumerate(train_dataloader):
preds = cur_model(x_batch.to('cuda'))
optimizer.zero_grad()
loss = criterion(preds, y_batch.to('cuda'))
loss.backward()
optimizer.step()
cur_model.eval()
with torch.no_grad() :
train_preds = cur_model(x_train.to('cuda'))
train_loss = criterion(train_preds, y_train.to('cuda')).item()
val_preds = cur_model(x_val.to('cuda'))
val_loss = criterion(val_preds, y_val.to('cuda')).item()
print(f'Epoch {epoch}: train loss: {train_loss}, val loss: {val_loss}')
 scheduler.step(val_loss)
 if val_loss < best_loss:
best_model = deepcopy(cur_model)
 best_loss = val_loss
models += [best_model]
return models
model= RandomForestClassifier(n_estimators=200,max_depth=5, random_state=1)
 model = model.fit(X_train, Y_train)
 predictions = model.predict(X_test )<compute_test_metric>
best_features = ['g-392', 'c-65', 'g-100', 'c-9', 'g-50', 'c-79', 'c-98', 'g-37',
'c-6', 'c-26', 'g-439', 'g-628', 'g-744', 'g-351', 'g-298', 'c-42',
'g-410', 'g-761', 'c-18', 'g-418', 'g-146', 'c-57', 'c-64',
'g-322', 'c-48', 'c-38', 'c-82', 'g-63', 'g-534', 'c-92', 'g-186',
'c-28', 'g-486', 'c-70', 'g-672', 'g-91', 'g-731', 'c-10', 'g-386',
'g-121', 'g-443', 'g-206', 'g-723', 'c-81', 'c-36', 'c-33', 'g-85',
'g-235', 'g-406', 'g-683', 'c-52', 'c-62', 'c-63', 'g-365', 'c-21',
'c-60', 'c-15', 'c-66', 'g-629', 'c-49', 'g-248', 'c-59', 'c-24',
'c-76', 'g-669', 'g-106', 'g-38', 'g-140', 'c-30', 'c-22', 'g-72',
'c-25', 'c-23', 'c-8', 'c-83', 'g-489', 'g-369', 'c-47', 'g-158',
'g-297', 'g-147', 'c-5', 'c-77', 'g-163', 'g-332', 'g-344', 'c-50',
'g-335', 'c-2', 'g-503', 'g-208', 'g-152', 'c-17', 'c-41', 'g-353',
'c-34', 'g-664', 'c-96', 'g-228', 'c-67', 'g-569', 'g-750', 'g-30',
'g-578', 'c-90', 'c-72', 'g-257', 'c-75', 'c-97', 'g-98', 'g-500',
'c-1', 'g-728', 'c-44', 'g-360', 'c-85', 'g-195', 'c-31', 'c-11',
'c-40', 'g-135', 'g-65', 'c-95', 'c-80', 'g-261', 'g-590', 'c-54',
'c-51', 'c-13', 'c-12', 'g-201', 'g-83', 'g-468', 'g-58', 'g-478',
'g-460', 'g-574', 'c-45', 'c-94', 'c-4', 'g-367', 'c-69', 'g-407',
'c-73', 'g-349', 'g-155', 'g-113', 'g-350', 'c-91', 'g-546',
'g-131', 'g-52', 'g-745', 'c-55', 'c-27', 'c-14', 'g-379', 'g-51',
'g-199', 'g-241', 'g-568', 'g-10', 'c-93', 'g-508', 'c-84', 'c-78',
'g-433', 'c-20', 'c-39', 'g-7', 'g-177', 'g-185']
def preprocess(features, target=None, is_train=True):
out_features = features.drop(['sig_id'], axis=1 ).copy()
out_features['cp_type'] =(features.cp_dose == 'trt_cp' ).astype(float)
 out_features['cp_dose'] =(features.cp_dose == 'D1' ).astype(float)
 out_features['cp_time'] =(features.cp_time == 48 ).astype(float)
 out_target = None
if target is not None:
out_target = target.drop('sig_id', axis=1 ).copy()
return out_features, out_target
accuracy_score(predictions, Y_test)
<compute_train_metric>
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir+'train_features.csv')
 test_features = pd.read_csv(data_dir+'test_features.csv')
 train_target = pd.read_csv(data_dir+'train_targets_scored.csv')
gnb = GaussianNB()
NB_model_predictions = gnb.fit(X_train, Y_train ).predict(X_test)
 accuracy_score(NB_model_predictions, Y_test)
<train_model>
x_train, y_train = preprocess(train_features, train_target)
 test_zero_mask = test_features.cp_type == 'ctl_vehicle'
x_test, _ = preprocess(test_features, is_train=False )
decisiontreeModel = DecisionTreeClassifier()
decisiontreeModel = decisiontreeModel.fit(X_train, Y_train)
 decisiontreePredicition = decisiontreeModel.predict(X_test)
 accuracy_score(decisiontreePredicition, Y_test)
<train_model>
class Autoencoder(nn.Module):
def __init__(self, n_inputs, dropout=0.1):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(n_inputs),
nn.Dropout(dropout),
nn.Linear(n_inputs, n_inputs // 2),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 2),
nn.Dropout(dropout),
nn.Linear(n_inputs // 2, n_inputs // 4),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 4),
nn.Dropout(dropout),
nn.Linear(n_inputs // 4, n_inputs // 8),
nn.ReLU() 
)
 self.decoder = nn.Sequential(
 nn.Linear(n_inputs // 8, n_inputs // 4),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 4),
nn.Dropout(dropout),
nn.Linear(n_inputs // 4, n_inputs // 2),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 2),
nn.Dropout(dropout),
nn.Linear(n_inputs // 2, n_inputs)
)
 def forward(self, x):
encoded = self.encoder(x)
 decoded = self.decoder(encoded)
 return decoded
knn = KNeighborsClassifier(n_neighbors=5, metric='euclidean')
 knn.fit(X_train, Y_train)
 knnPredictions = knn.predict(X_test)
 accuracy_score(Y_test, knnPredictions)
<train_model>
gen_f = [col for col in x_train.columns if col.startswith('g-')]
cell_f = [col for col in x_train.columns if col.startswith('c-')]
LRModel = LogisticRegression(max_iter = 200)
 LRModel.fit(X_train, Y_train)
 LRModel_Prediction = LRModel.predict(X_test)
 accuracy_score(Y_test, LRModel_Prediction)
<compute_train_metric>
x_train_cell, x_test_cell = train_test_split(x_train[cell_f], test_size = 0.2, random_state=111 )
cv = KFold(n_splits=10, random_state=1, shuffle=True)
 scores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, n_jobs=-1)
 print("%0.2f accuracy" %(scores.mean()))<compute_train_metric>
train_dataset = TensorDataset(torch.tensor(x_train_cell.values, dtype=torch.float32),
torch.tensor(x_train_cell.values, dtype=torch.float32))
test_dataset = TensorDataset(torch.tensor(x_test_cell.values, dtype=torch.float32),
torch.tensor(x_test_cell.values, dtype=torch.float32))
train_dataloader = DataLoader(train_dataset, batch_size=256)
 test_dataloader = DataLoader(test_dataset, batch_size=256 )
scores = cross_val_score(gnb, X, y, scoring='accuracy', cv=cv, n_jobs=-1)
 scores
print("%0.2f accuracy" %(scores.mean()))<compute_train_metric>
cells_autoencoder = Autoencoder(len(cell_f))
optimizer = torch.optim.Adam(cells_autoencoder.parameters() , weight_decay=1e-3)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4)
 train_model(cells_autoencoder, train_dataloader, test_dataloader, max_epoch=30, criterion=nn.MSELoss() )
scores = cross_val_score(decisiontreeModel, X, y, scoring='accuracy', cv=cv, n_jobs=-1)
 scores
print("%0.2f accuracy" %(scores.mean()))<compute_train_metric>
with torch.no_grad() :
ae_cells_train = cells_autoencoder.encoder(torch.tensor(x_train[cell_f].values, dtype=torch.float32)).numpy()
scores = cross_val_score(knn, X, y, scoring='accuracy', cv=cv, n_jobs=-1)
 scores
print("%0.2f accuracy" %(scores.mean()))<compute_train_metric>
x_train_gen, x_test_gen = train_test_split(x_train[gen_f], test_size = 0.2, random_state=111 )
scores = cross_val_score(LRModel, X, y, scoring='accuracy', cv=cv, n_jobs=-1)
 scores
print("%0.2f accuracy" %(scores.mean()))<predict_on_test>
train_dataset = TensorDataset(torch.tensor(x_train_gen.values, dtype=torch.float32),
torch.tensor(x_train_gen.values, dtype=torch.float32))
test_dataset = TensorDataset(torch.tensor(x_test_gen.values, dtype=torch.float32),
torch.tensor(x_test_gen.values, dtype=torch.float32))
train_dataloader = DataLoader(train_dataset, batch_size=256)
 test_dataloader = DataLoader(test_dataset, batch_size=256 )
testing = mytestset[features]
testPredicitons = model.predict(testing )<save_to_csv>
gen_autoencoder = Autoencoder(len(gen_f))
optimizer = torch.optim.Adam(gen_autoencoder.parameters() , weight_decay=1e-3)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4)
 train_model(gen_autoencoder, train_dataloader, test_dataloader, max_epoch=30, criterion=nn.MSELoss() )
testPredicitons = {'PassengerId':mytestset["PassengerId"], "Survived": testPredicitons}
pd.DataFrame(testPredicitons ).to_csv("predictions.csv", index = False )<set_options>
with torch.no_grad() :
ae_gen_train = gen_autoencoder.encoder(torch.tensor(train_features[gen_f].values, dtype=torch.float32)).numpy()
%matplotlib inline<load_from_csv>
train_enc = pd.DataFrame(np.concatenate(( ae_gen_train, ae_cells_train),axis=1))
train_enc = pd.concat(( x_train, train_enc), axis=1 )
train = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
 test = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
 sample_submission = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/sample_submission.csv" )<correct_missing_values>
ensemble = train_ensemble(train_enc, y_train )
print(train[train["text"].isnull() == True])
 train.dropna(inplace = True )<feature_engineering>
np.save('best_ensemble_pca.npy', ensemble )
def len_sent(data):
return len(data.split())
 train["num_words_text"] = train["text"].apply(lambda x : len_sent(x))
test["num_words_text"] = test["text"].apply(lambda x : len_sent(x))
train["num_words_selected_text"] = train["selected_text"].apply(lambda x : len_sent(x))<feature_engineering>
ensemble = np.load('best_ensemble_pca.npy', allow_pickle=True )
train["text"] = train["text"].apply(lambda x : x.strip())
 train["selected_text"] = train["selected_text"].apply(lambda x : x.strip())
 train["is_equal"] =(train["text"] == train["selected_text"])
 df_neutral = train[train["sentiment"] == "neutral"]
percentage =(len(df_neutral[df_neutral["is_equal"] == True])/len(df_neutral)) * 100
print(f"Percentage of text column sentences is equal selected_text column for neutral sentiment --> {percentage}" )<feature_engineering>
with torch.no_grad() :
ae_gen_test= gen_autoencoder.encoder(torch.tensor(x_test[gen_f].values, dtype=torch.float32)).numpy()
ae_cells_test = cells_autoencoder.encoder(torch.tensor(x_test[cell_f].values, dtype=torch.float32)).numpy()
def punc_count(data):
return len([w for w in data if w in string.punctuation])
 train["text_punc_count"] = train["text"].apply(lambda x : punc_count(x))
train["selected_text_punc_count"] = train["selected_text"].apply(lambda x : punc_count(x))<filter>
test_enc = pd.DataFrame(np.concatenate(( ae_gen_test, ae_cells_test),axis=1))
test_enc = pd.concat(( x_test, test_enc), axis=1 )
word_cloud(train[train["sentiment"] == "neutral"]["text"], "Most Repeated words in neural sentences train['text']" )<filter>
preds = []
for model in ensemble:
model.eval()
with torch.no_grad() :
preds += [model(torch.tensor(test_enc.values, dtype=torch.float32, device='cuda')) ]
preds = torch.stack(preds, dim=0 ).mean(dim=0 )
word_cloud(train[train["sentiment"] == "negative"]["text"], "Most Repeated words in negative sentences train['text']" )<filter>
preds[test_zero_mask] = 0 .
word_cloud(train[train["sentiment"] == "positive"]["text"], "Most Repeated words in positive sentences train['text']" )<string_transform>
sample_submission = pd.read_csv(data_dir + 'sample_submission.csv' )
def n_grams(ngram, data):
freq_dict = defaultdict(int)
 for text in data:
tokens = [w for w in text.lower().split() if w != " " if w not in stopwords]
ngrams = zip(*[tokens[i:] for i in range(ngram)])
 list_grams = [" ".join(ngram)for ngram in ngrams]
for word in list_grams:
freq_dict[word] += 1
fd_sorted = pd.DataFrame(sorted(freq_dict.items() , key=lambda x: x[1])[::-1])
 fd_sorted.columns = ["word", "wordcount"]
return fd_sorted
fd_sorted_neutral1 = n_grams(1, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative1 = n_grams(1, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive1 = n_grams(1, train[train["sentiment"] == "positive"]["text"])
 fd_sorted_neutral2 = n_grams(2, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative2 = n_grams(2, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive2 = n_grams(2, train[train["sentiment"] == "positive"]["text"])
 fd_sorted_neutral3 = n_grams(3, train[train["sentiment"] == "neutral"]["text"])
 fd_sorted_negative3 = n_grams(3, train[train["sentiment"] == "negative"]["text"])
 fd_sorted_positive3 = n_grams(3, train[train["sentiment"] == "positive"]["text"] )<split>
sample_submission.iloc[:, 1:] = preds.cpu()
<sort_values><EOS> 
sample_submission.to_csv('submission.csv', index=False )