Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<string_transform>
import seaborn as sns
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
def compute_input_text(texts, tokenizer, max_sequence_length=512):
input_ids = []
input_masks = []
for text in texts:
text = tokenizer.tokenize(text)
 text_ids = tokenizer.convert_tokens_to_ids(text)
 cls_ids = tokenizer.convert_tokens_to_ids(["[CLS]"])
 sep_ids = tokenizer.convert_tokens_to_ids(["[SEP]"])
 if(len(text_ids)+2)> max_sequence_length:
text_ids = text_ids[:max_sequence_length-2]
token_ids = cls_ids + text_ids + sep_ids
padding = [0] *(max_sequence_length - len(token_ids))
ids = token_ids + padding
masks = [1]*len(token_ids)+ padding
input_ids.append(ids)
 input_masks.append(masks)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long()
]<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_id = test['id']
train.drop(['id'], axis=1, inplace=True)
 test.drop(['id'], axis=1, inplace=True )
def get_bert_feature(tokenizer, model, df, column):
print(column)
 inputs = compute_input_text(df[column].values, tokenizer)
 dataset = QuestBertDataset(inputs, None)
 loader = torch.utils.data.DataLoader(dataset, batch_size=6, shuffle=False, drop_last=False)
 preds_fold = []
model.eval()
with torch.no_grad() :
for ids, masks in tqdm(loader):
ids = ids.cuda()
masks = masks.cuda()
outputs = model(input_ids=ids, attention_mask=masks)
 x = outputs[0][:, 0, :]
preds_fold.extend(x.cpu().data.numpy().tolist())
 preds_fold = np.array(preds_fold)
 return preds_fold<feature_engineering>
bytreatment = train.groupby(['color','type'])
 bytreatment['bone_length'].agg([np.median, np.mean, np.std, len] )
tokenizer = transformers.DistilBertTokenizer.from_pretrained(pretrained_distilbert_base_uncased)
 model = transformers.DistilBertModel.from_pretrained(pretrained_distilbert_base_uncased)
 model.cuda()
print('train:')
 train_question_title_dense = get_bert_feature(tokenizer, model, train_df, 'question_title')
 train_question_body_dense = get_bert_feature(tokenizer, model, train_df, 'question_body')
 train_answer_dense = get_bert_feature(tokenizer, model, train_df, 'answer')
 print('test:')
 test_question_title_dense = get_bert_feature(tokenizer, model, test_df, 'question_title')
 test_question_body_dense = get_bert_feature(tokenizer, model, test_df, 'question_body')
 test_answer_dense = get_bert_feature(tokenizer, model, test_df, 'answer')
 del tokenizer, model<categorify>
train['hair_soul'] = train['hair_length'] * train['has_soul']
train['hair_bone'] = train['hair_length'] * train['bone_length']
train['soul_bone'] = train['has_soul'] * train['bone_length']
test['hair_soul'] = test['hair_length'] * test['has_soul']
test['hair_bone'] = test['hair_length'] * test['bone_length']
test['soul_bone'] = test['has_soul'] * test['bone_length']
embed = hub.load(universal_sentence_encoder_path)
 embeddings_train = {}
embeddings_test = {}
for text in input_columns:
print(text)
 train_text = train_df[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
test_text = test_df[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
curr_train_emb = []
curr_test_emb = []
batch_size = 4
ind = 0
while ind*batch_size < len(train_text):
curr_train_emb.append(embed(train_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
ind = 0
while ind*batch_size < len(test_text):
curr_test_emb.append(embed(test_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
embeddings_train[text + '_embedding'] = np.vstack(curr_train_emb)
 embeddings_test[text + '_embedding'] = np.vstack(curr_test_emb)
 del embed
gc.collect()<create_dataframe>
features = ["bone_length", "rotting_flesh", "hair_length", "has_soul", "hair_soul", "hair_bone", "soul_bone"]
X_Train = pd.concat([train[features], pd.get_dummies(train["color"])], axis=1)
 y_Train = train["type"]
X_Test = pd.concat([test[features], pd.get_dummies(test["color"])], axis=1 )
embeddings_train_df = pd.DataFrame()
embeddings_test_df = pd.DataFrame()
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 abs_dist = lambda x, y: np.abs(x-y ).sum(axis=1)
 sum_dist = lambda x, y:(x+y ).sum(axis=1)
 dist_columns = [
['question_title_embedding', 'answer_embedding'],
['question_body_embedding', 'answer_embedding'],
['question_body_embedding', 'question_title_embedding'],
]
for i, columns in enumerate(dist_columns):
embeddings_train_df[f'l2_dist_embedding_{i}'] = l2_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_train_df[f'cos_dist_embedding_{i}'] = cos_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_train_df[f'abs_dist_embedding_{i}'] = abs_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_train_df[f'l2_dist_embedding_{i}'] = sum_dist(embeddings_train[columns[0]], embeddings_train[columns[1]])
 embeddings_test_df[f'l2_dist_embedding_{i}'] = l2_dist(embeddings_test[columns[0]], embeddings_test[columns[1]])
 embeddings_test_df[f'cos_dist_embedding_{i}'] = cos_dist(embeddings_test[columns[0]], embeddings_test[columns[1]])
 embeddings_test_df[f'abs_dist_embedding_{i}'] = abs_dist(embeddings_test[columns[0]], embeddings_test[columns[1]])
 embeddings_test_df[f'l2_dist_embedding_{i}'] = sum_dist(embeddings_test[columns[0]], embeddings_test[columns[1]] )<prepare_x_and_y>
le_y = LabelEncoder()
y_Train = le_y.fit_transform(y_Train)
 print(X_Train.shape)
 print(y_Train.shape )
x_train = np.hstack([item for k, item in embeddings_train.items() ] + [train_question_title_dense, train_question_body_dense, train_answer_dense, embeddings_train_df.values, train_feature.values])
 x_test = np.hstack([item for k, item in embeddings_test.items() ] + [test_question_title_dense, test_question_body_dense, test_answer_dense, embeddings_test_df.values, test_feature.values])
 y_train = train_df[target_columns].values<drop_column>
X_Train1, X_Test1, y_Train1, y_Test1 = train_test_split(X_Train, y_Train, test_size=0.20, random_state=36)
 print(X_Train1.shape)
 print(y_Train1.shape )
del train_feature, test_feature
del embeddings_train, embeddings_test
del embeddings_train_df, embeddings_test_df
del train_question_title_dense, train_question_body_dense, train_answer_dense
del test_question_title_dense, test_question_body_dense, test_answer_dense<compute_test_metric>
forest = RandomForestClassifier(n_estimators = 20,
criterion = 'entropy',
max_features = 'auto')
 parameter_grid = {
'max_depth' : [None, 5, 20, 100],
'min_samples_split' : [2, 5, 7],
'min_weight_fraction_leaf' : [0.0, 0.1],
'max_leaf_nodes' : [20, 30],
}
grid_search_rf = GridSearchCV(forest, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_rf.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_rf.best_score_))
print('Best parameters: {}'.format(grid_search_rf.best_params_))
def mean_spearmanr_correlation_score(y_true, y_pred):
num_labels = y_pred.shape[1]
return np.nanmean([ spearmanr(y_pred[:, idx], y_true[:, idx] ).correlation for idx in range(num_labels)] )<prepare_x_and_y>
y_test_pred = grid_search_rf.predict(X_Test1)
 y_test_pred
accuracy_score(y_Test1, y_test_pred )
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, x, y):
self.x = x
self.y = y
def __len__(self):
return len(self.x)
 def __getitem__(self, idx):
if self.y is not None:
return self.x[idx], self.y[idx]
else:
return self.x[idx]<choose_model_class>
logreg = LogisticRegression()
parameter_grid = {'solver' : ['newton-cg', 'lbfgs'],
'multi_class' : ['multinomial'],
'C' : [0.005, 0.01, 1, 10],
'tol': [0.0001, 0.001, 0.005, 0.01]
}
grid_search_logit = GridSearchCV(logreg, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_logit.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_logit.best_score_))
print('Best parameters: {}'.format(grid_search_logit.best_params_))
class NeuralNet(nn.Module):
def __init__(self, num_features, num_labels):
super(NeuralNet, self ).__init__()
self.num_features = num_features
self.num_labels = num_labels
self.classifier = nn.Sequential(
 nn.Linear(self.num_features, self.num_features),
nn.Dropout(0.2),
nn.Linear(self.num_features, self.num_features),
nn.Linear(self.num_features, self.num_labels), 
)
 def forward(self, features):
return self.classifier(features )<feature_engineering>
y_test_pred = grid_search_logit.predict(X_Test1)
 y_test_pred
accuracy_score(y_Test1, y_test_pred )
test_pred_datas = {}
test_pred_weights = {}
def init_test_pred_data() :
for column in target_columns:
test_pred_datas[column] = np.zeros(len(test_df))
test_pred_weights[column] = 0.0
def add_test_pred_data(prediction, columns, weight):
for column_idx, column in enumerate(columns):
test_pred_datas[column] += weight * prediction[:, column_idx]
test_pred_weights[column] += weight<feature_engineering>
svr = svm.SVC()
parameter_grid = {'kernel':('linear', 'rbf', 'poly'), 'C':[0.005, 0.01, 1, 10, 100, 1000], 'degree':[2, 3]}
grid_search_svc = GridSearchCV(svr, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_svc.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_svc.best_score_))
print('Best parameters: {}'.format(grid_search_svc.best_params_))
validation_datas = {}
validation_counts = {}
def init_validation_data() :
for column in target_columns:
validation_datas[column] = np.zeros(len(train_df))
validation_counts[column] = np.zeros(len(train_df))
def add_validation_data(prediction, columns, idx):
for column_idx, column in enumerate(columns):
validation_datas[column][idx] += prediction[:, column_idx]
validation_counts[column][idx] += 1.0<train_model>
y_test_pred = grid_search_svc.predict(X_Test1)
 y_test_pred
accuracy_score(y_Test1, y_test_pred )
init_test_pred_data()
init_validation_data()<train_on_grid>
warnings.filterwarnings("ignore")
 nnet = neural_network.MLPClassifier()
parameter_grid = {'activation':['logistic','relu'], 'hidden_layer_sizes':[10,15,25], 'alpha':[1e-4, 1e-3, 1e-2, 1e-1], 'solver':['lbfgs','sgd'], 'learning_rate':['constant', 'adaptive']}
grid_search_nnet = GridSearchCV(nnet, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_nnet.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_nnet.best_score_))
print('Best parameters: {}'.format(grid_search_nnet.best_params_))
n_splits = 5
cv = GroupKFold(n_splits=n_splits)
 kfold_split = list(cv.split(X=train_df.question_body, groups=train_df.question_body))<create_dataframe>
sgd = SGDClassifier()
parameter_grid = {'average':[True],'loss':['log','perceptron','squared_loss'], 'penalty':[None,'l2','l1'], 'alpha':[0.001,0.01, 0.1, 1], 'l1_ratio':[0.15, 0.1, 0.3], 'learning_rate':['optimal'], 'random_state':[0]}
grid_search_sgd = GridSearchCV(sgd, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_sgd.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_sgd.best_score_))
print('Best parameters: {}'.format(grid_search_sgd.best_params_))
n_epochs = 50
patience = 5
scores = []
batch_size = 32
num_features = x_train.shape[1]
num_labels = y_train.shape[1]
for k,(train_idx, valid_idx)in enumerate(kfold_split):
print("k:", k+1)
 x_train_train = x_train[train_idx]
y_train_train = y_train[train_idx]
train_dataset = QuestDataset(x_train_train, y_train_train)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)
 x_train_valid = x_train[valid_idx]
y_train_valid = y_train[valid_idx]
valid_dataset = QuestDataset(x_train_valid, y_train_valid)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False)
 model = NeuralNet(num_features, num_labels)
 model.cuda()
best_weights = copy.deepcopy(model.state_dict())
 loss_fn = nn.BCEWithLogitsLoss()
optimizer = RAdam(model.parameters() , lr=2e-5)
 schduler = CyclicScheduler2(min_lr=2e-6, max_lr=2e-5, period=20, warm_start=0, max_decay=0.9)
 min_loss = np.inf
counter = 0
for epoch in range(n_epochs):
lr = schduler(epoch)
 adjust_learning_rate(optimizer, lr)
 model.train()
train_loss = []
for features, labels in train_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 loss = loss_fn(y_pred, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss.append(loss.item())
 avg_train_loss = np.mean(train_loss)
 model.eval()
val_loss = []
with torch.no_grad() :
for features, labels in valid_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 loss = loss_fn(y_pred, labels)
 val_loss.append(loss.item())
 avg_val_loss = np.mean(val_loss)
 if avg_val_loss < min_loss:
min_loss = avg_val_loss
counter = 0
best_weights = copy.deepcopy(model.state_dict())
 else:
counter += 1
if counter >= patience and epoch > 12:
break
model.load_state_dict(best_weights)
 model.eval()
valid_preds_fold = []
valid_true_fold = []
with torch.no_grad() :
for features, labels in valid_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
labels = torch.tensor(labels,dtype=torch.float32 ).cuda()
y_pred = model(features)
 valid_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 valid_true_fold.extend(labels.cpu().data.numpy().tolist())
 valid_preds_fold = np.array(valid_preds_fold)
 valid_true_fold = np.array(valid_true_fold)
 add_validation_data(valid_preds_fold, target_columns, valid_idx)
 score = mean_spearmanr_correlation_score(valid_preds_fold, valid_true_fold)
 print('Score:', score)
 scores.append(score)
 test_preds_fold = []
test_dataset = QuestDataset(x_test, None)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 with torch.no_grad() :
for features in test_loader:
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(features)
 test_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 test_preds_fold = np.array(test_preds_fold)
 add_test_pred_data(test_preds_fold, target_columns, 1.0)
 del model, optimizer, loss_fn, best_weights
del train_dataset, train_loader
del valid_dataset, valid_loader
del test_dataset, test_loader
torch.cuda.empty_cache()
gc.collect()
print('================')
 print('Mean score:', np.mean(scores))<prepare_output>
bag = ensemble.BaggingClassifier()
parameter_grid = {'base_estimator':[None,DummyClassifier() ,Perceptron() ,DecisionTreeClassifier() ,KNeighborsClassifier() ,SVC() ], "max_samples": [0.5, 1.0], "max_features": [1, 2, 4],"bootstrap": [True, False],"bootstrap_features": [True, False],'random_state':[0]}
grid_search_bag = GridSearchCV(bag, param_grid=parameter_grid, scoring='log_loss', refit='True', cv=StratifiedKFold(3))
grid_search_bag.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_bag.best_score_))
print('Best parameters: {}'.format(grid_search_bag.best_params_))
validation0 = pd.DataFrame()
for column in target_columns:
preds = validation_datas[column]
count = validation_counts[column]
count = np.where(count < 0.5, 1.0, count)
 validation0[column] = preds / count<compute_test_metric>
x=tf.contrib.learn.infer_real_valued_columns_from_input(X_Train1)
 tf_clf_dnn = learn.DNNClassifier(hidden_units=[16], n_classes=3, feature_columns=x, activation_fn=tf.sigmoid)
 tf_clf_dnn.fit(X_Train1, y_Train1, max_steps=5000)
 print(acc_s(y_Test1,tf_clf_dnn.predict(X_Test1)) )
mean_spearmanr_correlation_score(train_df[target_columns].values, validation0.values )<load_from_csv>
clf = tf.contrib.learn.LinearClassifier(
 feature_columns=tf.contrib.learn.infer_real_valued_columns_from_input(X_Train),
n_classes=3,
optimizer=tf.train.AdagradOptimizer(
 learning_rate=0.5, 
))
clf.fit(X_Train, y_Train, steps=500)
 y_pred = clf.predict(X_Test)
 pred = le_y.inverse_transform(y_pred )
test_preds0 = pd.read_csv(f"{folder}sample_submission.csv")
 for column in target_columns:
output = test_pred_datas[column] / test_pred_weights[column]
test_preds0[column] = output<choose_model_class>
submission = pd.DataFrame({'id':test_id,
'type':pred} )
LSTM_UNITS = 512
DENSE_HIDDEN_UNITS = 6 * LSTM_UNITS
class MODEL_v001(nn.Module):
def __init__(self, num_features, num_labels):
super().__init__()
self.lstm1 = nn.LSTM(num_features , LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True,dropout=0.2)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linearnorm = nn.LayerNorm(DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linearnorm2 = nn.LayerNorm(DENSE_HIDDEN_UNITS)
 self.linear_sub_out = nn.Linear(DENSE_HIDDEN_UNITS, num_labels)
 def forward(self, x, lengths=None):
h_lstm1, _ = self.lstm1(x)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool1 = torch.mean(h_lstm1, 1)
 avg_pool2 = torch.mean(h_lstm2, 1)
 max_pool2, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( avg_pool1, max_pool2, avg_pool2), 1)
 h_conc_linear1 = self.linearnorm(F.relu(self.linear1(h_conc)))
 h_conc_linear2 = self.linearnorm2(F.relu(self.linear2(h_conc)))
 hidden = h_conc + h_conc_linear1 + h_conc_linear2
out = self.linear_sub_out(hidden)
 return out<concatenate>
submission.to_csv('submission.csv', index=False )
<train_model><EOS> 
submission.to_csv('submission.csv', index=False )