Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
<SOS> metric: categorizationaccuracy Kaggle data source: ghouls,-goblins,-and-ghosts...-boo!<compute_test_metric>
init_notebook_mode(connected=True)
 color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
print(os.listdir(".. /input"))
def mean_spearmanr_correlation_score(y_true, y_pred):
num_labels = y_pred.shape[1]
score = np.nanmean([ spearmanr(y_pred[:, idx], y_true[:, idx] ).correlation for idx in range(num_labels)])
 return score<prepare_output>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, inputs, features, labels = None):
self.inputs = inputs
self.features = features
if labels is not None:
self.labels = labels
else:
self.labels = None
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
input_segments = self.inputs[2][idx]
input_features = self.features[idx]
if self.labels is not None:
input_labels = self.labels[idx]
return input_ids, input_masks, input_segments, input_features, input_labels
return input_ids, input_masks, input_segments, input_features
def __len__(self):
return len(self.inputs[0] )<create_dataframe>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def model_test_validation(label_columns, train_inputs, test_inputs, x_train, x_test, weight_files):
if len(kfold_split)!= len(weight_files):
return
batch_size = 6
num_features = x_test.shape[1]
num_labels = len(label_columns)
 test_dataset = QuestDataset(test_inputs, x_test, None)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 for k,(train_idx, valid_idx)in enumerate(kfold_split):
fname = weight_files[k]
print(k+1, fname)
 x_train_valid = x_train[valid_idx]
train_inputs_valid = [x[valid_idx] for x in train_inputs]
valid_dataset = QuestDataset(train_inputs_valid, x_train_valid, None)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False)
 model = NeuralNet(num_features, num_labels, pretrained_bert)
 model.to('cuda:0')
 model.load_state_dict(torch.load(fname))
model.eval()
valid_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in valid_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 valid_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 valid_preds_fold = np.array(valid_preds_fold)
 add_validation_data(valid_preds_fold, label_columns, valid_idx)
 test_preds_fold = []
with torch.no_grad() :
for ids, masks, segments, features in test_loader:
ids = ids.cuda()
masks = masks.cuda()
segments = segments.cuda()
features = torch.tensor(features,dtype=torch.float32 ).cuda()
y_pred = model(ids, masks, segments, features)
 test_preds_fold.extend(torch.sigmoid(y_pred ).cpu().data.numpy().tolist())
 test_preds_fold = np.array(test_preds_fold)
 add_test_pred_data(test_preds_fold, label_columns, 1.0)
 del model, valid_dataset, valid_loader
torch.cuda.empty_cache()
gc.collect()
del test_dataset, test_loader<load_pretrained>
missing = train.isnull().sum()
tokenizer = transformers.BertTokenizer.from_pretrained(pretrained_bert )<define_variables>
train_dummy = pd.get_dummies(pd.read_csv('.. /input/train.csv'))
train_dummy.head()
train_inputs = compute_input_title_question(train_df, tokenizer, max_sequence_length=512)
 test_inputs = compute_input_title_question(test_df, tokenizer, max_sequence_length=512)
 x_train = train_feature.values
x_test = test_feature.values
print(x_train.shape)
 print(x_test.shape)
 print(len(target_question_columns))
weight_files = [
model_weight_path1 + 'best_weights_003_55_7_3_question_1.pth',
model_weight_path1 + 'best_weights_003_55_7_3_question_2.pth',
model_weight_path2 + 'best_weights_003_55_7_3_question_3.pth',
model_weight_path2 + 'best_weights_003_55_7_3_question_4.pth',
model_weight_path3 + 'best_weights_003_55_7_3_question_5.pth',
model_weight_path3 + 'best_weights_003_55_7_3_question_6.pth',
]
model_test_validation(target_question_columns, train_inputs, test_inputs, x_train, x_test, weight_files)
 del x_train, x_test
del train_inputs, test_inputs<define_variables>
ghost_num = {"type": {"Ghoul": 1, "Goblin": 2, "Ghost": 3} }
train_inputs = compute_input_arrays(train_df, tokenizer, max_sequence_length=512)
 test_inputs = compute_input_arrays(test_df, tokenizer, max_sequence_length=512)
 x_train = train_feature.values
x_test = test_feature.values
print(x_train.shape)
 print(x_test.shape)
 print(len(target_answer_columns))
weight_files = [
model_weight_path1 + 'best_weights_003_55_7_3_answer_1.pth',
model_weight_path1 + 'best_weights_003_55_7_3_answer_2.pth',
model_weight_path2 + 'best_weights_003_55_7_3_answer_3.pth',
model_weight_path2 + 'best_weights_003_55_7_3_answer_4.pth',
model_weight_path3 + 'best_weights_003_55_7_3_answer_5.pth',
model_weight_path3 + 'best_weights_003_55_7_3_answer_6.pth',
]
model_test_validation(target_answer_columns, train_inputs, test_inputs, x_train, x_test, weight_files)
 del x_train, x_test
del train_inputs, test_inputs 
<drop_column>
ntrain = train.shape[0]
ntest = test.shape[0]
y = train.type.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['type'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
del train_feature, test_feature
del target_question_columns, target_answer_columns<prepare_output>
all_data.drop(['color'], axis=1, inplace=True)
validationT = pd.DataFrame()
for column in target_columns:
preds = validation_datas[column]
count = validation_counts[column]
count = np.where(count < 0.5, 1.0, count)
 validationT[column] = preds / count<compute_test_metric>
all_data["bone_soul"] = all_data["bone_length"]*all_data["has_soul"]
all_data["hair_soul"] = all_data["hair_length"]*all_data["has_soul"]
all_data["flesh_soul"] = all_data["rotting_flesh"]*all_data["has_soul"]
mean_spearmanr_correlation_score(validationT.values, train_df[target_columns].values )<load_from_csv>
all_data["bone_hair"] = all_data["bone_length"]*all_data["hair_length"]
all_data["flesh_hair"] = all_data["rotting_flesh"]*all_data["hair_length"]
test_predsT = pd.read_csv(f"{folder}sample_submission.csv")
 for column in target_columns:
preds = test_pred_datas[column]
weight = test_pred_weights[column]
test_predsT[column] = preds / weight<drop_column>
all_data_simple = pd.DataFrame()
all_data_simple["bone_hair"] = all_data["bone_hair"]
all_data_simple["rotting_flesh"] = all_data["rotting_flesh"]
all_data_simple["bone_soul"] = all_data["bone_soul"]
all_data_simple["hair_soul"] = all_data["hair_soul"]
del target_columns, input_columns
del test_pred_datas, test_pred_weights
del validation_datas, validation_counts<set_options>
train[train['bone_length']>0.8]
torch.cuda.empty_cache()
gc.collect()<define_variables>
X = all_data_simple[:ntrain]
folder = '.. /input/google-quest-challenge/'
pretrained_bert = ".. /input/pretrainedbertpytorch/pretrained-bert-pytorch/gpt2/"
model_weight_path1 = '.. /input/photostage1/best_weights_006-55-gpt2-01_question_1.pth'
model_weight_path2 = '.. /input/photostage2/best_weights_006-55-gpt2-01_question_2.pth'
model_weight_path3 = '.. /input/photostage3/best_weights_006-55-gpt2-01_question_3.pth'
model_weight_path4 = '.. /input/photostage4/best_weights_006-55-gpt2-01_question_4.pth'
model_weight_path5 = '.. /input/photostage5/best_weights_006-55-gpt2-01_question_5.pth'
model_weight_path6 = '.. /input/photostage6/best_weights_006-55-gpt2-01_question_6.pth'
model_weight_path7 = '.. /input/photostagea1/best_weights_006-55-gpt2-01_answer_1.pth'
model_weight_path8 = '.. /input/photostagea2/best_weights_006-55-gpt2-01_answer_2.pth'
model_weight_path9 = '.. /input/photostagea3/best_weights_006-55-gpt2-01_answer_3.pth'
model_weight_path10 = '.. /input/photostagea4/best_weights_006-55-gpt2-01_answer_4.pth'
model_weight_path11 = '.. /input/photostagea5/best_weights_006-55-gpt2-01_answer_5.pth'
model_weight_path12 = '.. /input/photostagea6/best_weights_006-55-gpt2-01_answer_6.pth' 
<import_modules>
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
BertTokenizer, BertModel, BertForSequenceClassification, BertConfig,
WEIGHTS_NAME, CONFIG_NAME, AdamW, get_linear_schedule_with_warmup,
get_cosine_schedule_with_warmup, GPT2Config,GPT2Model,
GPT2LMHeadModel,
GPT2Tokenizer, 
)
<normalization>
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="rbf", C=0.025, probability=True),
NuSVC(probability=True),
DecisionTreeClassifier() ,
RandomForestClassifier() ,
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
GaussianNB() ,
LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ]
log_cols=["Classifier", "Accuracy", "Log Loss"]
log = pd.DataFrame(columns=log_cols)
 for clf in classifiers:
clf.fit(train_X, train_y)
 name = clf.__class__.__name__
print("="*30)
 print(name)
 print('****Results****')
 score = clf.score(val_X, val_y)
 print("Score: {:.4%}".format(score))
print("="*30 )
print("transformers:", transformers.__version__ )<define_variables>
accuracy_scorer = metrics.make_scorer(metrics.accuracy_score)
 params = {'n_estimators':[10, 20, 50, 100], 'criterion':['gini', 'entropy'], 'max_depth':[None, 5, 10, 25, 50]}
rf = RandomForestClassifier(random_state = 0)
 clf = GridSearchCV(rf, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(train_X, train_y)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
SEED = 12345
start_time_all = time.time()<set_options>
rf_best = RandomForestClassifier(criterion= 'entropy', max_depth= 5, n_estimators= 50 )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )<load_pretrained>
params = {'n_estimators':[10, 25, 50, 100], 'max_samples':[1, 3, 5, 10]}
bag = BaggingClassifier(random_state = 0)
 clf = GridSearchCV(bag, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(train_X, train_y)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
def read_pickle_from_file(pickle_file):
with open(pickle_file,'rb')as f:
x = pickle.load(f)
 return x
def write_pickle_to_file(pickle_file, x):
with open(pickle_file, 'wb')as f:
pickle.dump(x, f, pickle.HIGHEST_PROTOCOL)
 def time_to_str(t, mode='min'):
if mode=='min':
t = int(t)/60
hr = t//60
min = t%60
return '%2d hr %02d min'%(hr,min)
 elif mode=='sec':
t = int(t)
 min = t//60
sec = t%60
return '%2d min %02d sec'%(min,sec)
 else:
raise NotImplementedError<load_from_csv>
bag_best = BaggingClassifier(max_samples = 5, n_estimators = 100, random_state = 0 )
train_df = pd.read_csv(f"{folder}train.csv")
 train_df.shape
test_df = pd.read_csv(f"{folder}test.csv")
 test_df.shape<define_variables>
ensemble=VotingClassifier(estimators=[('4', classifiers[4]),('3', classifiers[3]),('5', classifiers[5])],
voting='soft', weights=[1,1,1] ).fit(train_X,train_y)
 print('The accuracy for DecisionTree and Random Forest is:',ensemble.score(val_X,val_y))
target_table = [
['question_asker_intent_understanding', 'question'],
['question_body_critical', 'question'],
['question_conversational', 'question'],
['question_expect_short_answer', 'question'],
['question_fact_seeking', 'question'],
['question_has_commonly_accepted_answer', 'question'],
['question_interestingness_others', 'question'],
['question_interestingness_self', 'question'],
['question_multi_intent', 'question'],
['question_not_really_a_question', 'question'],
['question_opinion_seeking', 'question'],
['question_type_choice', 'question'],
['question_type_compare', 'question'],
['question_type_consequence', 'question'],
['question_type_definition', 'question'],
['question_type_entity', 'question'],
['question_type_instructions', 'question'],
['question_type_procedure', 'question'],
['question_type_reason_explanation', 'question'],
['question_type_spelling', 'question'],
['question_well_written', 'question'],
['answer_helpful', 'answer'],
['answer_level_of_information', 'answer'],
['answer_plausible', 'answer'],
['answer_relevance', 'answer'],
['answer_satisfaction', 'answer'],
['answer_type_instructions', 'answer'],
['answer_type_procedure', 'answer'],
['answer_type_reason_explanation', 'answer'],
['answer_well_written', 'answer'],
]
input_columns = [
'question_title',
'question_body',
'answer'
]<define_variables>
voting_clf = VotingClassifier(estimators=[('rf', rf_best),('bag', bag_best)]
, voting='hard')
 voting_clf.fit(train_X, train_y)
 print('The accuracy for DecisionTree and Random Forest is:',voting_clf.score(val_X,val_y))
target_question_columns = []
target_answer_columns = []
for table in target_table:
if table[1] == 'question':
target_question_columns.append(table[0])
 elif table[1] == 'answer':
target_answer_columns.append(table[0])
 target_columns = target_question_columns + target_answer_columns
print('question:', len(target_question_columns))
print('answer:', len(target_answer_columns))
print('total:', len(target_columns))<feature_engineering>
ghost_cat = {"type": {1: "Ghoul", 2: "Goblin", 3: "Ghost"} }
for df in [train_df, test_df]:
df['domcom'] = df['url'].apply(lambda s: s.split('://')[1].split('/')[0].split('.'))
df['dom_cnt'] = df['domcom'].apply(lambda s: len(s))
df['domcom'] = df['domcom'].apply(lambda s: s + ['none', 'none'])
 for ii in range(0,4):
df['url_'+str(ii)] = df['domcom'].apply(lambda s: s[ii])
 df.drop('domcom', axis = 1, inplace = True)
 df.question_body = df.question_body.apply(html.unescape)
 df.answer = df.answer.apply(html.unescape)
 df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&gt",">"))
df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&lt","<"))
df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&amp","&"))
df['question_body'] = df['question_body'].apply(lambda s: s.replace("\&quot;",<feature_engineering>
test = all_data_simple[ntrain:]
<feature_engineering><EOS> 
sub = pd.DataFrame()
sub['id'] = test_ID
sub['type'] = voting_clf.predict(test)
 sub.replace(ghost_cat, inplace=True)
 sub.to_csv('subvoting_clf.csv',index=False)