Kaggle competition: Ghouls, Goblins, and Ghosts... Boo!
data_dir = '.. /input/google-quest-challenge/'
train = pd.read_csv(path_join(data_dir, 'train.csv'))
test = pd.read_csv(path_join(data_dir, 'test.csv'))
sample = pd.read_csv(path_join(data_dir, 'sample_submission.csv'))
print(train.shape, test.shape)
 train.head()<define_variables>
%matplotlib inline
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]<compute_test_metric>
train_data = pd.read_csv('.. /input/ghouls-goblins-and-ghosts-boo/train.csv.zip')
 test_data = pd.read_csv('.. /input/ghouls-goblins-and-ghosts-boo/test.csv.zip' )
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos )<compute_train_metric>
train_data.isnull().sum()
class OptimizedRounder(object):
def __init__(self,correlation):
self.correlation = correlation
self.coef_ = 0
self.score = 0
def _kappa_loss(self, coef, X, y):
a= X.copy()
b=y.copy()
X_p = pd.cut(a, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0,1,2])
 a[X_p == 0] = 0
a[X_p == 2] = 1
if spearmanr(a, b ).correlation < self.correlation:
self.score = spearmanr(a, b ).correlation
return - spearmanr(a, b ).correlation +(self.correlation - spearmanr(a, b ).correlation + 1)**10
else:
self.score = spearmanr(a, b ).correlation
return - spearmanr(a, b ).correlation
def fit(self, X, y,coef_ini):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = coef_ini
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def coefficients(self):
return self.coef_['x']
def score_fc(self):
return self.score<compute_test_metric>
X = train_data.drop(['id','type'],axis=1)
 y = train_data['type']
def optimize_preds(y_df, x_df, preds_df):
for title in targets:
y1 = np.asarray(y_df[title].copy())
 X1 = np.asarray(x_df[title].copy())
 x_original = np.asarray(x_df[title].copy())
 preds_original = np.asarray(preds_df[title].copy())
 correlation = spearmanr(y1, X1 ).correlation
print(title)
 print(correlation)
 coefficients_target = [0,0]
correlation = spearmanr(y1, X1 ).correlation
okcor = spearmanr(y1, X1 ).correlation
maxi = correlation
liste = [[0.1,0.9],[0.2,0.9],[0.3,0.9],[0.3,0.8],[0.2,0.7],[0.3,0.7],[0.2,0.6],[0.5,0.9],[0.3,0.6],[0.4,0.8],[0.7,0.9],[0.8,0.9]]
for L in liste:
optR = OptimizedRounder(correlation)
 optR.fit(X1,y1,L)
 coefficients = optR.coefficients()
if optR.score_fc() > maxi:
maxi = optR.score_fc()
coefficients_target = coefficients
if maxi != spearmanr(y1, X1 ).correlation:
oof = X1.copy()
oof[oof > coefficients_target[1]] = 1
oof[oof <= coefficients_target[0]] = 0
X1 = np.asarray(preds_df[title].copy())
 oof_test = X1.copy()
oof_test[oof_test > coefficients_target[1]] = 1
oof_test[oof_test <= coefficients_target[0]] = 0
score = spearmanr(y1, oof ).correlation
if score - okcor > 0.001:
print("difference validation score = {}".format(score - okcor))
x_df[title] = oof
dist = Counter(x_df[title])
 if 0 in list(dist.keys()):
dist[0] /= len(x_df)
 if 1 in list(dist.keys()):
dist[1] /= len(x_df)
 acum = 0
bound = {}
if 0 in list(dist.keys()):
acum = dist[0]
bound[0] = np.percentile(preds_df[title], acum * 100)
 if 1 in list(dist.keys()):
acum = 1 - dist[1]
bound[1] = np.percentile(preds_df[title], acum * 100)
 def classify(x):
if 0 in list(dist.keys()):
if 1 in list(dist.keys()):
if x <= bound[0]:
return 0
elif x <= bound[1]:
return x
else:
return 1
else:
if x <= bound[0]:
return 0
else:
return x
else:
if 1 in list(dist.keys()):
if x <= bound[1]:
return x
else:
return 1
else:
return x
final_pred = np.array(list(map(classify, preds_df[title])))
 if len(np.unique(oof_test)) != 1:
print(coefficients_target)
 preds_df[title] = final_pred<load_from_csv>
X = pd.get_dummies(X,columns=['color'])
 X
validationN=pd.read_csv('.. /input/nirjharbert003/validation_preds_df_003.csv')
 test_predsN=pd.read_csv('.. /input/nirjharbert003/submission_003.csv')
 print("Nirjhar model : ")
 compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validationN[targets].copy())) 
<load_from_csv>
y[y=='Ghoul'] = 0
y[y=='Goblin'] = 1
y[y=='Ghost'] = 2
validationN1=pd.read_csv('.. /input/dataset-nirjhar/validation_preds_df_002.csv')
 test_predsN1=pd.read_csv('.. /input/dataset-nirjhar/submission_002.csv')
 print("Nirjhar model : ")
 compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validationN1[targets].copy())) 
<create_dataframe>
y = y.astype('int' )
data_table = []
data_table.append({'Name': 'QUEST-003-55', 'valid': copy.deepcopy(validationT), 'test_preds': copy.deepcopy(test_predsT)})
 data_table.append({'Name': 'QUEST-006-03-GPT2', 'valid': copy.deepcopy(validationTG), 'test_preds': copy.deepcopy(test_predsTG)})
 data_table.append({'Name': 'QUEST-005-07-distilbert', 'valid': copy.deepcopy(validation0), 'test_preds': copy.deepcopy(test_preds0)})
 data_table.append({'Name': 'QUEST-005-07-lstm', 'valid': copy.deepcopy(validationLSTM), 'test_preds': copy.deepcopy(test_predsLSTM)})
 data_table.append({'Name': 'QUEST-Bert-Nirjhar', 'valid': copy.deepcopy(validationN), 'test_preds': copy.deepcopy(test_predsN)})
 data_table.append({'Name': 'QUEST-Bert-Nirjhar1', 'valid': copy.deepcopy(validationN1), 'test_preds': copy.deepcopy(test_predsN1)} )<feature_engineering>
cret = pd.concat([X,y],axis=1)
 cret
n_original = len(data_table)
 score_list = []
for i in range(n_original):
print('=========================')
 print(data_table[i]['Name'])
 print('=========================')
 data_table[i]['valid'] = pd.DataFrame(data_table[i]['valid'] ).rank() / len(data_table[i]['valid'])
 data_table[i]['test_preds'] = pd.DataFrame(data_table[i]['test_preds'] ).rank() / len(data_table[i]['test_preds'])
 data_table.append(copy.deepcopy(data_table[i]))
score1 = compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(data_table[i]['valid'].copy()))
optimize_preds(train, data_table[i]['valid'], data_table[i]['test_preds'])
 score2 = compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(data_table[i]['valid'].copy()))
score_list.append([score1, score2] )<compute_test_metric>
color_ = preprocessing.LabelEncoder()
color_.fit(train_data['color'])
 train_data['color_int'] = color_.transform(train_data['color'])
 sns.pairplot(train_data.drop('color', axis = 1), hue = 'type')
 train_data.drop('color_int', axis = 1, inplace = True)
for i in range(len(score_list)) :
score1, score2 = score_list[i][0], score_list[i][1]
print('{:.6f} -> {:.6f}({:.6f}): {}'.format(score1, score2, score2 - score1, data_table[i]['Name']))<define_variables>
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=77,test_size=0.3 )
validation_totale = train[targets].copy()
test_preds_totale = sample.copy()<categorify>
scaler = MinMaxScaler()
X_test = scaler.fit_transform(X_test)
 X_train = scaler.fit_transform(X_train )
for title in targets:
y1 = np.asarray(train[title].copy())
 X_list = [ np.asarray(x['valid'][title].copy())for x in data_table ]
test_list = [ np.asarray(x['test_preds'][title].copy())for x in data_table ]
corr_list = [ spearmanr(y1, x ).correlation for x in X_list ]
i = np.argmax(corr_list)
 corr_best = copy.deepcopy(corr_list[i])
 x_best = copy.deepcopy(X_list[i])
 test_best = copy.deepcopy(test_list[i])
 for weight_range in np.arange(0.45, 0.95, 0.005):
corr_list2 = [c*c for c in corr_list]
cmin = np.min(corr_list2)
 cmax = np.max(corr_list2)
 weights =(corr_list2 - cmin)/(cmax - cmin)
 weights *= weight_range
weights +=(1.0 - weight_range)
 x = np.average(X_list, axis=0, weights=weights)
 x = rankdata(x)/ len(x)
 corr = spearmanr(y1, x ).correlation
if corr > corr_best:
corr_best = copy.deepcopy(corr)
 x_best = copy.deepcopy(x)
 t = np.average(test_list, axis=0, weights=weights)
 t = rankdata(t)/ len(t)
 test_best = copy.deepcopy(t)
 validation_totale[title] = copy.deepcopy(x_best)
 test_preds_totale[title] = copy.deepcopy(test_best )<statistical_test>
clflog = LogisticRegression(random_state=1)
 clfdt = DecisionTreeClassifier(random_state=1)
 clfrf = RandomForestClassifier()
clfgnb = GaussianNB()
eclf_h =VotingClassifier(estimators = [('log',clflog),('dt',clfdt),('rf',clfrf),('gnb',clfgnb)],voting='hard')
 eclf_s =VotingClassifier(estimators = [('log',clflog),('dt',clfdt),('rf',clfrf),('gnb',clfgnb)],voting='soft')
 models = [clflog,clfdt,clfrf,clfgnb,eclf_h,eclf_s]
compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validation_totale[targets].copy()))<feature_engineering>
for model in models:
model.fit(X_train,y_train)
 predictions = model.predict(X_test)
 print(classification_report(y_test,predictions))
print("%s"%model )
def function_spelling(row):
if row in 'CULTURE':
return 1
else:
return 0
validation_totale['question_type_spelling'] = train['category'].apply(function_spelling)*validation_totale['question_type_spelling']<statistical_test>
models_ = [clflog,clfdt,clfrf,clfgnb]
for model in models_:
pred = model.predict(X_test)
 print("%s"%model)
 print(confusion_matrix(y_test,pred))
compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validation_totale[targets].copy()))<load_from_csv>
c_params = [0.001,0.01,0.1, 1.0, 10.0, 100.0]
params = {
'solver':['newton-cg','lbfgs','liblinear','saga','sag'],
'penalty':['l1','l2','elasticnet'],
'C':c_params,
'max_iter':[1,10,100,1000],
'multi_class':['auto','ovr','multinomial']
}
grid = GridSearchCV(LogisticRegression() ,param_grid=params,refit=True,verbose=True,cv=5);
grid = grid.fit(X_train,y_train);
submission = pd.read_csv(path_join(data_dir, 'sample_submission.csv'))
for title in targets:
for i, row in test_preds_totale.iterrows() :
submission.loc[submission['qa_id'] == row['qa_id'], title] = row[title] 
<feature_engineering>
grid.best_params_
submission['question_type_spelling'] = test['category'].apply(function_spelling)*submission['question_type_spelling'] 
<statistical_test>
lr = LogisticRegression(C=1.0,max_iter=10,multi_class='multinomial',penalty='l2',solver='sag')
 lr.fit(X_train,y_train)
 predictions = lr.predict(X_test)
 print(classification_report(y_test,predictions))
compute_spearmanr_ignore_nan(np.asarray(train[targets].copy()), np.asarray(validation_totale[targets].copy()))<feature_engineering>
id_ = test_data['id']
test_data = test_data.drop(['id'],axis=1 )
for title in targets:
submission[title] = submission[title].apply(lambda x: 1 if x>1 else x )<save_to_csv>
test_data = pd.get_dummies(test_data, columns = ['color'] )
submission.to_csv("submission.csv", index = False)
 submission.head()<install_modules>
result = pd.DataFrame(lr.predict(test_data))
result.columns=['type']
!pip install.. /input/sacremoses/sacremoses-master/
!pip install.. /input/transformers/transformers-master/<define_variables>
result[result['type']==0]='Ghoul'
result[result['type']==1]='Goblin'
result[result['type']==2]='Ghost'
result
DATA_DIR = '.. /input/google-quest-challenge'<load_from_csv>
result = pd.concat([id_,result],axis=1 )
sub = pd.read_csv(f'{DATA_DIR}/sample_submission.csv')
 sub.head()<load_from_csv>
result.to_csv('result.csv',index=False )