Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<save_model>
color = sns.color_palette()
%matplotlib inline
eng_stopwords = set(stopwords.words("english"))
pd.options.mode.chained_assignment = None
learner.save('stage1' )<train_model>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Number of rows in train dataset : ",train_df.shape[0])
 print("Number of rows in test dataset : ",test_df.shape[0] )
learner.unfreeze()
learner.fit_one_cycle(8, slice(1e-5,1e-4))<save_model>
grouped_df = train_df.groupby('author')
 for name, group in grouped_df:
print("Author name : ", name)
 cnt = 0
for ind, row in group.iterrows() :
print(row["text"])
 cnt += 1
if cnt == 5:
break
print("
" )
learner.save('MnistStage2' )<predict_on_test>
train_df["num_words"] = train_df["text"].apply(lambda x: len(str(x ).split()))
test_df["num_words"] = test_df["text"].apply(lambda x: len(str(x ).split()))
train_df["num_unique_words"] = train_df["text"].apply(lambda x: len(set(str(x ).split())))
 test_df["num_unique_words"] = test_df["text"].apply(lambda x: len(set(str(x ).split())))
 train_df["num_chars"] = train_df["text"].apply(lambda x: len(str(x)))
 test_df["num_chars"] = test_df["text"].apply(lambda x: len(str(x)))
 train_df["num_stopwords"] = train_df["text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
test_df["num_stopwords"] = test_df["text"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
train_df["num_punctuations"] =train_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test_df["num_punctuations"] =test_df['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train_df["num_words_upper"] = train_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
test_df["num_words_upper"] = test_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
train_df["num_words_title"] = train_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
test_df["num_words_title"] = test_df["text"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
pred, y, losses = learner.get_preds(ds_type=DatasetType.Test, with_loss=True )<define_variables>
author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}
train_y = train_df['author'].map(author_mapping_dict)
 train_id = train_df['id'].values
test_id = test_df['id'].values
train_df["num_words"] = train_df["text"].apply(lambda x: len(str(x ).split()))
test_df["num_words"] = test_df["text"].apply(lambda x: len(str(x ).split()))
train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
cols_to_drop = ['id', 'text']
train_X = train_df.drop(cols_to_drop+['author'], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1 )
labels = torch.argmax(pred, dim=1 )<prepare_output>
def runXGB(train_X, train_y, test_X, test_y=None, test_X2=None, seed_val=0, child=1, colsample=0.3):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 3
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = child
param['subsample'] = 0.8
param['colsample_bytree'] = colsample
param['seed'] = seed_val
num_rounds = 2000
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=50, verbose_eval=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest, ntree_limit = model.best_ntree_limit)
 if test_X2 is not None:
xgtest2 = xgb.DMatrix(test_X2)
 pred_test_y2 = model.predict(xgtest2, ntree_limit = model.best_ntree_limit)
 return pred_test_y, pred_test_y2, model
submission_df = pd.DataFrame({'ImageId': range(1, len(y)+ 1), 'Label': labels}, columns=['ImageId', 'Label'])
 submission_df.head()<save_to_csv>
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runXGB(dev_X, dev_y, val_X, val_y, test_X, seed_val=0)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
break
print("cv scores : ", cv_scores )
submission_df.to_csv('MnistSubmission.csv', index=False )<set_options>
tfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1,3))
full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist() )
%matplotlib inline
np.random.seed(2)
 print(os.listdir(".. /input")) 
<load_from_csv>
def runMNB(train_X, train_y, test_X, test_y, test_X2):
model = naive_bayes.MultinomialNB()
model.fit(train_X, train_y)
 pred_test_y = model.predict_proba(test_X)
 pred_test_y2 = model.predict_proba(test_X2)
 return pred_test_y, pred_test_y2, model
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5 .
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<feature_engineering>
n_comp = 20
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_word_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
X_train = X_train / 255.0
test = test / 255.0<train_model>
tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,3))
tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist() )
X_train = X_train.values.reshape(-1, 28, 28, 1)
 test = test.values.reshape(-1, 28, 28, 1 )<categorify>
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2]
Y_train = to_categorical(Y_train, num_classes = 10 )<define_variables>
tfidf_vec = CountVectorizer(ngram_range=(1,7), analyzer='char')
 tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_cvec_char_eap"] = pred_train[:,0]
train_df["nb_cvec_char_hpl"] = pred_train[:,1]
train_df["nb_cvec_char_mws"] = pred_train[:,2]
test_df["nb_cvec_char_eap"] = pred_full_test[:,0]
test_df["nb_cvec_char_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_char_mws"] = pred_full_test[:,2]
random_seed = 2<split>
tfidf_vec = TfidfVectorizer(ngram_range=(1,5), analyzer='char')
 full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 train_df["nb_tfidf_char_eap"] = pred_train[:,0]
train_df["nb_tfidf_char_hpl"] = pred_train[:,1]
train_df["nb_tfidf_char_mws"] = pred_train[:,2]
test_df["nb_tfidf_char_eap"] = pred_full_test[:,0]
test_df["nb_tfidf_char_hpl"] = pred_full_test[:,1]
test_df["nb_tfidf_char_mws"] = pred_full_test[:,2]
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.05, random_state = random_seed )<choose_model_class>
n_comp = 20
svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')
 svd_obj.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))
test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))
train_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
test_svd.columns = ['svd_char_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df, train_svd], axis=1)
 test_df = pd.concat([test_df, test_svd], axis=1)
 del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'Same', activation = 'relu', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'Same', activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size =(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'Same', activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'Same', activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size =(2, 2), strides =(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = 'relu'))
model.add(Dropout(0.25))
model.add(Dense(10, activation = 'softmax'))<choose_model_class>
cols_to_drop = ['id', 'text']
train_X = train_df.drop(cols_to_drop+['author'], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runXGB(dev_X, dev_y, val_X, val_y, test_X, seed_val=0, colsample=0.7)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
break
print("cv scores : ", cv_scores)
 out_df = pd.DataFrame(pred_full_test)
 out_df.columns = ['EAP', 'HPL', 'MWS']
out_df.insert(0, 'id', test_id)
 out_df.to_csv("sub_fe.csv", index=False )
<choose_model_class><EOS> 
import af