Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<set_options>
%matplotlib inline
rcParams['figure.figsize'] = 12, 4
%matplotlib inline 
<train_on_grid>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
def modelfit(alg, dtrain, target, useTrainCV=True,
cv_folds=5, early_stopping_rounds=50):
if useTrainCV:
xgb_param = alg.get_xgb_params()
xgtrain = xgb.DMatrix(dtrain.values,
label=y.values)
 print("
Getting Cross-validation result.. ")
 cvresult = xgb.cv(xgb_param, xgtrain,
num_boost_round=alg.get_params() ['n_estimators'],
nfold=cv_folds,metrics='rmse',
early_stopping_rounds=early_stopping_rounds,
verbose_eval = True)
 alg.set_params(n_estimators=cvresult.shape[0])
 print("
Fitting algorithm to data...")
 alg.fit(dtrain, target, eval_metric='rmse')
 print("
Predicting from training data...")
 dtrain_predictions = alg.predict(dtrain)
 print("
Model Report")
 print("RMSE : %.4g" % np.sqrt(mean_squared_error(target.values,
dtrain_predictions)) )<train_model>
train_df['author'].value_counts()
xgb3 = XGBRegressor(learning_rate =0.01, n_estimators=3460, max_depth=3,
min_child_weight=0 ,gamma=0, subsample=0.7,
colsample_bytree=0.7,objective= 'reg:linear',
nthread=4,scale_pos_weight=1,seed=27, reg_alpha=0.00006)
 xgb_fit = xgb3.fit(X, y )<train_model>
train_df['text_length'] = train_df['text'].str.len()
svr_opt = svm.SVR(C = 100000, gamma = 1e-08)
 svr_fit = svr_opt.fit(X, y )<choose_model_class>
test_df['text_length'] = test_df['text'].str.len()
lgbm_model = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_train_metric>
train_df['author_num'] = train_df.author.map({'EAP':0, 'HPL':1, 'MWS':2})
 train_df.head()
cv_rmse(lgbm_model ).mean()<train_model>
train_df = train_df.rename(columns={'text':'original_text'})
 train_df['text'] = train_df['original_text'].str[:700]
train_df['text_length'] = train_df['text'].str.len()
lgbm_fit = lgbm_model.fit(X, y )<set_options>
test_df = test_df.rename(columns={'text':'original_text'})
 test_df['text'] = test_df['original_text'].str[:700]
test_df['text_length'] = test_df['text'].str.len()
Image(url = "http://i.imgur.com/QBuDOjs.jpg" )<choose_model_class>
X = train_df['text']
y = train_df['author_num']
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas = alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas = alphas2,
random_state = 42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
lightgbm = make_pipeline(RobustScaler() ,
LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6,
min_sum_hessian_in_leaf = 11))
xgboost = make_pipeline(RobustScaler() ,
XGBRegressor(learning_rate =0.01, n_estimators=3460,
max_depth=3,min_child_weight=0 ,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective= 'reg:linear',nthread=4,
scale_pos_weight=1,seed=27,
reg_alpha=0.00006))
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 stackX = np.array(X)
 stacky = np.array(y )<compute_test_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)
 print(X_train.shape, y_train.shape, X_test.shape, y_test.shape )
<train_model>
print(y_train.value_counts() ,'
', y_test.value_counts() )
stack_gen_model = stack_gen.fit(stackX, stacky )<predict_on_test>
vect = CountVectorizer(lowercase=False, token_pattern=r'(?u)\b\w+\b|\,|\.|\;|\:')
 vect
em_preds = elastic_model3.predict(testing_features)
 lasso_preds = lasso_model2.predict(testing_features)
 ridge_preds = ridge_model2.predict(testing_features)
 stack_gen_preds = stack_gen_model.predict(testing_features)
 xgb_preds = xgb_fit.predict(testing_features)
 svr_preds = svr_fit.predict(testing_features)
 lgbm_preds = lgbm_fit.predict(testing_features )<define_variables>
X_train_dtm = vect.fit_transform(X_train)
 X_train_dtm
stack_preds =(( 0.2*em_preds)+(0.1*lasso_preds)+(0.1*ridge_preds)+ 
(0.2*xgb_preds)+(0.1*lgbm_preds)+(0.3*stack_gen_preds))<load_from_csv>
X_test_dtm = vect.transform(X_test)
 X_test_dtm
submission = pd.read_csv(".. /input/sample_submission.csv" )<feature_engineering>
X_train_chars = X_train.str.len()
X_train_punc = X_train.apply(lambda x: len([c for c in str(x)if c in punctuation]))
X_test_chars = X_test.str.len()
X_test_punc = X_test.apply(lambda x: len([c for c in str(x)if c in punctuation]))
X_train_dtm = add_feature(X_train_dtm, [X_train_chars, X_train_punc])
 X_test_dtm = add_feature(X_test_dtm, [X_test_chars, X_test_punc] )
submission.iloc[:,1] = np.expm1(stack_preds )<save_to_csv>
nb = MultinomialNB()
nb
submission.to_csv("final_submission.csv", index=False )<define_variables>
grid_values = {'alpha':[0.01, 0.1, 1.0, 10.0, 100.0]}
grid_nb = GridSearchCV(nb, param_grid=grid_values, scoring='neg_log_loss')
 grid_nb.fit(X_train_dtm, y_train)
 grid_nb.best_params_
root = Path(".. /input")
 val_size = 0.2
rseed = 7<define_variables>
nb = MultinomialNB(alpha=1.0)
 nb.fit(X_train_dtm, y_train )
np.random.seed(rseed)
 set_random_seed(rseed )<load_from_csv>
y_pred_test = nb.predict(X_test_dtm )
train_data = pd.read_csv(root/'train.csv' )<load_from_csv>
metrics.accuracy_score(y_test, y_pred_test )
test_data = pd.read_csv(root/'test.csv' )<count_missing_values>
y_pred_train = nb.predict(X_train_dtm)
 metrics.accuracy_score(y_train, y_pred_train )
def num_missing(data):
print(f"Missing values: {data.isnull().sum().sum() }" )<count_missing_values>
metrics.confusion_matrix(y_test, y_pred_test )
num_missing(train_data )<categorify>
y_pred_prob = nb.predict_proba(X_test_dtm)
 y_pred_prob[:10]
train_enc = pd.get_dummies(train_data,columns=['label'])
 train_enc.head()<split>
metrics.log_loss(y_test, y_pred_prob )
split = StratifiedShuffleSplit(n_splits=1,test_size=val_size,random_state=rseed)
 for train_index,val_index in split.split(train_data,train_data['label']):
train=train_enc.iloc[train_index]
val=train_enc.iloc[val_index]
train.shape,val.shape<prepare_x_and_y>
logreg = LogisticRegression(random_state=8)
 logreg
train_X,train_y=train.loc[:,'pixel0':'pixel783'],train.loc[:,'label_0':'label_9']
val_X,val_y=val.loc[:,'pixel0':'pixel783'],val.loc[:,'label_0':'label_9']<normalization>
grid_values = {'C':[0.01, 0.1, 1.0, 3.0, 5.0]}
grid_logreg = GridSearchCV(logreg, param_grid=grid_values, scoring='neg_log_loss')
 grid_logreg.fit(X_train_dtm, y_train)
 grid_logreg.best_params_
train_X=normalize(train_X)
 val_X=normalize(val_X)
 show_inst(( 6,6),train_X[:9],train_y[:9] )<define_variables>
logreg = LogisticRegression(C=1.0, random_state=8)
 logreg.fit(X_train_dtm, y_train )
bs=256<set_options>
y_pred_test = logreg.predict(X_test_dtm )
traingen = ImageDataGenerator(
 rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2 )<define_search_model>
metrics.accuracy_score(y_test, y_pred_test )
def conv_concat(X, filters):
X1 = Conv2D(filters,(3,3),padding='same' )(X)
 X1 = BatchNormalization(axis=3 )(X1)
 X1 = Activation('relu' )(X1)
 X2 = Conv2D(filters,(5,5),padding='same' )(X)
 X2 = BatchNormalization(axis=3 )(X2)
 X2 = Activation('relu' )(X2)
 X3 = Conv2D(filters,(7,7),padding='same' )(X)
 X3 = BatchNormalization(axis=3 )(X3)
 X3 = Activation('relu' )(X3)
 X = concatenate([X1,X2,X3],axis=3)
 return X
def model(input_shape =(28, 28, 1), classes = 10):
X_input = Input(input_shape)
 X = conv_concat(X_input, 32)
 X = MaxPooling2D(pool_size=(2,2))(X)
 X = conv_concat(X, 64)
 X = MaxPooling2D(pool_size=(2,2))(X)
 X = conv_concat(X, 128)
 X = MaxPooling2D(pool_size=(4,4))(X)
 X = Flatten()(X)
 X = Dense(100,activation='relu' )(X)
 X = Dropout(rate=0.6 )(X)
 X = Dense(50,activation='relu' )(X)
 X = Dropout(rate=0.4 )(X)
 X = Dense(10,activation='softmax' )(X)
 fmodel = Model(inputs=X_input,outputs=X,name='fmodel')
 return fmodel<choose_model_class>
y_pred_train = logreg.predict(X_train_dtm)
 metrics.accuracy_score(y_train, y_pred_train )
kmodel = model(train_X.shape[1:] )<choose_model_class>
metrics.confusion_matrix(y_test, y_pred_test )
kmodel.compile('adam','binary_crossentropy',metrics=['accuracy'] )<train_model>
y_pred_prob = logreg.predict_proba(X_test_dtm)
 y_pred_prob[:10]
kmodel.fit_generator(traingen.flow(train_X,y=train_y, batch_size=bs),epochs=100,steps_per_epoch=train_X.shape[0]//bs )<compute_test_metric>
metrics.log_loss(y_test, y_pred_prob )
val_preds = kmodel.evaluate(val_X,val_y)
 print(f'Loss: {val_preds[0]}')
 print(f'Val accuracy: {val_preds[1]}' )<prepare_output>
X_dtm = vect.fit_transform(X)
 X_dtm
test = normalize(test_data)
 preds = kmodel.predict(test)
 labels = np.argmax(preds,axis=1)
 sub_df = pd.DataFrame({'ImageId':test_data.index+1,'Label':labels} )<save_to_csv>
X_chars = X.str.len()
X_punc = X.apply(lambda x: len([c for c in str(x)if c in punctuation]))
X_dtm = add_feature(X_dtm, [X_chars, X_punc])
 X_dtm
sub_df.to_csv('sub.csv',index=False )<set_options>
logreg.fit(X_dtm, y )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<train_model>
y_pred_train = logreg.predict(X_dtm)
 metrics.accuracy_score(y, y_pred_train )
print('Working on "%s"' % Path('.' ).absolute() )<define_variables>
test = test_df['text']
test_dtm = vect.transform(test)
 test_dtm
path = Path('.. /input' )<load_from_csv>
test_chars = test.str.len()
test_punc = test.str.count(r'\W')
 test_dtm = add_feature(test_dtm, [test_chars, test_punc])
 test_dtm
train = pd.read_csv(path/'train.csv', header='infer' )<load_from_csv>
LR_y_pred = logreg.predict(test_dtm)
 print(LR_y_pred )
test = pd.read_csv(path/'test.csv', header='infer' )<categorify>
LR_y_pred_prob = logreg.predict_proba(test_dtm)
 LR_y_pred_prob[:10]
class CustomImageList(ImageList):
def open(self, fn):

 fn = fn.reshape(28,28)
 fn = np.stack(( fn,)*3, axis=-1)
 return Image(pil2tensor(fn,dtype=np.float32))
@classmethod
def from_df_custom(cls, df, path:PathOrStr, **kwargs)->'ItemList':
res = super().from_df(df, path=path, cols=0, **kwargs)
 if 'label' in df.columns:
df = df.drop('label', axis=1)
 df = np.array(df,dtype=np.float32)/255.
 df = df/255.
 mean = df.mean()
std = df.std()
res.items =(df-mean)/std
return res<create_dataframe>
nb.fit(X_dtm, y )
test_data = CustomImageList.from_df_custom(test,path=path)
 test_data<categorify>
y_pred_train = nb.predict(X_dtm)
 metrics.accuracy_score(y, y_pred_train )
tfms = get_transforms(do_flip=False)
 data =(CustomImageList.from_df_custom(train, path=path)
.split_by_rand_pct (.2, seed=2019)
.label_from_df(cols='label')
.add_test(test_data, label=0)
.transform(tfms)
.databunch(bs=128, num_workers=0)
.normalize(imagenet_stats))
data<choose_model_class>
NB_y_pred = nb.predict(test_dtm)
 print(NB_y_pred )
learner = cnn_learner(data, models.resnet50,metrics=accuracy,model_dir='/kaggle/working/models' )<find_best_params>
NB_y_pred_prob = nb.predict_proba(test_dtm)
 NB_y_pred_prob[:10]
learner.lr_find()<train_model>
alpha = 0.6
y_pred_prob =(( 1-alpha)*LR_y_pred_prob + alpha*NB_y_pred_prob)
 y_pred_prob[:10]
learner.fit_one_cycle(7, 1e-2 )<load_pretrained>
result = pd.DataFrame(y_pred_prob, columns=['EAP','HPL','MWS'])
 result.insert(0, 'id', test_df['id'])
 result.head()
<train_model><EOS> 
result.to_csv('rhodium_submission_16.csv', index=False, float_format='%.20f' )