Kaggle competition: Digit Recognizer
train_data = pd.read_csv('/kaggle/input/tabular-playground-series-feb-2021/train.csv')
 test_data = pd.read_csv('/kaggle/input/tabular-playground-series-feb-2021/test.csv')
 print(train_data.head() , "
")
 print(test_data.head() )<import_modules>
import random
import keras
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import matplotlib.patches as mpatches
from skimage.filters import threshold_otsu
from skimage.segmentation import clear_border
from skimage.measure import label, regionprops
from skimage.morphology import closing, square
from skimage.color import label2rgb
from math import sqrt
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools
from keras.utils.np_utils import to_categorical
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, Input, BatchNormalization
from keras.optimizers import RMSprop, Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import LearningRateScheduler
from keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns<define_variables>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
cat_features = ['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat6', 'cat7', 'cat8', 'cat9']
numerical_features = ['cont0', 'cont1', 'cont2', 'cont3', 'cont4', 'cont5','cont6', 'cont7', 'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13']<filter>
y_train = train['label']
X = train.drop(['label'],axis=1)
outlier = train_data.loc[train_data.target < 1.0]
print(outlier, "
")
 print(outlier.index )<drop_column>
labels =[]
frequencies = []
for i in range(len(y_train)) :
lab, freq = str(y_train[i]), len([n for n in X.values[i] if n > 0])
 labels.append(lab)
 frequencies.append(freq)
 data = {'Labels':labels, 'Frequencies':frequencies}
df = pd.DataFrame(data )
train_data.drop([99682], inplace = True )<prepare_x_and_y>
df.groupby('Labels' ).mean()
categorical_features = cat_features
y_train = train_data["target"]
train_data.drop(columns = ['target'], inplace = True)
 test_data_backup = test_data.copy()
train_data.drop(columns = ["id"], inplace = True)
 test_data.drop(columns = ["id"], inplace = True )<choose_model_class>
test_y = df['Labels']
test_x = df
test_x.drop('Labels', axis =1, inplace = True)
model_ctb = CatBoostRegressor(iterations = 3000,
learning_rate = 0.02,
od_type = 'Iter',
loss_function = 'RMSE',
grow_policy = 'SymmetricTree',
subsample = 0.8,
verbose = 3,
random_seed = 17)
 model_ctb.fit(train_data, y_train, cat_features=categorical_features)
 y_pred = model_ctb.predict(test_data)
 print(y_pred )<save_to_csv>
X_train1, X_val1, y_train1, y_val1 = train_test_split(test_x, test_y, test_size=0.1, random_state=1337 )
solution = pd.DataFrame({"id":test_data_backup.id, "target":y_pred})
 solution.to_csv("solution.csv", index = False)
 print("saved successful!" )<set_options>
clf_rfc = RandomForestClassifier(n_estimators = 100)
 clf_rfc.fit(X_train1,y_train1 )
%matplotlib inline 
<install_modules>
accuracy_score(clf_rfc.predict(X_val1), y_val1 )
!pip install --upgrade xgboost
xgb.__version__<set_options>
clf_knn = KNeighborsClassifier()
clf_knn.fit(X_train1,y_train1)
 accuracy_score(clf_knn.predict(X_val1), y_val1 )
shap.initjs()<load_from_csv>
imaginary_data_pca = pca.transform(imaginary_data)
 print("original shape: ", imaginary_data.shape)
 print("transformed shape:", imaginary_data_pca.shape )
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-feb-2021/sample_submission.csv')
<prepare_x_and_y>
pca = PCA(n_components=2)
 pca.fit(imaginary_data)
 imaginary_data_pca = pca.transform(imaginary_data)
target = train['target'].values<categorify>
imaginary_data_pca_new = pca.inverse_transform(imaginary_data_pca)
for feature in cat_features:
le = LabelEncoder()
le.fit(train[feature])
 train[feature] = le.transform(train[feature])
 test[feature] = le.transform(test[feature] )<define_variables>
data = {'Labels':labels, 'diagonals':diags, 'widths':widths, 'heights':heights, 'Area':frequencies, 'PC1':principalComponents[:, 0],'PC2':principalComponents[:, 1]}
df = pd.DataFrame(data)
 df
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape<init_hyperparams>
test_y = df['Labels']
test_x = df
test_x.drop('Labels', axis =1, inplace = True)
 X_train1, X_val1, y_train1, y_val1 = train_test_split(test_x, test_y, test_size=0.1, random_state=1337)
 clf_knn = KNeighborsClassifier()
clf_knn.fit(X_train1,y_train1)
 accuracy_score(clf_knn.predict(X_val1), y_val1 )
xgb_params= {
"objective": "reg:squarederror",
"max_depth": 6,
"learning_rate": 0.01,
"colsample_bytree": 0.4,
"subsample": 0.6,
"reg_alpha" : 6,
"min_child_weight": 100,
"n_jobs": 2,
"seed": 2001,
'tree_method': "gpu_hist",
"gpu_id": 0,
'predictor': 'gpu_predictor'
}<prepare_x_and_y>
mbdl = MiniBatchDictionaryLearning(n_components = 2)
 mbdl.fit(X )
test = xgb.DMatrix(test[columns] )<train_model>
comps = mbdl.transform(X )
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(xgb_params, train_df, 3800)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(test)
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))<compute_test_metric>
def test_model(d1,d1_lab, d2, d2_lab,data):
data = {'Labels':labels, 'diagonals':diags, 'widths':widths, 'heights':heights, 'Area':frequencies, 'PC1':principalComponents[:, 0],'PC2':principalComponents[:, 1]}
data[d1_lab] = d1
data[d2_lab] = d2
df = pd.DataFrame(data)
 print(df.columns)
 test_y = df['Labels']
test_x = df
test_x.drop('Labels', axis =1, inplace = True)
 X_train1, X_val1, y_train1, y_val1 = train_test_split(test_x, test_y, test_size=0.1, random_state=1337)
 clf_knn = KNeighborsClassifier()
clf_knn.fit(X_train1,y_train1)
 return accuracy_score(clf_knn.predict(X_val1), y_val1)
 test_model(comps[:, 0],'d1', comps[:, 1],'d2', data )
mean_squared_error(train_oof, target, squared=False)
<save_model>
lda = LinearDiscriminantAnalysis(n_components = 2,)
 comps = lda.fit_transform(X,y_train.values )
np.save('train_oof', train_oof)
 np.save('test_preds', test_preds )<predict_on_test>
test_model(comps[:, 0],'lda1', comps[:, 1],'lda2', data )
%%time
shap_preds = model.predict(test, pred_contribs=True )<load_from_csv>
X = X / 255.0
test = test / 255.0
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 for feature in cat_features:
le = LabelEncoder()
le.fit(train[feature])
 train[feature] = le.transform(train[feature])
 test[feature] = le.transform(test[feature] )<predict_on_test>
X = X.values.reshape(X.shape[0], 28, 28,1)
 test = test.values.reshape(test.shape[0], 28, 28,1)
%%time
shap_interactions = model.predict(xgb.DMatrix(test[columns]), pred_interactions=True )<feature_engineering>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 valid_datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False )
train['cont8_cont0'] = train['cont8']*train['cont0']
train['cont9_cont0'] = train['cont9']*train['cont0']
train['cont9_cont5'] = train['cont9']*train['cont5']
train['cont8_cont5'] = train['cont8']*train['cont5']
test['cont8_cont0'] = test['cont8']*test['cont0']
test['cont9_cont0'] = test['cont9']*test['cont0']
test['cont9_cont5'] = test['cont9']*test['cont5']
test['cont8_cont5'] = test['cont8']*test['cont5'] 
<set_options>
y_train = to_categorical(y_train,num_classes=10)
del shap_interactions, shap_preds
gc.collect()
gc.collect()<prepare_x_and_y>
learning_rate_reduction = ReduceLROnPlateau(monitor='accuracy',
patience=3,
verbose=10,
factor=0.5,
min_lr=0.00001)
test = xgb.DMatrix(test[columns] )<init_hyperparams>
def build_model(input_shape=(28, 28, 1), classes = 10):
activation = 'relu'
padding = 'same'
gamma_initializer = 'uniform'
input_layer = Input(shape=input_shape)
 hidden=Conv2D(32,(3,3), padding=padding,activation = activation, name="conv1" )(input_layer)
 hidden=BatchNormalization(name="batch1" )(hidden)
 hidden=Conv2D(32,(3,3), padding=padding,activation = activation, name="conv2" )(hidden)
 hidden=BatchNormalization(name="batch2" )(hidden)
 hidden=Conv2D(32,(5,5), padding=padding,activation = activation, name="conv3" )(hidden)
 hidden=BatchNormalization(name="batch3" )(hidden)
 hidden=MaxPool2D(pool_size=2, padding=padding, name="max1" )(hidden)
 hidden=Dropout(0.4 )(hidden)
 hidden=Conv2D(64,(3,3), padding =padding, activation = activation, name="conv4" )(hidden)
 hidden=BatchNormalization(name = 'batch4' )(hidden)
 hidden=Conv2D(64,(3,3), padding =padding, activation = activation, name="conv45" )(hidden)
 hidden=BatchNormalization(name = 'batch5' )(hidden)
 hidden=Conv2D(64,(5,5), padding =padding, activation = activation, name="conv6" )(hidden)
 hidden=BatchNormalization(name = 'batch6' )(hidden)
 hidden=MaxPool2D(pool_size=2, padding="same", name="max2" )(hidden)
 hidden=Dropout(0.4 )(hidden)
 hidden=Flatten()(hidden)
 hidden=Dense(264,activation = activation, name="Dense1" )(hidden)
 hidden=Dropout(0.3 )(hidden)
 output = Dense(classes, activation = "softmax" )(hidden)
 model = Model(inputs=input_layer, outputs=output)
 return model
xgb_params= {
"objective": "reg:squarederror",
"max_depth": 6,
"learning_rate": 0.01,
"colsample_bytree": 0.4,
"subsample": 0.6,
"reg_alpha" : 6,
"min_child_weight": 100,
"n_jobs": 2,
"seed": 2001,
'tree_method': "gpu_hist",
"gpu_id": 0,
'predictor': 'gpu_predictor'
}<train_model>
epochs = 50
initial_learningrate=2e-3
batch_size = 264
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(xgb_params, train_df, 3800)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(test)
 train_oof_2[val_ind] = temp_oof
test_preds_2 += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))<compute_test_metric>
optimizer = Adam(learning_rate=initial_learningrate)
 model = build_model(input_shape=(28, 28, 1), classes = 10)
 model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
mean_squared_error(train_oof_2, target, squared=False)
<compute_test_metric>
X_train, X_val, y_train, y_val = train_test_split(X, y_train, test_size=0.1, random_state=1337)
 datagen.fit(X_train)
 valid_datagen.fit(X_val)
mean_squared_error(0.5*train_oof+0.5*train_oof_2, target, squared=False)
<save_model>
callbacks = [learning_rate_reduction]
history = model.fit_generator(datagen.flow(X_train,y_train),
epochs = epochs,
validation_data=valid_datagen.flow(X_val,y_val),
verbose = 1,
callbacks = callbacks)
np.save('train_oof_2', train_oof_2)
 np.save('test_preds_2', test_preds_2 )<predict_on_test>
y_pre_test=model.predict(X_val)
 y_pre_test=np.argmax(y_pre_test,axis=1)
 y_test=np.argmax(y_val,axis=1)
 conf=confusion_matrix(y_test,y_pre_test)
 conf=pd.DataFrame(conf,index=range(0,10),columns=range(0,10))
%%time
shap_preds = model.predict(test, pred_contribs=True )<load_from_csv>
print('out of {} samples, we got {} incorrect'.format(len(X_train), round(len(X_train)- history.history['accuracy'][-1] * len(X_train))))
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 for feature in cat_features:
le = LabelEncoder()
le.fit(train[feature])
 train[feature] = le.transform(train[feature])
 test[feature] = le.transform(test[feature])
 train['cont8_cont0'] = train['cont8']*train['cont0']
train['cont9_cont0'] = train['cont9']*train['cont0']
train['cont9_cont5'] = train['cont9']*train['cont5']
train['cont8_cont5'] = train['cont8']*train['cont5']
test['cont8_cont0'] = test['cont8']*test['cont0']
test['cont9_cont0'] = test['cont9']*test['cont0']
test['cont9_cont5'] = test['cont9']*test['cont5']
test['cont8_cont5'] = test['cont8']*test['cont5']
columns = test.columns[1:]<save_to_csv>
predictions = model.predict(test )
sub['target'] = test_preds
sub.to_csv('submission.csv', index=False )<save_to_csv>
predictions = predictions.argmax(axis = -1)
 predictions
sub['target'] = test_preds_2
sub.to_csv('submission_2.csv', index=False )<save_to_csv>
submission['Label'] = predictions
sub['target'] = 1.1*test_preds-0.1*test_preds_2
sub.to_csv('submission_average_0.csv', index=False )<save_to_csv>
submission.to_csv('submission.csv',index=False )