Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<split>
%matplotlib inline
sns.set(style='white', context='notebook', palette='deep')
 warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape)
 Y_train = train['label']
X_train = train.drop(labels = ["label"],axis = 1)
 fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot(111)
 sns.countplot(Y_train);
ax.set_title(f'Labelled Digits Count({Y_train.size} total)', size=20);
ax.set_xlabel('Digits');
for patch in ax.patches:
ax.annotate('{:}'.format(patch.get_height()),(patch.get_x() +0.1, patch.get_height() -150), color='w' )
oof_val_preds_xgb = np.zeros(X.shape[0])
 oof_test_preds_xgb = np.zeros(X_test.shape[0])
 for idx,(train_idx, valid_idx)in enumerate(folds.split(X, y)) :
print('
X_train, y_train = X[train_idx], y[train_idx]
X_valid, y_valid = X[valid_idx], y[valid_idx]
dtrain = xgb.DMatrix(X_train, y_train)
 dvalid = xgb.DMatrix(X_valid, y_valid)
 dtest = xgb.DMatrix(X_test)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
xgb_model = xgb.train(params=max_params_xgb,
dtrain=dtrain,
num_boost_round=1000,
evals=watchlist,
maximize=True,
feval=gini_xgb,
early_stopping_rounds=150,
verbose_eval=100)
 best_iter = xgb_model.best_iteration
oof_test_preds_xgb += xgb_model.predict(dtest,
ntree_limit=best_iter)/folds.n_splits
oof_val_preds_xgb[valid_idx] += xgb_model.predict(dvalid, ntree_limit=best_iter)
 gini_score = eval_gini(y_valid, oof_val_preds_xgb[valid_idx])
 print(f'Fold {idx+1} gini score: {gini_score}
' )<compute_test_metric>
def check_missing_data(df):
flag=df.isna().sum().any()
if flag==True:
total = df.isnull().sum()
percent =(df.isnull().sum())/(df.isnull().count() *100)
 output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 data_type = []
for col in df.columns:
dtype = str(df[col].dtype)
 data_type.append(dtype)
 output['Types'] = data_type
return(np.transpose(output))
else:
return(False)
 check_missing_data(train)
 check_missing_data(test )
print('LightGBM OOF Gini Score:', eval_gini(y, oof_val_preds_lgb))<compute_test_metric>
X_train = X_train / 255.0
test = test / 255.0
mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 def standardize(x):
return(x-mean_px)/std_px
X_train = X_train.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1)
 for i in range(6, 9):
plt.subplot(330 +(i+1))
plt.imshow(X_train[i,:, :, 0], cmap=plt.get_cmap('gray'))
plt.title(f'Label: {Y_train[i]}');
Y_train = to_categorical(Y_train, num_classes = 10)
 train_x, test_x, train_y, test_y = train_test_split(X_train, Y_train, test_size = 0.1 )
print('XGBoost OOF Gini Score:', eval_gini(y, oof_val_preds_xgb))<save_to_csv>
validation_split = 0.15
img_gen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False,
validation_split=validation_split)
 batch_size = 86
train_gen = img_gen.flow(x=train_x, y=train_y, subset='training', batch_size=batch_size)
 valid_gen = img_gen.flow(x=train_x, y=train_y, subset='validation', batch_size=batch_size)
 visualizaion_flow = img_gen.flow(train_x, train_y, batch_size=1, shuffle=False)
 fig, axs = plt.subplots(2, 4, figsize=(20,10))
fig.suptitle('Augmentation Results', size=32)
 for axs_col in range(axs.shape[1]):
idx = np.random.randint(0, train_x.shape[0])
 img = train_x[idx,:, :, 0]
aug_img, aug_label = visualizaion_flow[idx]
axs[0, axs_col].imshow(img, cmap='gray');
axs[0, axs_col].set_title(f'example
axs[1, axs_col].imshow(aug_img[0, :, :, 0], cmap='gray');
axs[1, axs_col].set_title(f'Augmented example
oof_test_preds = oof_test_preds_lgb * 0.6 + oof_test_preds_xgb * 0.4
submission['target'] = oof_test_preds
submission.to_csv('submission.csv' )<import_modules>
seed = 43
np.random.seed(seed)
 learning_rate=0.00065
optimizer = Adam(lr=learning_rate)
 def conv_block(x, filters, kernel_size, strides, layer_no, use_pool=False, padding='same'):
x = Convolution2D(filters=filters, kernel_size=kernel_size, strides=strides, name=f'conv{layer_no}',
padding=padding )(x)
 x = BatchNormalization(name=f'bn{layer_no}' )(x)
 x = Activation('relu', name=f'activation{layer_no}' )(x)
 if use_pool:
x = MaxPooling2D(pool_size=[2, 2], strides=[2, 2], name=f'pool{layer_no}', padding='same' )(x)
 return x
def build_model(X):
h, w, c = X.shape[1:]
X = Input(shape=(h, w, c))
conv1 = conv_block(X, filters=8, kernel_size=[3, 3], strides=[1, 1], layer_no=1)
 conv2 = conv_block(conv1, filters=16, kernel_size=[2, 2], strides=[1, 1], layer_no=2)
 conv3 = conv_block(conv2, filters=32, kernel_size=[2, 2], strides=[1, 1], layer_no=3, use_pool=True)
 conv4 = conv_block(conv3, filters=64, kernel_size=[3, 3], strides=[2, 2], layer_no=4)
 conv5 = conv_block(conv4, filters=128, kernel_size=[2, 2], strides=[1, 1], layer_no=5)
 conv6 = conv_block(conv5, filters=256, kernel_size=[2, 2], strides=[1, 1], layer_no=6, use_pool=True)
 flat1 = Flatten(name='flatten1' )(conv6)
 drop1 = Dropout(0.35, name='Dopout1' )(flat1)
 dens1 = Dense(128, name='dense1' )(drop1)
 bn7 = BatchNormalization(name='bn7' )(dens1)
 drop2 = Dropout(0.35, name='Dopout2' )(bn7)
 relu1 = Activation('relu', name='activation7' )(drop2)
 dens1 = Dense(256, name='dense01' )(relu1)
 bn7 = BatchNormalization(name='bn07' )(dens1)
 drop2 = Dropout(0.5, name='Dopout02' )(bn7)
 relu1 = Activation('relu', name='activation07' )(drop2)
 dens2 = Dense(10, name='dense2' )(relu1)
 bn8 = BatchNormalization(name='bn8' )(dens2)
 output_layer = Activation('softmax', name='softmax' )(bn8)
 model = Model(inputs=X, outputs=output_layer)
 model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
 return model
def build_seq_model(X):
h, w, c = X.shape[1:]
model = Sequential([
Convolution2D(32,(3,3), activation='relu', input_shape =(h, w, c)) ,
BatchNormalization(axis=1),
Convolution2D(32,(3,3), activation='relu'),
MaxPooling2D() ,
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
BatchNormalization(axis=1),
Convolution2D(64,(3,3), activation='relu'),
MaxPooling2D() ,
Dropout(0.25),
Flatten() ,
Dense(512, activation='relu'),
Dropout(0.25),
Dense(1024, activation='relu'),
Dropout(0.5),
Dense(10, activation='softmax')
 ])
 model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
 return model
model = build_model(train_x)
 learning_rate_annealer = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 epochs = 10
hist = model.fit_generator(generator=train_gen, steps_per_epoch=train_x.shape[0]*(1-validation_split)//batch_size, epochs=epochs,
validation_data=valid_gen, validation_steps=train_x.shape[0]*validation_split//batch_size, callbacks=[learning_rate_annealer])
 fig, ax = plt.subplots(2,1)
 ax[0].plot(hist.history['loss'], color='b', label="Training loss")
 ax[0].plot(hist.history['val_loss'], color='r', label="validation loss",axes =ax[0])
 legend = ax[0].legend(loc='best', shadow=True)
 ax[1].plot(hist.history['acc'], color='b', label="Training accuracy")
 ax[1].plot(hist.history['val_acc'], color='r',label="Validation accuracy")
 legend = ax[1].legend(loc='best', shadow=True )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from numpy import nan<load_from_csv>
def calculate_performance(labels, pred, dataset):
pred_cat = np.argmax(pred, axis=1)
 labels_cat = np.argmax(labels, axis=1)
 corrects =(pred_cat == labels_cat)
 falses = dataset[~corrects]
falses_labels = labels_cat[~corrects]
falses_preds = pred[~corrects]
examples_num = labels.shape[0]
accuracy = np.count_nonzero(corrects)/ examples_num
return accuracy, [falses, falses_labels, falses_preds], [labels_cat, pred_cat]
test_y_pred = model.predict(test_x)
 test_accuracy,(test_falses, test_falses_labels, test_falses_preds),(true_labels, pred_labels)= calculate_performance(test_y, test_y_pred, test_x)
 test_f1 = f1_score(y_pred=pred_labels, y_true=true_labels, average='micro')
 print(f'Test Dataset Accuracy: {np.round(test_accuracy*100, 3)}%')
 print(f'F1 Score = {test_f1}')
 plt.figure(figsize=(10, 10))
test_matrix = confusion_matrix(y_pred=pred_labels, y_true=true_labels)
 sns.heatmap(data=test_matrix, annot=True, cmap='Blues', fmt=f'.0f')
 plt.title('Confusion Matrix - Test Dataset', size=24)
 plt.xlabel('Predictions', size=20);
plt.ylabel('Labels', size=20);
final_loss, final_acc = model.evaluate(test_x, test_y, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<prepare_x_and_y>
styles=[':','-.','--','-',':','-.','--','-',':','-.','--','-']
nets = 3
model = [0] *nets
for j in range(3):
model[j] = Sequential()
model[j].add(Conv2D(j*16+16,kernel_size=5,activation='relu',input_shape=(28,28,1)))
 model[j].add(MaxPool2D())
 model[j].add(Conv2D(j*32+32,kernel_size=5,activation='relu'))
model[j].add(MaxPool2D())
 model[j].add(Flatten())
 model[j].add(Dense(256, activation='relu'))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
 X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.333)
 history = [0] * nets
names = ["16 maps","32 maps","64 maps"]
epochs = 10
for j in range(nets):
history[j] = model[j].fit(X_train2,Y_train2, batch_size=80, epochs = epochs,
validation_data =(X_val2,Y_val2), verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 names[j],epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])))
 plt.figure(figsize=(15,5))
for i in range(nets):
plt.plot(history[i].history['val_acc'],linestyle=styles[i])
 plt.title('model accuracy')
 plt.ylabel('accuracy')
 plt.xlabel('epoch')
 plt.legend(names, loc='upper left')
 axes = plt.gca()
axes.set_ylim([0.98,1])
 plt.show()
y_train = train['SalePrice']
train.drop(['SalePrice'],axis=1,inplace=True)
 features = pd.concat([train, test])
 features.shape<drop_column>
model = Sequential()
model.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
 annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 **(x+epochs))
hist = model.fit_generator(img_gen.flow(X_train,Y_train, batch_size=64), epochs = 100,
steps_per_epoch = X_train.shape[0]//64, callbacks=[annealer], verbose=0)
 print("Train accuracy={0:.5f}, Train loss={1:.5f}".format(max(hist.history['acc']),max(hist.history['loss'])) )
<data_type_conversions><EOS> 
predictions = model.predict(test)
 predictions = np.argmax(predictions,axis = 1)
 submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("submission.csv", index=False, header=True)
 print(submissions.head(5))