Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_from_csv>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras.utils.np_utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
df3 = pd.read_csv('.. /input/0-286-private-norm/df_rcnn286.csv' )<merge>
train = pd.read_csv(".. /input/train.csv" )
df4 = pd.merge(df, df3, on = 'image_id', how = 'left' )<feature_engineering>
test = pd.read_csv(".. /input/test.csv")
 print(test.info())
 test.head()
list1 = [0,1,2,3,4,5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
for i in range(df4.shape[0]):
if df4.loc[i,'PredictionString'] == '14 1 0 0 1 1':
continue
a = df4.loc[i,'PredictionString']
b = a.split()
for j in range(int(len(a.split())/6)) :
for k in list1:
if int(b[0 + 6 * j])== k:
c = b[0 + 6 * j + 1]
b[0 + 6 * j + 1] = str(df4.loc[i,f'{k}'] * 0.05 + float(c)* 0.95)
 df4.loc[i,'PredictionString'] = ' '.join(b)
<save_to_csv>
X_train = train.drop(['label'],axis=1 ).astype('float32')
 y_train = train['label'].astype('float32')
 X_test = test.values.astype('float32' )
df_final = df4[['image_id', 'PredictionString']]
df_final.to_csv('submission.csv',index = False)
<import_modules>
def normalize(m):
return m / 255
X_train = normalize(X_train)
 X_test = normalize(X_test )
import numpy as np
import pandas as pd 
<load_from_csv>
print('Labels')
 print(y_train[:5])
 y_train = to_categorical(y_train, 10)
 print('Encoded labels')
 print(y_train[:5] )
pred_2class = pd.read_csv(".. /input/vinbigdata-2class-prediction/2-cls test pred.csv")
 low_threshold = 0.0
high_threshold = 0.874
pred_2class<load_from_csv>
checkpoint = ModelCheckpoint(filepath='mnist.model.best.hdf5',
verbose=1,
save_best_only=True,
monitor='val_acc')
 def build_model() :
model = Sequential([
Convolution2D(16,(3,3), activation='relu', input_shape=(28, 28, 1)) ,
BatchNormalization() ,
Convolution2D(16,(3,3), activation='relu'),
BatchNormalization() ,
MaxPooling2D() ,
Dropout(0.25),
Convolution2D(32,(3,3), activation='relu'),
Convolution2D(32,(3,3), activation='relu'),
BatchNormalization() ,
MaxPooling2D() ,
Dropout(0.25),
Flatten() ,
Dense(256, activation='relu'),
BatchNormalization() ,
Dropout(0.25),
Dense(10, activation='softmax')
 ])
 model.compile(optimizer=Adam() ,
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model
model = build_model()
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(".. /input/pp-old/submission_postprocessed(4 ).csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}")
<install_modules>
X_train, X_val, y_train, y_val = train_test_split(X_train,
y_train,
test_size=0.1,
random_state=42 )
!pip install -U ensemble-boxes<import_modules>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
import pandas as pd
import numpy as np
from ensemble_boxes import *
from glob import glob
import copy
from tqdm import tqdm
import shutil<load_from_csv>
image_generator = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
height_dict = pd.read_csv('.. /input/vinbigdata-original-image-dataset/vinbigdata/test.csv' ).to_dict('records')
 fnl_dict ={}
for ix,i in enumerate(height_dict):
fnl_dict[i['image_id']] = [i['width'],i['height'],i['width'],i['height']]<load_from_csv>
batch_size = 96
epochs = 60
steps_per_epoch = X_train.shape[0] / batch_size
batches = image_generator.flow(X_train, y_train, batch_size=batch_size )
subs = [
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_1.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_2.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_3.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_4.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_5.csv'),
pd.read_csv('.. /input/vinbigdata-cxr-ad-yolov5-14-class-infer-184dd1/submission.csv')
 ]
pred_2cls = pd.read_csv('.. /input/vinbigdata-2class-prediction/2-cls test pred.csv' )<categorify>
history = model.fit_generator(generator=batches,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=(X_val, y_val),
callbacks=[checkpoint, learning_rate_reduction] )
def submission_decoder(df:pd.DataFrame)-> np.ndarray:
info = df.values
df_lst = []
for i in info:
pre_lst = i[1].split(' ')
 for j in range(0,len(pre_lst),6):
df_lst.append([i[0],int(pre_lst[j]),float(pre_lst[j+1]),int(pre_lst[j+2]),int(pre_lst[j+3]),\
int(pre_lst[j+4]),int(pre_lst[j+5]),fnl_dict.get(i[0])[0],fnl_dict.get(i[0])[1]])
 return pd.DataFrame(df_lst,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max','width','height'] )<categorify>
model.load_weights('mnist.model.best.hdf5' )
subs = [submission_decoder(subs[i])for i in range(len(subs)) ]<count_unique_values>
_, train_acc = model.evaluate(X_train, y_train, verbose=0)
 _, test_acc = model.evaluate(X_val, y_val, verbose=0)
 print('Train accuracy: %.3f, Test accuracy: %.3f' %(train_acc, test_acc))
boxes_dict = {}
scores_dict = {}
labels_dict = {}
whwh_dict = {}
for i in tqdm(subs[0].image_id.unique()):
if not i in boxes_dict.keys() :
boxes_dict[i] = []
scores_dict[i] = []
labels_dict[i] = []
whwh_dict[i] = []
size_ratio = fnl_dict.get(i)
 whwh_dict[i].append(size_ratio)
 tmp_df = [subs[x][subs[x]['image_id']==i] for x in range(len(subs)) ]
for x in range(len(tmp_df)) :
boxes_dict[i].append(((tmp_df[x][['x_min','y_min','x_max','y_max']].values)/size_ratio ).tolist())
 scores_dict[i].append(tmp_df[x]['score'].values.tolist())
 labels_dict[i].append(tmp_df[x]['class_id'].values.tolist() )<statistical_test>
errors =(Y_pred_classes - y_true != 0)
 Y_pred_classes_errors = Y_pred_classes[errors]
Y_pred_errors = Y_pred[errors]
Y_true_errors = y_true[errors]
weights = [1]*5
weights += [3]
weights1 = [3,2,4,5]
iou_thr = 0.25
skip_box_thr = 0.0
sigma = 0.1
fnl = {}
for i in tqdm(boxes_dict.keys()):
boxes, scores, labels = weighted_boxes_fusion(boxes_dict[i], scores_dict[i], labels_dict[i],\
weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 if not i in fnl.keys() :
fnl[i] = {'boxes':[],'scores':[],'labels':[]}
fnl[i]['boxes'] = boxes*whwh_dict[i]
fnl[i]['scores'] = scores
fnl[i]['labels'] = labels<remove_duplicates>
predictions = model.predict_classes(X_test, verbose=2 )
pd_form = []
for i in fnl.keys() :
b = fnl[i]
for j in range(len(b['boxes'])) :
pd_form.append([i,int(b['labels'][j]),round(b['scores'][j],2),\
int(b['boxes'][j][0]),int(b['boxes'][j][1]),\
int(b['boxes'][j][2]),int(b['boxes'][j][3])])
 final_df = pd.DataFrame(pd_form,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max'])
 final_df = final_df.drop_duplicates(keep = 'first' )<categorify>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['Label'] = predictions
sub.to_csv('submission.csv',index=False )