Kaggle competition: Digit Recognizer
warnings.filterwarnings("ignore" )<load_from_csv>
fastai.__version__
train_data = pd.read_csv(path+'train.csv')
 samp_subm = pd.read_csv(path+'sample_submission.csv' )<define_variables>
assert torch.cuda.is_available() , "GPU not available"
print('Number train samples:', len(train_data.index))
print('Number test samples:', len(samp_subm.index))<load_from_csv>
torch.__version__
idnum = 2
image_id = train_data.loc[idnum, 'image_id']
data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')
 img = data_file.pixel_array<load_from_csv>
from fastai import *
from fastai.vision.all import *
pred_2class = pd.read_csv(".. /input/vinbigdata-2class-prediction/2-cls test pred.csv")
 low_threshold = 0.001
high_threshold = 0.87
pred_2class<load_from_csv>
path = Path('/kaggle/input/digit-recognizer' )
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(".. /input/vinbigdatastack/submission_postprocessed.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1<save_to_csv>
path.ls()
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )<install_modules>
df = pd.read_csv(path/'train.csv', low_memory=False)
 df_test = pd.read_csv(path/'test.csv', low_memory=False )
!pip install -U ensemble-boxes<import_modules>
path_train = Path(".. /train")
 path_test = Path(".. /test" )
import pandas as pd
import numpy as np
from ensemble_boxes import *
from glob import glob
import copy
from tqdm import tqdm
import shutil<load_from_csv>
def saveDigit(digit_row, filepath):
digit = digit_row.reshape(28,28)
 digit = digit.astype(np.uint8)
 img = Image.fromarray(digit)
 img.save(filepath )
height_dict = pd.read_csv('.. /input/vinbigdata-original-image-dataset/vinbigdata/test.csv' ).to_dict('records')
 fnl_dict ={}
for ix,i in enumerate(height_dict):
fnl_dict[i['image_id']] = [i['width'],i['height'],i['width'],i['height']]<load_from_csv>
for index, row in df.iterrows() :
filePath = f'.. /train/{row[0]}/{index}.jpg'
saveDigit(row[1:].values, filePath)
subs = [
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_1.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_2.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_3.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_4.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_5.csv')
 ]
pred_2cls = pd.read_csv('.. /input/vinbigdata-2-class-classifier-complete-pipeline/results/tmp_debug/test_pred.csv' )<categorify>
for index, row in df_test.iterrows() :
filePath = f'.. /test/{index+1}.jpg'
saveDigit(row.values, filePath )
def submission_decoder(df:pd.DataFrame)-> np.ndarray:
info = df.values
df_lst = []
for i in info:
pre_lst = i[1].split(' ')
 for j in range(0,len(pre_lst),6):
df_lst.append([i[0],int(pre_lst[j]),float(pre_lst[j+1]),int(pre_lst[j+2]),int(pre_lst[j+3]),\
int(pre_lst[j+4]),int(pre_lst[j+5]),fnl_dict.get(i[0])[0],fnl_dict.get(i[0])[1]])
 return pd.DataFrame(df_lst,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max','width','height'] )<categorify>
imgs = get_image_files(path_train )
subs = [submission_decoder(subs[i])for i in range(len(subs)) ]<count_unique_values>
Image.open(imgs[0] )
boxes_dict = {}
scores_dict = {}
labels_dict = {}
whwh_dict = {}
for i in tqdm(subs[0].image_id.unique()):
if not i in boxes_dict.keys() :
boxes_dict[i] = []
scores_dict[i] = []
labels_dict[i] = []
whwh_dict[i] = []
size_ratio = fnl_dict.get(i)
 whwh_dict[i].append(size_ratio)
 tmp_df = [subs[x][subs[x]['image_id']==i] for x in range(len(subs)) ]
for x in range(len(tmp_df)) :
boxes_dict[i].append(((tmp_df[x][['x_min','y_min','x_max','y_max']].values)/size_ratio ).tolist())
 scores_dict[i].append(tmp_df[x]['score'].values.tolist())
 labels_dict[i].append(tmp_df[x]['class_id'].values.tolist() )<compute_train_metric>
dls = db.dataloaders(path_train)
 dls.show_batch()
weights = [1]*5
weights1 = [3,2,4,5]
iou_thr = 0.5
skip_box_thr = 0.0001
sigma = 0.1
fnl = {}
for i in tqdm(boxes_dict.keys()):
boxes, scores, labels = weighted_boxes_fusion(boxes_dict[i], scores_dict[i], labels_dict[i],\
weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 if not i in fnl.keys() :
fnl[i] = {'boxes':[],'scores':[],'labels':[]}
fnl[i]['boxes'] = boxes*whwh_dict[i]
fnl[i]['scores'] = scores
fnl[i]['labels'] = labels<remove_duplicates>
xb,yb = dls.one_batch()
xb.shape,yb.shape
pd_form = []
for i in fnl.keys() :
b = fnl[i]
for j in range(len(b['boxes'])) :
pd_form.append([i,int(b['labels'][j]),round(b['scores'][j],2),\
int(b['boxes'][j][0]),int(b['boxes'][j][1]),\
int(b['boxes'][j][2]),int(b['boxes'][j][3])])
 final_df = pd.DataFrame(pd_form,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max'])
 final_df = final_df.drop_duplicates(keep = 'first' )<categorify>
dls.bs = 32
def submission_encoder(df:pd.DataFrame)-> np.ndarray:
dct = {}
for i in tqdm(df['image_id'].unique()):
if not i in dct.keys() :
dct[i] = []
tmp = df[df['image_id'] == i].values
for j in tmp:
dct[i].append(int(j[1]))
dct[i].append(float(j[2]))
dct[i].append(int(j[3]))
dct[i].append(int(j[4]))
dct[i].append(int(j[5]))
dct[i].append(int(j[6]))
dct[i] = map(str,dct[i])
 dct[i] = ' '.join(dct[i])
 dct = [[k, v] for k, v in dct.items() ]
return pd.DataFrame(dct,columns = ['image_id','PredictionString'] ).reset_index(drop = True)
 df = submission_encoder(final_df)
 df.to_csv('Fold5Yolo.csv', index=False )<merge>
learn = cnn_learner(dls, resnet50, metrics=accuracy ).to_fp16()
NORMAL = "14 1 0 0 1 1"
low_threshold = 0.00
high_threshold = 0.99
pred_det_df = df
n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2cls, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
if ' 14 ' not in merged_df.loc[i, "PredictionString"]:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )<set_options>
learn.fine_tune(6, freeze_epochs=3 )
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50)
<install_modules>
test_dl = dls.test_dl(get_image_files(path_test))
class_score, y = learn.get_preds(dl=test_dl )
!pip install detectron2 -f \
https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html
!pip install pytorch-pfn-extras timm<load_pretrained>
class_score = np.argmax(class_score, axis=1)
def save_yaml(filepath: str, content: Any, width: int = 120):
with open(filepath, "w")as f:
yaml.dump(content, f, width=width )<init_hyperparams>
predicted_classes = [dls.vocab[i] for i in class_score]
predicted_classes[:10]
<init_hyperparams><EOS> 
output = pd.DataFrame({'ImageId': image_id_list, 'Label': predicted_classes})
 output.ImageId = output.ImageId.astype(int)
 output = output.sort_values(by='ImageId', ignore_index=True)
 output.to_csv('submission.csv', index=False)
 output.head()