Kaggle competition: iFood - 2019 at FGVC6
<SOS> metric: meanbesterroratk Kaggle data source: ifood-2019-at-fgvc6<feature_engineering>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
test_df['diagnosis'] = np.zeros(test_df.shape[0])
 test_generator = ImageDataGenerator(preprocessing_function=preprocess_image,
rescale=1 / 128.).flow_from_dataframe(test_df,
x_col='id_code',
y_col='diagnosis',
directory=TEST_IMG_PATH,
target_size=(IMG_WIDTH, IMG_HEIGHT),
batch_size=BATCH_SIZE,
class_mode='other',
shuffle=False )<save_to_csv>
! pip install pretrainedmodels
y_test, _ = get_preds_and_labels(model, test_generator)
 y_test = optR.predict(y_test, coefficients ).astype(np.uint8)
 test_df['diagnosis'] = y_test
test_df['id_code'] = test_df['id_code'].str.replace(r'.png$', '')
 test_df.to_csv('submission.csv', index=False )<feature_engineering>
%matplotlib inline
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 style.use('fivethirtyeight')
 sns.set(style='whitegrid',color_codes=True)
t_start = time.time()<set_options>
from fastai import *
from fastai.vision import *
import pretrainedmodels
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
train_dir = '.. /input/ifood-2019-fgvc6/train_set/train_set/'
val_dir = '.. /input/ifood-2019-fgvc6/val_set/val_set/'
from fastai.vision import *
from fastai.metrics import error_rate
from fastai.callbacks import *<import_modules>
train_df = pd.read_csv('.. /input/ifood-2019-fgvc6/train_labels.csv')
 train_df['path'] = train_df['img_name'].map(lambda x: os.path.join(train_dir,x))
val_df = pd.read_csv('.. /input/ifood-2019-fgvc6/val_labels.csv')
 val_df['path'] = val_df['img_name'].map(lambda x: os.path.join(val_dir,x))
from functools import partial
from sklearn import metrics
from collections import Counter<import_modules>
df = pd.concat([train_df, val_df], ignore_index=True)
 df.head()
import os
import PIL
import cv2<define_variables>
val_idx = [i for i in range(len(train_df), len(df)) ]
dat_path = Path('.. /input/aptos2019-blindness-detection')
 trn_path = Path('.. /input/aptos2019-blindness-detection/train_images')
(dat_path,trn_path )<load_from_csv>
sz = 256
bs = 32
train_df = pd.read_csv(dat_path/'train.csv')
 train_df.shape<train_on_grid>
data.show_batch(rows=3, figsize=(12,9))
IMG_SIZE = 512
def _load_format(path, convert_mode, after_open)->Image:
image = cv2.imread(path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 image = crop_image_from_gray(image)
 image = cv2.resize(image,(IMG_SIZE, IMG_SIZE))
image=cv2.addWeighted(image,4, cv2.GaussianBlur(image ,(0,0), 10),-4 ,128)
 return Image(pil2tensor(image, np.float32 ).div_(255))
vision.data.open_image = _load_format<feature_engineering>
gc.collect()
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2 )<define_variables>
def top_3_accuracy(preds, targs):
return top_k_accuracy(preds, targs, 3 )
bs = 64<create_dataframe>
model_name = 'se_resnext101_32x4d'
def get_cadene_model(pretrained=True, model_name='se_resnext101_32x4d'):
if pretrained:
arch = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')
 else:
arch = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained=None)
 return arch
data = ImageDataBunch.from_df(path=dat_path, folder='train_images', df=train_df, suffix='.png',
fn_col=0, label_col=1,valid_pct=0.2, size=256, ds_tfms=tfms ).normalize(imagenet_stats )<choose_model_class>
arch = get_cadene_model
kappa = KappaScore()
kappa.weights = "quadratic"<choose_model_class>
learn = cnn_learner(data, get_cadene_model, metrics=[top_3_accuracy], model_dir='/tmp/models', loss_func = LabelSmoothingCrossEntropy() ).mixup()
learn = cnn_learner(data, models.resnet50, metrics=[error_rate, kappa] )<train_model>
stage = 1
csvlogger = callbacks.CSVLogger(learn=learn, filename='history_stage_'+str(stage)+'_'+model_name, append=True)
 saveModel = callbacks.SaveModelCallback(learn, every='epoch',
monitor='top_3_accuracy', mode='max',
name='stage_'+str(stage))
reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'top_3_accuracy', mode = 'max', patience = 1, factor = 0.5)
learn.fit_one_cycle(1 )<train_model>
lr = 3e-3
learn.fit_one_cycle(4, slice(lr))
learn.fit_one_cycle(1)
<train_model>
learn.save('stage-1-SE_Resnext101' )
learn.unfreeze()
learn.fit_one_cycle(9, max_lr=slice(1e-4,1e-3))<define_variables>
stage = 2
csvlogger = callbacks.CSVLogger(learn=learn, filename='history_stage_'+str(stage)+'_'+model_name, append=True)
 saveModel = callbacks.SaveModelCallback(learn, every='epoch',
monitor='top_3_accuracy', mode='max',
name='stage_'+str(stage))
reduceLR = callbacks.ReduceLROnPlateauCallback(learn=learn, monitor = 'top_3_accuracy', mode = 'max', patience = 1, factor = 0.5)
learn.model_dir = Path('/kaggle/working/' )<save_model>
learn.fit_one_cycle(10 , slice(1e-5, 1e-3))
learn.save('/kaggle/working/FastAI_APTOS_epoch_11' )<load_from_csv>
learn.save('stage-2-SE_Resnext101' )
test_df = pd.read_csv(dat_path/'sample_submission.csv' )<create_dataframe>
test = ImageList.from_folder('.. /input/ifood-2019-fgvc6/test_set')
 len(test )
learn.data.add_test(ImageList.from_df(test_df,dat_path,folder='test_images',suffix='.png'))<predict_on_test>
learn.export('/tmp/export.pkl')
 learn = load_learner('/tmp/', test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test )
preds,y = learn.get_preds(DatasetType.Test )<prepare_output>
fnames = [f.name for f in learn.data.test_ds.items]
fnames[:4]
test_df.diagnosis = preds.argmax(1)
 test_df.head()<save_to_csv>
col = ['img_name']
test_df = pd.DataFrame(fnames, columns=col)
 test_df['label'] = ''
test_df.to_csv('submission.csv',index=False )<import_modules>
for i, pred in T(enumerate(predictions), total=len(predictions)) :
test_df.loc[i, 'label'] = ' '.join(str(int(i)) for i in np.argsort(pred)[::-1][:3] )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
import tensorflow as tf
import itertools
import sys
import gc
from IPython import get_ipython<set_options>
test_df.to_csv('submission_SE_Resnext101_fastai_mixup_2.csv', index=False )
%matplotlib inline
pd.options.display.max_columns = 50
sns.set_style('darkgrid')
 sns.set_palette('bone')
 warnings.filterwarnings('ignore' )<concatenate>
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html )
<data_type_conversions><EOS> 
create_download_link(test_df, filename='submission_SE_Resnext101_fastai_mixup_2.csv' )