Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<create_dataframe>
fastai.__version__
sab = sub.copy()<predict_on_test>
BATCH = 32
SIZE = 250
sab['FVC1'] = pe[:, 1]
sab['Confidence1'] =(pe[:, 2] - pe[:, 0] )<feature_engineering>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
subm = sab[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = abs(subm.loc[~subm.FVC1.isnull() ,'Confidence1'])
 subm.describe().T<feature_engineering>
path = Path('.. /input/imet-2019-fgvc6/' )
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
subm.describe().T<load_from_csv>
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/resnet15216epochs/stage-1.pth' 'models/resnet152.pth'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'resnet152.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1 
<save_to_csv>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False)
 print('Saved',GROUPS )<import_modules>
labels_df = pd.read_csv(path/'labels.csv')
 labels_df.head()
<install_modules>
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.head()
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index 
<set_options>
tfms = get_transforms(do_flip=True, flip_vert=False, max_rotate=0.10, max_zoom=1.5, max_warp=0.2, max_lighting=0.2,
xtra_tfms=[(symmetric_warp(magnitude=(-0,0), p=0)) ,])
 train, test = [ImageList.from_df(df, path=path, cols='id', folder=folder, suffix='.png')
 for df, folder in zip([train_df, test_df], ['train', 'test'])]
data =(train.split_by_rand_pct(0.1, seed=42)
.label_from_df(cols='attribute_ids', label_delim=' ')
.add_test(test)
.transform(tfms, size=SIZE, resize_method=ResizeMethod.PAD, padding_mode='border',)
.databunch(path=Path('.'), bs=BATCH ).normalize(imagenet_stats))
%matplotlib inline
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)
 palette_ro = ["
ROOT = ".. /input/osic-pulmonary-fibrosis-progression/"<load_from_csv>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
target = target.float()
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val + \ 
(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
train = pd.read_csv(ROOT + "train.csv")
 test = pd.read_csv(ROOT + "test.csv")
 sub = pd.read_csv(ROOT + "sample_submission.csv")
 print("Training data shape: ", train.shape)
 print("Test data shape: ", test.shape)
 train.head(10 )<count_missing_values>
learn = cnn_learner(data, base_arch=models.resnet152, loss_func=FocalLoss() , metrics=fbeta, pretrained=False)
 learn.load('resnet152' )
train.isnull().sum()<count_missing_values>
learn.lr_find()
learn.recorder.plot()
train.isnull().sum()<count_missing_values>
lr = 1e-2
learn.fit_one_cycle(10, slice(1e-5, 1e-3))
test.isnull().sum()<count_missing_values>
learn.save('stage-1', return_path=True)
 learn.export()
test.isnull().sum()<count_duplicates>
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )
dupRows_train = train[train.duplicated(subset=['Patient', 'Weeks'], keep=False)]
print("There are {} duplicate rows here({:.2f} percent of the total ).".format(len(dupRows_train), len(dupRows_train)/len(train)*100))
dupRows_train<remove_duplicates>
test_preds = learn.TTA(ds_type=DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()
train.drop_duplicates(subset=['Patient', 'Weeks'], keep=False, inplace=True )<count_unique_values>
test_df.to_csv('submission.csv', index=False )