Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<set_options>
!pip install -q pretrainedmodels
gc.collect()<import_modules>
def model_f(pretrained=True,**kwargs):
return pretrainedmodels.se_resnext50_32x4d(num_classes=1000,pretrained='imagenet' )
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import KFold<import_modules>
path = Path('/kaggle/input/plant-pathology-2020-fgvc7')
 train = pd.read_csv(path/'train.csv')
 test = pd.read_csv(path/'test.csv')
 train['label'] = train.set_index('image_id' ).idxmax(axis=1 ).reset_index() [0]
train
import lightgbm as lgb<init_hyperparams>
pseudos = ['Test_1149', 'Test_1158','Test_1321','Test_1590','Test_1634','Test_168','Test_1728','Test_1801','Test_211','Test_282','Test_506','Test_543','Test_714','Test_727','Test_921','Test_932','Test_987','Test_991']
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'max_depth':-1,
'learning_rate': 0.01,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'num_leaves':1000,
'max_bin':1000,
'verbose': 0
}<create_dataframe>
c=['image_id', 'healthy', 'multiple_diseases', 'rust', 'scab', 'label']
pseudos = pd.DataFrame([(x, 0,1,0,0,'multiple_diseases')for x in pseudos], columns=c)
 pseudos
lgb_train = lgb.Dataset(Scaled_X_train, y_train)
 lgb_eval = lgb.Dataset(Scaled_X_test, y_test, reference=lgb_train )<train_model>
train = pd.concat([train, pseudos], axis=0 )
print('Starting training...')
 gbm = lgb.train(params,
lgb_train,
num_boost_round=100,
valid_sets=lgb_eval
)<predict_on_test>
dups = [["Test_1800", "Train_558"], ["Train_616", "Train_966"], ["Test_383", "Test_635"], ["Train_500", "Train_647"], ["Test_1691", "Test_683"], ["Test_431", "Test_776"], ["Test_218", "Train_1598"], ["Test_611", "Train_1698"], ["Train_909", "Train_914"], ["Test_1688", "Train_1154"], ["Test_700", "Train_1142"], ["Test_1047", "Train_1733"], ["Train_592", "Train_782"], ["Train_1675", "Train_417"], ["Test_1450", "Train_761"], ["Test_989", "Train_1347"], ["Test_1117", "Train_105"], ["Test_119", "Train_772"], ["Test_1503", "Test_232"], ["Test_1264", "Test_708"], ["Test_483", "Train_1407"], ["Test_1212", "Test_570"], ["Train_174", "Train_1817"], ["Test_1722", "Test_889"], ["Test_693", "Train_1277"], ["Test_1494", "Train_937"], ["Test_938", "Train_861"], ["Test_53", "Train_114"], ["Test_581", "Train_1615"], ["Train_1731", "Train_591"], ["Test_1802", "Train_574"], ["Train_665", "Train_694"], ["Train_1634", "Train_921"], ["Train_1115", "Train_1683"], ["Train_1661", "Train_815"], ["Test_1062", "Train_1368"], ["Test_1720", "Test_459"], ["Test_1561", "Test_443"], ["Train_759", "Train_828"], ["Test_1671", "Test_899"], ["Train_1581", "Train_1613"], ["Test_768", "Train_851"], ["Test_1203", "Test_1600"], ["Test_797", "Train_1585"], ["Test_1163", "Test_542"], ["Test_886", "Train_1541"], ["Test_1011", "Train_608"], ["Train_1173", "Train_379"], ["Test_1588", "Test_376"], ["Test_1675", "Train_1066"], ["Test_1020", "Train_683"], ["Test_150", "Test_1501"], ["Train_1678", "Train_931"], ["Test_965", "Train_1341"], ["Train_1587", "Train_1593"], ["Train_341", "Train_763"], ["Test_895", "Test_979"], ["Train_162", "Train_356"], ["Test_1407", "Train_1703"], ["Test_1322", "Train_1600"], ["Test_1524", "Train_919"], ["Train_1150", "Train_1365"], ["Train_1392", "Train_438"], ["Test_829", "Train_1505"], ["Test_826", "Train_847"], ["Train_115", "Train_1709"]]
dups = pd.DataFrame(dups, columns=['col1', 'col2'])
 dups.head()
dups['col1t'] = dups['col1'].str.split('_' ).str.get(0)
 dups['col2t'] = dups['col2'].str.split('_' ).str.get(0)
 dups.head()
print('Starting predicting...')
 y_pred = gbm.predict(Scaled_X_test, num_iteration=gbm.best_iteration)
 print('Done !' )<compute_test_metric>
train_dups = dups[(dups.col1t == 'Train')&(dups.col1t == dups.col2t)].merge(train, left_on='col1', right_on='image_id' ).merge(train, left_on='col2', right_on='image_id')
 train_dups.head()
print('The rmse of prediction is:', mean_squared_error(y_test, y_pred)** 0.5 )<compute_test_metric>
confused = train_dups[train_dups.label_x != train_dups.label_y]
confused
gbm_score=r2_score(y_pred=y_pred,y_true=y_test)
 print("Variance score(r2_score): %f"%gbm_score)
 print('Model accuracy:%.2f '%(gbm_score*100))
print("Root mean squared error of test:%f"%sqrt(mean_squared_error(y_test,y_pred)) )<train_model>
print(f'list of duplicates train / train: {len(train_dups)}')
 print(len(train))
train = train[~train.image_id.isin(list(train_dups['col2'])) ].reset_index(drop=True)
 print(len(train))
train = train[~train.image_id.isin(list(confused['col1'])) ].reset_index(drop=True)
 train = train[~train.image_id.isin(list(confused['col2'])) ].reset_index(drop=True)
 print(len(train))
print('' )<predict_on_test>
dups[(dups.col1t == 'Test')&(dups.col1t == dups.col2t)]
final_pred2 = gbm.predict(scaled_test, num_iteration=gbm.best_iteration)
<prepare_output>
solutions = dups[dups.col1t != dups.col2t].merge(train, left_on='col2', right_on='image_id' ).sort_values(by='label')
 solutions.head()
submission['winPlacePerc'] = final_pred2<save_to_csv>
skf = StratifiedKFold(shuffle=True, random_state=42)
 for i,(train_index, val_index)in enumerate(skf.split(train, train['label'])) :
print(train_index.shape, val_index.shape)
 train.loc[val_index, 'fold'] = i
submission.to_csv('final_submission.csv',index=False )<set_options>
train = train.iloc[list(train.index)+ [x for x in train[train.label == 'multiple_diseases'].index for j in range(0,2)]]
train = train.reset_index(drop=True )
pd.set_option('display.max_columns', 500)
 warnings.filterwarnings("ignore")
 folder = ".. /input/"
print(os.listdir(folder))
NORMFACTOR = 1000
TRAIN_SIZE = 0.9
RANDOM_STATE = 212
EARLY_STOP_ROUNDS = 10
DEBUG = False 
<prepare_x_and_y>
folds = range(0,3)
cID = ['Id', 'groupId', 'matchId']
cOrigGP= ['assists', 'boosts', 'damageDealt',
'DBNOs', 'headshotKills', 'heals',
'kills', 'killStreaks', 'longestKill', 'revives',
'rideDistance', 'roadKills', 'swimDistance', 'teamKills',
'vehicleDestroys', 'walkDistance', 'weaponsAcquired', 'killPlace']
cRKAll = ['matchDuration', 'killPoints', 'maxPlace', 'numGroups','rankPoints','winPoints']
cRKPop =['matchDuration', 'maxPlace', 'numGroups']
cMT = ['matchType']
cY = ['winPlacePerc']
cAgg = ['totalKills','netKills', 'totalDistance','teamwork','itemUse','netKillsPerDist','netItemsPerDist','headsPerKill','netWeaponsPerDist']
cGPAggMOnly = ['teamSize']
cGP = cOrigGP+cAgg
cGPAggM = cGP+cGPAggMOnly
cGPNZip = [(col, col+"N")for col in cGP]
cGPN = [col+"N" for col in cGP]
cGPT = [col+"T" for col in cGP]
cGPM = [col+"M" for col in cGPAggM]
cGPMInherit = [col+"M" for col in cGP]
cGPAll = cGPAggM+cGPT+cGPM+cGPN
def preprocess(train, cOrigFeat):
train = train.copy()
cOrig = train.columns.values
train['teamSizeFeat'] = train.groupby('groupId')['groupId'].transform('count')
 train['totalKills'] = train['DBNOs'] + train['kills'] + train['teamKills']
train['totalDistance'] = train['rideDistance'] + train['walkDistance'] + train['walkDistance']
train['netKills'] = train['DBNOs'] + train['kills'] - train['teamKills']
train['teamwork'] = train['assists'] - train['teamKills']
train['itemUse'] = train['boosts'] + train['heals']
train['netKillsPerDist'] = train['netKills']/(train['totalDistance']+1)
 train['netItemsPerDist'] = train['itemUse']/(train['totalDistance']+1)
 train['headsPerKill'] = train['headshotKills']/(train['totalKills']+1)
 train['netWeaponsPerDist'] = train['weaponsAcquired']/(train['totalDistance']+1)
 train['totalMatchPlayers'] = train.groupby('matchId')['Id'].transform('count')
 train['killsInvolved'] = train['DBNOs'] + train['kills'] + train['teamKills'] + train['assists']
train['pctMatchKills'] = train['totalKills']/train['totalMatchPlayers']
train['pctMatchKillInvolvement'] = train['killsInvolved']/train['totalMatchPlayers']
cEng = [col for col in train.columns.values if col not in cOrig]
cBase = list(cOrigFeat)+ list(cEng)
 train['teamSize'] = train['teamSizeFeat']
train['sampleWeight'] = 1/train['teamSize']
cNotFeatures = [col for col in train.columns.values if col not in cBase]
print('features done, scaling by match')
 train[cBase] = minMidMaxGroupScale(train.groupby('matchId'), 0.75, base=train, col=cBase)
 print('match aggregation done, aggregating teams')
 train = train.merge(train.groupby('groupId')[cBase].transform(np.min), how='left', suffixes=["","_min"], left_index=True, right_index=True)
 train = train.merge(train.groupby('groupId')[cBase].transform(np.max), how='left', suffixes=["","_max"], left_index=True, right_index=True)
 train = train.merge(train.groupby('groupId')[cBase].transform(np.mean), how='left', suffixes=["","_mean"], left_index=True, right_index=True)
 train[cBase] = train.groupby('groupId')[cBase].transform(np.sum)
 cDerived = [col for col in train.columns.values if col not in cBase+cNotFeatures]
cAll = cBase+cDerived
return(train, cAll)
 def minMidMaxGroupScale(g, midQuant, base, col=cGPAggM):
gMax = g[col].transform(np.max)
 print("max done, quantiles")
 g75 = g[col].transform(lambda x: np.quantile(x, midQuant))
print("quantiles done, mins")
 gMin = g[col].transform(np.min)
 print("mins done, time for big calc")
 outdf =(base[col] <= g75[col])*(base[col]-gMin[col])/(g75[col]-gMin[col] ).replace(to_replace=0,value=1)*midQuant+(base[col] > g75[col])*(( base[col]-g75[col])/(gMax[col]-g75[col] ).replace(to_replace=0,value=1)*(1-midQuant)+ midQuant)
 print("big calc done")
 return outdf<count_unique_values>
R1=(400,600)
 R2=(320,512)
 TTA=True
EPOCH1=15
EPOCH2=-1
LR1=5e-2
LR2=slice(1e-7, 1e-4)
 BS=16
tfms = [Rotate(max_deg=30),
Zoom() ,
Brightness() ,
Flip() ,
Contrast() ,
Resize(R2),
Normalize.from_stats(*imagenet_stats)]
comp = setup_aug_tfms(tfms)
 for fold in folds:
plant = DataBlock(blocks=(ImageBlock, CategoryBlock),
splitter=IndexSplitter(train[train['fold'] == fold].index),
get_x=ColReader('image_id', pref=path/"images", suff='.jpg'),
get_y=ColReader('label'),
item_tfms=Resize(R1),
batch_tfms=comp)
 dls = plant.dataloaders(train, bs=BS, seed=42)
 learn = cnn_learner(dls,
model_f,
pretrained=True,
loss_func=LabelSmoothingCrossEntropy() ,
cbs=SaveModelCallback(monitor='roc_auc_score', comp=np.greater),
model_dir='/kaggle/working',
metrics=[RocAuc() , error_rate])
 print(f'-------- {fold} ---------')
 if EPOCH1 > 0:
learn.fine_tune(EPOCH1,
base_lr=LR1,
cbs=SaveModelCallback(monitor='roc_auc_score', comp=np.greater))
learn.recorder.plot_loss()
if EPOCH2 > 0:
learn.unfreeze()
learn.fit_one_cycle(EPOCH2,
lr_max=LR2,
cbs=SaveModelCallback(monitor='roc_auc_score', comp=np.greater))
learn.recorder.plot_loss()
learn.save(f'seresnext50-fold-{fold}')
 tst_dl = dls.test_dl(test)
 if TTA:
preds, _ = learn.tta(dl=tst_dl)
 else:
preds, _ = learn.get_preds(dl=tst_dl)
 cols=['image_id'] + list(dls.vocab)
 res = pd.concat([test, pd.DataFrame(preds,columns = learn.dls.vocab)],axis=1)[cols]
res.to_csv(f'submission_resnext_{fold}.csv',index=False)
 print('confident multiple_diseases')
 my_ids = res[res['multiple_diseases'] > 0.9].sort_values(by='multiple_diseases' ).image_id.values
print(my_ids)
 res = learn.get_preds()
cols= dls.vocab
r = pd.DataFrame(res[0], columns=cols)
 valid_with_preds = pd.concat([train[train['fold'] == fold].reset_index(drop=True), r], axis=1)
 print(roc_auc_score(train[train['fold'] == fold][cols], r))
for col in cols:
score = roc_auc_score(train[train['fold'] == fold][col], r[col])
 print(f'{col} -> {score}')
 valid_with_preds.to_csv(f'validation_resnext_{fold}.csv',index=False )
<load_from_csv><EOS> 
if len(folds)> 1:
my_df = {}
for fold in folds:
my_df[fold] = pd.read_csv(f'submission_resnext_{fold}.csv' ).set_index('image_id')
 for x in ['healthy','multiple_diseases','rust','scab']:
for fold in folds:
my_df[fold][x] = rankdata(my_df[fold][x], method='min')
 for x in ['healthy','multiple_diseases','rust','scab']:
for fold in folds[1:]:
my_df[folds[0]][x] += my_df[fold][x]
my_df[folds[0]][x] = my_df[folds[0]][x] / 5
my_df[folds[0]].reset_index().to_csv('submission_resnext.csv', index=False)
 else:
print('no need for rankdata' )