Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<define_variables>
!pip install efficientnet_pytorch
warnings.filterwarnings("ignore" )
IMAGE_SIZE = 299
batch_size = 32
TEST_DIR = 'test'<load_from_csv>
IMAGE_FOLDER = '.. /input/plant-pathology-npy-images/kaggle/working/image_pickles/'
def get_image_path(filename):
return(IMAGE_FOLDER + filename + '.npy' )
data=pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 subm = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 data.head()<define_variables>
train = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/test.csv')
 train['image_path'] = train['image_id'].apply(get_image_path)
 test['image_path'] = test['image_id'].apply(get_image_path)
 train_labels = train.loc[:, 'healthy':'scab']
train_paths = train.image_path
test_paths = test.image_path
if len(data)% 32 :
split_count = len(data)//32 + 1
else :
split_count = len(data)// 32
split_batch = np.array_split(np.arange(split_count), 4)
 batch_end =[min(len(data),(split_batch[i].max() +1)*batch_size)for i in range(len(split_batch)) ]
batch_start = [0] + batch_end[:-1]
batch_start, batch_end<statistical_test>
train_paths, valid_paths, train_labels, valid_labels = train_test_split(train_paths, train_labels, test_size = 0.2, random_state=23, stratify = train_labels)
 train_paths.reset_index(drop=True,inplace=True)
 train_labels.reset_index(drop=True,inplace=True)
 valid_paths.reset_index(drop=True,inplace=True)
 valid_labels.reset_index(drop=True,inplace=True )
def get_head(linear_size = [512,256,128] , probs_dropout = 0.5):
inp = Input(shape =(None,None,base_model.output_shape[-1]))
x_avg = GlobalAveragePooling2D()(inp)
 x_max = GlobalMaxPooling2D()(inp)
 x = Concatenate()([x_avg,x_max])
 x = BatchNormalization()(x)
 x = Dropout(rate = probs_dropout )(x)
 x = Dense(linear_size[0], activation = 'tanh', kernel_regularizer = l2(1e-5))(x)
 for n in linear_size[1:] :
x = BatchNormalization()(x)
 x = Dropout(rate = probs_dropout )(x)
 x = Dense(n, kernel_regularizer = l2(1e-4))(x)
 x = PReLU()(x)
 x = Dense(1 , kernel_regularizer = l2(1e-4))(x)
 return Model(inp,x )<load_pretrained>
class LeafDataset(Data.Dataset):
def __init__(self, image_paths, labels = None, train = True, test = False):
self.paths = image_paths
self.test = test
if self.test == False:
self.labels = labels
self.train = train
self.train_transform = Compose([HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(rotate_limit=25.0, p=0.7),
OneOf([IAAEmboss(p=1),
IAASharpen(p=1),
Blur(p=1)], p=0.5),
IAAPiecewiseAffine(p=0.5)])
 self.test_transform = Compose([HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(rotate_limit=25.0, p=0.7)])
 self.default_transform = Compose([Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225), always_apply=True),
ToTensor() ])
 def __len__(self):
return self.paths.shape[0]
def __getitem__(self, i):
image = np.load(self.paths[i])
 if self.test==False:
label = torch.tensor(np.argmax(self.labels.loc[i,:].values))
if self.train:
image = self.train_transform(image=image)['image']
image = self.default_transform(image=image)['image']
elif self.test:
image = self.test_transform(image=image)['image']
image = self.default_transform(image=image)['image']
else:
image = self.default_transform(image=image)['image']
if self.test==False:
return image, label
return image
class ImageLoader(Sequence):
def __init__(self , image_list,
image_cat = None,
batch_size = 32,
shuffle = True,
include_last = True,
transforms = None,
use_ben = True,
sigma_ben = 15,
randomized_sigma_ben_range = None,
crop = True,
resized = True):
self.image_list = np.array(image_list)
 self.crop = crop
self.use_ben = use_ben
self.have_transform = False
if self.use_ben :
if not(randomized_sigma_ben_range is None):
self.sigma_ben_range = randomized_sigma_ben_range
self.random_ben = True
else :
self.random_ben = False
self.sigma_ben = sigma_ben
if transforms is None :
self.have_transform = False
if not(resized):
self.have_transform = True
self.transforms = Resize(IMAGE_SIZE,IMAGE_SIZE)
 else :
self.have_transform = True
self.transforms = transforms
if len(self.image_list)== 0 :
print('List is empty please recheck')
 else :
print('List contains {} images'.format(len(self.image_list)))
 if image_cat is None :
self.image_cat = None
else :
self.image_cat = np.array(image_cat)
 assert len(self.image_list)== len(self.image_cat), 'Image List and Cat Mismatch'
self.batch_size = batch_size
self.shuffle = shuffle
self.index = np.arange(len(self.image_list), dtype = int)
 if shuffle :
self.shuffle_index()
self.include_last = include_last
def __len__(self):
if self.include_last :
if len(self.image_list)% self.batch_size :
return len(self.image_list)// self.batch_size + 1
else :
return len(self.image_list)// self.batch_size
else :
return len(self.image_list)// self.batch_size
def __getitem__(self , index):
if self.include_last and(index ==(len(self.image_list)// self.batch_size)) :
batch_count = len(self.image_list)% self.batch_size
else :
batch_count = self.batch_size
X = np.empty(( batch_count,IMAGE_SIZE,IMAGE_SIZE,3))
idxs = np.arange(index*self.batch_size,(index*self.batch_size)+batch_count)
 if self.use_ben :
if self.random_ben :
sigma = np.random.randint(self.sigma_ben_range[0], self.sigma_ben_range[1] , size = batch_count)
 else :
sigma = [self.sigma_ben]*batch_count
for i , idx in enumerate(idxs):
img = cv2.imread(self.image_list[idx])
 img = cv2.cvtColor(img , cv2.COLOR_BGR2RGB)
 if self.crop :
img = crop_image_from_gray(img)
 if self.have_transform :
img = self.transforms(image=img)['image']
if self.use_ben :
img = cv2.addWeighted(img,4, cv2.GaussianBlur(img,(0,0), sigma[i]),-4 ,128)
 X[i] = img
X = preprocess_input(X)
 if not(self.image_cat is None):
return X, self.image_cat[idxs]
else :
return X
def shuffle_index(self):
np.random.shuffle(self.index)
 self.image_list = self.image_list[self.index]
if not(self.image_cat is None):
self.image_cat = self.image_cat[self.index]
def on_epoch_end(self):
if self.shuffle :
self.shuffle_index()<load_pretrained>
def train_fn(net, loader):
running_loss = 0
preds_for_acc = []
labels_for_acc = []
pbar = tqdm(total = len(loader), desc='Training')
 for _,(images, labels)in enumerate(loader):
images, labels = images.to(device), labels.to(device)
 net.train()
optimizer.zero_grad()
predictions = net(images)
 loss = loss_fn(predictions, labels)
 loss.backward()
optimizer.step()
scheduler.step()
running_loss += loss.item() *labels.shape[0]
labels_for_acc = np.concatenate(( labels_for_acc, labels.cpu().numpy()), 0)
 preds_for_acc = np.concatenate(( preds_for_acc, np.argmax(predictions.cpu().detach().numpy() , 1)) , 0)
 pbar.update()
accuracy = accuracy_score(labels_for_acc, preds_for_acc)
 pbar.close()
return running_loss/TRAIN_SIZE, accuracy
def valid_fn(net, loader):
running_loss = 0
preds_for_acc = []
labels_for_acc = []
pbar = tqdm(total = len(loader), desc='Validation')
 with torch.no_grad() :
for _,(images, labels)in enumerate(loader):
images, labels = images.to(device), labels.to(device)
 net.eval()
predictions = net(images)
 loss = loss_fn(predictions, labels)
 running_loss += loss.item() *labels.shape[0]
labels_for_acc = np.concatenate(( labels_for_acc, labels.cpu().numpy()), 0)
 preds_for_acc = np.concatenate(( preds_for_acc, np.argmax(predictions.cpu().detach().numpy() , 1)) , 0)
 pbar.update()
accuracy = accuracy_score(labels_for_acc, preds_for_acc)
 conf_mat = confusion_matrix(labels_for_acc, preds_for_acc)
 pbar.close()
return running_loss/VALID_SIZE, accuracy, conf_mat
def test_fn(net, loader):
preds_for_output = np.zeros(( 1,4))
with torch.no_grad() :
pbar = tqdm(total = len(loader))
for _, images in enumerate(loader):
images = images.to(device)
 net.eval()
predictions = net(images)
 preds_for_output = np.concatenate(( preds_for_output, predictions.cpu().detach().numpy()), 0)
 pbar.update()
pbar.close()
return preds_for_output
models = []
K.clear_session()
for i in range(4):
base_model = EfficientNetB3(weights = None , include_top = False , input_shape =(IMAGE_SIZE,IMAGE_SIZE,3))
head_model = get_head()
model = Model(base_model.input, head_model(base_model.output))
model.load_weights('.. /input/finetune-effnet/model_{}.h5'.format(i))
models.append(model )<normalization>
BATCH_SIZE = 8
NUM_EPOCHS = 30
TRAIN_SIZE = train_labels.shape[0]
VALID_SIZE = valid_labels.shape[0]
MODEL_NAME = 'efficientnet-b5'
device = 'cuda'
lr = 8e-4
def get_path(path):
def corrector(x):
return os.path.join(path, x)+ '.png'
return corrector
def clean_images() :
if os.path.isdir('test'):
for f in glob.glob('test/*.png'):
os.remove(f)
 os.rmdir('test' )<load_pretrained>
train_dataset = LeafDataset(train_paths, train_labels)
 trainloader = Data.DataLoader(train_dataset, shuffle=True, batch_size = BATCH_SIZE, num_workers = 2)
 valid_dataset = LeafDataset(valid_paths, valid_labels, train = False)
 validloader = Data.DataLoader(valid_dataset, shuffle=False, batch_size = BATCH_SIZE, num_workers = 2)
 test_dataset = LeafDataset(test_paths, train = False, test = True)
 testloader = Data.DataLoader(test_dataset, shuffle=False, batch_size = BATCH_SIZE, num_workers = 2 )
TTA_count = 8
preds = np.zeros([len(data),4])
 for i,(batch_s, batch_e)in enumerate(zip(batch_start,batch_end)) :
test_img_list = data.iloc[batch_s:batch_e,0].apply(get_path('.. /input/aptos2019-blindness-detection/test_images/')).tolist()
prepare_files(test_img_list, image_size =(IMAGE_SIZE,IMAGE_SIZE))
test_img_list = data.iloc[batch_s:batch_e,0].apply(get_path('test')).tolist()
loader = ImageLoader(test_img_list,
shuffle = False,
crop = False,
transforms=train_aug,
resized = True)
 for j in range(4):
for k in range(TTA_count):
preds[slice(batch_s,batch_e),j:j+1] += models[j].predict_generator(loader ,
use_multiprocessing = True,
workers = os.cpu_count())/TTA_count
clean_images()
try :
print(len(os.listdir('test')))
 except :
print('Batch {} Cleared'.format(i+1))
print()<categorify>
model = EfficientNet.from_pretrained(MODEL_NAME)
 num_ftrs = model._fc.in_features
model._fc = nn.Sequential(nn.Linear(num_ftrs,1000,bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(1000,4, bias = True))
model.to(device)
 optimizer = AdamW(model.parameters() , lr = lr, weight_decay = 1e-3)
 num_train_steps = int(len(train_dataset)/ BATCH_SIZE * NUM_EPOCHS)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_dataset)/BATCH_SIZE*5, num_training_steps=num_train_steps)
 loss_fn = torch.nn.CrossEntropyLoss()
def get_labels(X , coef = [0.5,1.5,2.5,3.5]):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
coef = [0.5353928773986922, 1.5749586701298883, 2.448495965240568, 3.393477738062221]<save_to_csv>
train_loss = []
valid_loss = []
train_acc = []
val_acc = []
subm['diagnosis'] = get_labels(preds.mean(axis = 1)+ 2, coef ).astype(int)
 subm.to_csv('submission.csv', index = False )<compute_test_metric>
for epoch in range(NUM_EPOCHS):
tl, ta = train_fn(model, loader = trainloader)
 vl, va, conf_mat = valid_fn(model, loader = validloader)
 train_loss.append(tl)
 valid_loss.append(vl)
 train_acc.append(ta)
 val_acc.append(va)
 if(epoch+1)%10==0:
path = 'epoch' + str(epoch)+ '.pt'
torch.save(model.state_dict() , path)
 printstr = 'Epoch: '+ str(epoch)+ ', Train loss: ' + str(tl)+ ', Val loss: ' + str(vl)+ ', Train acc: ' + str(ta)+ ', Val acc: ' + str(va)
 tqdm.write(printstr)
print('Kernel Runtime : {:.3f} minute'.format(( time.time() - s_time)/ 60))<load_from_csv>
subs = []
for i in range(5):
out = test_fn(model, testloader)
 output = pd.DataFrame(softmax(out,1), columns = ['healthy','multiple_diseases','rust','scab'])
 output.drop(0, inplace = True)
 output.reset_index(drop=True,inplace=True)
 subs.append(output)
 sub_eff1 = sum(subs)/5
df_train = pd.read_csv(".. /input/train_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv" )<count_missing_values>
sub1 = sub_eff1.copy()
sub1['image_id'] = test.image_id
sub1 = sub1[['image_id','healthy','multiple_diseases','rust','scab']]
sub1.to_csv('submission_efficientnet1.csv', index = False )
df_train.isnull().sum()<count_missing_values>
del model
del optimizer
del scheduler
torch.cuda.empty_cache()
df_test.isnull().sum()<filter>
model = EfficientNet.from_pretrained(MODEL_NAME)
 num_ftrs = model._fc.in_features
model._fc = nn.Sequential(nn.Linear(num_ftrs,1000,bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(1000,4, bias = True))
model.to(device)
 optimizer = AdamW(model.parameters() , lr = lr, weight_decay = 1e-3)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_dataset)/BATCH_SIZE*5, num_training_steps=num_train_steps )
df_train[df_train.winPlacePerc.isnull() ]<categorify>
train_loss = []
valid_loss = []
train_acc = []
val_acc = []
match_type = pd.get_dummies(df_train["matchType"])
 df_train["playersInMatch"] = df_train.groupby("matchId")["Id"].transform("count")
 df_train["playersInGroup"] = df_train.groupby("groupId")["Id"].transform("count" )<concatenate>
for epoch in range(NUM_EPOCHS):
tl, ta = train_fn(model, loader = trainloader)
 vl, va, conf_mat = valid_fn(model, loader = validloader)
 train_loss.append(tl)
 valid_loss.append(vl)
 train_acc.append(ta)
 val_acc.append(va)
 if(epoch+1)%10==0:
path = 'epoch' + str(epoch)+ '.pt'
torch.save(model.state_dict() , path)
 printstr = 'Epoch: '+ str(epoch)+ ', Train loss: ' + str(tl)+ ', Val loss: ' + str(vl)+ ', Train acc: ' + str(ta)+ ', Val acc: ' + str(va)
 tqdm.write(printstr)
df_train = pd.concat([df_train,match_type],sort=False, axis = 1)
 df_train.drop(["matchType","matchId","groupId"],axis=1,inplace=True )<define_variables>
subs = []
for i in range(5):
out = test_fn(model, testloader)
 output = pd.DataFrame(softmax(out,1), columns = ['healthy','multiple_diseases','rust','scab'])
 output.drop(0, inplace = True)
 output.reset_index(drop=True,inplace=True)
 subs.append(output)
 sub_eff2 = sum(subs)/5
threshold = 0.9
high_corr_pair = {}
i = -1
for key,rows in corr.iterrows() :
i += 1
for j,vals in enumerate(rows):
if i != j:
if abs(vals)> threshold:
high_corr_pair["_".join([key,corr.columns[j]])] = vals
high_corr_pair<string_transform>
sub2 = sub_eff2.copy()
sub2['image_id'] = test.image_id
sub2 = sub2[['image_id','healthy','multiple_diseases','rust','scab']]
sub2.to_csv('submission_efficientnet2.csv', index = False )
toRemoved = set()
for pair in high_corr_pair:
a,b = pair.split("_")
 if corrTarget[a] <= corrTarget[b]:
toRemoved.add(a)
 else:
toRemoved.add(b)
 toRemoved<drop_column>
sub =(sub_eff1 + sub_eff2)/2
sub['image_id'] = test.image_id
sub = sub[['image_id','healthy','multiple_diseases','rust','scab']]
sub.to_csv('submission.csv', index = False )
df_train.drop(["killPoints","maxPlace","winPoints"], axis = 1, inplace = True )<set_options>
!pip install efficientnet_pytorch
warnings.filterwarnings("ignore" )
df_train["walkDistance"].quantile(0.99 )<feature_engineering>
IMAGE_FOLDER = '.. /input/plant-pathology-npy-images/kaggle/working/image_pickles/'
def get_image_path(filename):
return(IMAGE_FOLDER + filename + '.npy' )
<import_modules>
train = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/test.csv')
 train['image_path'] = train['image_id'].apply(get_image_path)
 test['image_path'] = test['image_id'].apply(get_image_path)
 train_labels = train.loc[:, 'healthy':'scab']
train_paths = train.image_path
test_paths = test.image_path
<categorify>
train_paths, valid_paths, train_labels, valid_labels = train_test_split(train_paths, train_labels, test_size = 0.2, random_state=23, stratify = train_labels)
 train_paths.reset_index(drop=True,inplace=True)
 train_labels.reset_index(drop=True,inplace=True)
 valid_paths.reset_index(drop=True,inplace=True)
 valid_labels.reset_index(drop=True,inplace=True )
match_type = pd.get_dummies(df_test["matchType"])
 df_test["playersInMatch"] = df_test.groupby("matchId")["Id"].transform("count")
 df_test["playersInGroup"] = df_test.groupby("groupId")["Id"].transform("count")
 df_test = pd.concat([df_test,match_type],sort=False, axis = 1)
 df_test.drop(["matchType","matchId","groupId"],axis=1,inplace=True)
 df_test.drop(["killPoints","maxPlace","winPoints"], axis = 1, inplace = True )<rename_columns>
class LeafDataset(Data.Dataset):
def __init__(self, image_paths, labels = None, train = True, test = False):
self.paths = image_paths
self.test = test
if self.test == False:
self.labels = labels
self.train = train
self.train_transform = Compose([HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(rotate_limit=25.0, p=0.7),
OneOf([IAAEmboss(p=1),
IAASharpen(p=1),
Blur(p=1)], p=0.5),
IAAPiecewiseAffine(p=0.5)])
 self.test_transform = Compose([HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(rotate_limit=25.0, p=0.7)])
 self.default_transform = Compose([Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225), always_apply=True),
ToTensor() ])
 def __len__(self):
return self.paths.shape[0]
def __getitem__(self, i):
image = np.load(self.paths[i])
 if self.test==False:
label = torch.tensor(np.argmax(self.labels.loc[i,:].values))
if self.train:
image = self.train_transform(image=image)['image']
image = self.default_transform(image=image)['image']
elif self.test:
image = self.test_transform(image=image)['image']
image = self.default_transform(image=image)['image']
else:
image = self.default_transform(image=image)['image']
if self.test==False:
return image, label
return image
df_train.set_index("Id",inplace=True )<prepare_x_and_y>
def train_fn(net, loader):
running_loss = 0
preds_for_acc = []
labels_for_acc = []
pbar = tqdm(total = len(loader), desc='Training')
 for _,(images, labels)in enumerate(loader):
images, labels = images.to(device), labels.to(device)
 net.train()
optimizer.zero_grad()
predictions = net(images)
 loss = loss_fn(predictions, labels)
 loss.backward()
optimizer.step()
scheduler.step()
running_loss += loss.item() *labels.shape[0]
labels_for_acc = np.concatenate(( labels_for_acc, labels.cpu().numpy()), 0)
 preds_for_acc = np.concatenate(( preds_for_acc, np.argmax(predictions.cpu().detach().numpy() , 1)) , 0)
 pbar.update()
accuracy = accuracy_score(labels_for_acc, preds_for_acc)
 pbar.close()
return running_loss/TRAIN_SIZE, accuracy
def valid_fn(net, loader):
running_loss = 0
preds_for_acc = []
labels_for_acc = []
pbar = tqdm(total = len(loader), desc='Validation')
 with torch.no_grad() :
for _,(images, labels)in enumerate(loader):
images, labels = images.to(device), labels.to(device)
 net.eval()
predictions = net(images)
 loss = loss_fn(predictions, labels)
 running_loss += loss.item() *labels.shape[0]
labels_for_acc = np.concatenate(( labels_for_acc, labels.cpu().numpy()), 0)
 preds_for_acc = np.concatenate(( preds_for_acc, np.argmax(predictions.cpu().detach().numpy() , 1)) , 0)
 pbar.update()
accuracy = accuracy_score(labels_for_acc, preds_for_acc)
 conf_mat = confusion_matrix(labels_for_acc, preds_for_acc)
 pbar.close()
return running_loss/VALID_SIZE, accuracy, conf_mat
def test_fn(net, loader):
preds_for_output = np.zeros(( 1,4))
with torch.no_grad() :
pbar = tqdm(total = len(loader))
for _, images in enumerate(loader):
images = images.to(device)
 net.eval()
predictions = net(images)
 preds_for_output = np.concatenate(( preds_for_output, predictions.cpu().detach().numpy()), 0)
 pbar.update()
pbar.close()
return preds_for_output
X = df_train.drop(["winPlacePerc"], axis=1 ).values
Y = df_train["winPlacePerc"].values<split>
BATCH_SIZE = 8
NUM_EPOCHS = 30
TRAIN_SIZE = train_labels.shape[0]
VALID_SIZE = valid_labels.shape[0]
MODEL_NAME = 'efficientnet-b5'
device = 'cuda'
lr = 8e-4
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.2 )<normalization>
train_dataset = LeafDataset(train_paths, train_labels)
 trainloader = Data.DataLoader(train_dataset, shuffle=True, batch_size = BATCH_SIZE, num_workers = 2)
 valid_dataset = LeafDataset(valid_paths, valid_labels, train = False)
 validloader = Data.DataLoader(valid_dataset, shuffle=False, batch_size = BATCH_SIZE, num_workers = 2)
 test_dataset = LeafDataset(test_paths, train = False, test = True)
 testloader = Data.DataLoader(test_dataset, shuffle=False, batch_size = BATCH_SIZE, num_workers = 2 )
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test )<train_model>
model = EfficientNet.from_pretrained(MODEL_NAME)
 num_ftrs = model._fc.in_features
model._fc = nn.Sequential(nn.Linear(num_ftrs,1000,bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(1000,4, bias = True))
model.to(device)
 optimizer = AdamW(model.parameters() , lr = lr, weight_decay = 1e-3)
 num_train_steps = int(len(train_dataset)/ BATCH_SIZE * NUM_EPOCHS)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_dataset)/BATCH_SIZE*5, num_training_steps=num_train_steps)
 loss_fn = torch.nn.CrossEntropyLoss()
<train_model>
train_loss = []
valid_loss = []
train_acc = []
val_acc = []
%%time
lgb_train = lgb.Dataset(X_train, Y_train)
 lgb_eval = lgb.Dataset(X_test, Y_test, reference=lgb_train)
 lgbm_params = {'objective': 'regression','metric': 'rmse'}
m_lgb = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_eval,
num_boost_round=2000,
early_stopping_rounds=5 )<predict_on_test>
for epoch in range(NUM_EPOCHS):
tl, ta = train_fn(model, loader = trainloader)
 vl, va, conf_mat = valid_fn(model, loader = validloader)
 train_loss.append(tl)
 valid_loss.append(vl)
 train_acc.append(ta)
 val_acc.append(va)
 if(epoch+1)%10==0:
path = 'epoch' + str(epoch)+ '.pt'
torch.save(model.state_dict() , path)
 printstr = 'Epoch: '+ str(epoch)+ ', Train loss: ' + str(tl)+ ', Val loss: ' + str(vl)+ ', Train acc: ' + str(ta)+ ', Val acc: ' + str(va)
 tqdm.write(printstr)
Y_pred = m_lgb.predict(X_test, num_iteration=m_lgb.best_iteration)
 r2_score(Y_test,Y_pred )<prepare_x_and_y>
subs = []
for i in range(5):
out = test_fn(model, testloader)
 output = pd.DataFrame(softmax(out,1), columns = ['healthy','multiple_diseases','rust','scab'])
 output.drop(0, inplace = True)
 output.reset_index(drop=True,inplace=True)
 subs.append(output)
 sub_eff1 = sum(subs)/5
X_for_pred = df_test.values[:,1:]<save_to_csv>
sub1 = sub_eff1.copy()
sub1['image_id'] = test.image_id
sub1 = sub1[['image_id','healthy','multiple_diseases','rust','scab']]
sub1.to_csv('submission_efficientnet1.csv', index = False )
X_for_pred = sc.transform(X_for_pred)
 Y_test_pred = m_lgb.predict(X_for_pred, num_iteration=m_lgb.best_iteration)
 for_submission = pd.DataFrame({"Id":df_test.Id,"winPlacePerc":Y_test_pred})
 for_submission.to_csv('submission.csv', index=False )<set_options>
del model
del optimizer
del scheduler
torch.cuda.empty_cache()
sns.set(style="darkgrid")
 py.init_notebook_mode(connected=True)
 cf.go_offline() 
<load_from_csv>
model = EfficientNet.from_pretrained(MODEL_NAME)
 num_ftrs = model._fc.in_features
model._fc = nn.Sequential(nn.Linear(num_ftrs,1000,bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(1000,4, bias = True))
model.to(device)
 optimizer = AdamW(model.parameters() , lr = lr, weight_decay = 1e-3)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_dataset)/BATCH_SIZE*5, num_training_steps=num_train_steps )
train = pd.read_csv('.. /input/train_V2.csv' ).dropna()
test = pd.read_csv('.. /input/test_V2.csv' )<count_unique_values>
train_loss = []
valid_loss = []
train_acc = []
val_acc = []
print("There are %s matches in the training set and %s in the test set" %(train['matchId'].nunique() , test['matchId'].nunique()))<categorify>
for epoch in range(NUM_EPOCHS):
tl, ta = train_fn(model, loader = trainloader)
 vl, va, conf_mat = valid_fn(model, loader = validloader)
 train_loss.append(tl)
 valid_loss.append(vl)
 train_acc.append(ta)
 val_acc.append(va)
 if(epoch+1)%10==0:
path = 'epoch' + str(epoch)+ '.pt'
torch.save(model.state_dict() , path)
 printstr = 'Epoch: '+ str(epoch)+ ', Train loss: ' + str(tl)+ ', Val loss: ' + str(vl)+ ', Train acc: ' + str(ta)+ ', Val acc: ' + str(va)
 tqdm.write(printstr)
def addNbPlayersFeature(df):
df_res = df
df_res = df_res.assign(nb_players_total=df_res.groupby('matchId')['Id'].transform('count'))
df_res = df_res.assign(nb_players_team=df_res.groupby(['matchId', 'groupId'])['Id'].transform('count'))
return df_res
train = addNbPlayersFeature(train )<categorify>
subs = []
for i in range(5):
out = test_fn(model, testloader)
 output = pd.DataFrame(softmax(out,1), columns = ['healthy','multiple_diseases','rust','scab'])
 output.drop(0, inplace = True)
 output.reset_index(drop=True,inplace=True)
 subs.append(output)
 sub_eff2 = sum(subs)/5
def addNbKillsFeature(df):
df_res = df.assign(nb_kills_team = df.groupby(['matchId', 'groupId'])['kills'].transform('sum'))
return df_res
train = addNbKillsFeature(train )<categorify>
sub2 = sub_eff2.copy()
sub2['image_id'] = test.image_id
sub2 = sub2[['image_id','healthy','multiple_diseases','rust','scab']]
sub2.to_csv('submission_efficientnet2.csv', index = False )
<feature_engineering><EOS> 
sub =(sub_eff1 + sub_eff2)/2
sub['image_id'] = test.image_id
sub = sub[['image_id','healthy','multiple_diseases','rust','scab']]
sub.to_csv('submission.csv', index = False )