Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<prepare_output>
import numpy as np
import pandas as pd
import random
import torch
import torch.nn.functional as F
import torchvision
import cv2
import albumentations
import albumentations.pytorch
import os
import PIL
import copy
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns
preds = preds/6<create_dataframe>
!pip install --upgrade git+git://github.com/carloalbertobarbano/pytorch-train-utils
ds = get_dataset(test_filenames, augment=False, repeat=False,
labeled=False, return_img_name=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
from pytorchtrainutils import trainer
from pytorchtrainutils import metrics
from pytorchtrainutils import utils
submission = pd.DataFrame(dict(image_name=list(image_names), target=preds))
submission = submission.sort_values('image_name')
 submission = submission.set_index("image_name")
 submission.to_csv('submission.csv', index=True)
 submission.head()<predict_on_test>
seed = 42
utils.set_seed(seed)
 device = torch.device('cuda')
 lr = 1e-2
batch_size = 8
n_epochs_224 = 30
n_epochs_448 = 20
arch = 'resnet18'
mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
<count_values>
dataset_path = '/kaggle/input/plantpathology2020fgvc7pickles/plant-pathology-2020-fgvc7-pickles'
train_df = pd.read_csv(os.path.join(dataset_path, 'train.csv'))
test_df = pd.read_csv(os.path.join(dataset_path, 'test.csv'))
np.zeros(( count_data_items(test_filenames),1))<import_modules>
def preprocess_df(df):
df['label'] = df.multiple_diseases * 2 + \
df.rust * 3 + \
df.scab * 4 + \
df.healthy
df.label -= 1
return df
import numpy as np
import pandas as pd<load_from_csv>
train_df = preprocess_df(train_df)
 train_df.head()
data1 = pd.read_csv('.. /input/siimmy-submissions/submission(3 ).csv')
 data2 = pd.read_csv('.. /input/stacking-ensemble-on-my-submissions/submission_mean.csv')
 data3 = pd.read_csv('.. /input/stacking-ensemble-on-my-submissions/submission_median.csv')
 data4 = pd.read_csv('.. /input/eda-modelling-of-the-external-data-inc-ensemble/external_meta_ensembled.csv')
 data5 = pd.read_csv('.. /input/new-basline-np-log2-ensemble-top-10/submission.csv')
 data6 = pd.read_csv('.. /input/minmax-highest-public-lb-9619/submission.csv')
 submission = data1.copy()<feature_engineering>
class PlantDataset(torch.utils.data.dataset.Dataset):
def __init__(self, df, path, transform):
super().__init__()
self.df = df
self.path = os.path.join(path, 'images')
 self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
entry = self.df.iloc[index]
fname = entry.image_id + '.npy'
fname = os.path.join(self.path, fname)
 img = np.load(fname)
 img = self.transform(img)
 return img, entry.values[1:5].astype('float64' )
ncol = data1.shape[1]
data2['target'] = data4.iloc[:, 1:ncol].mean(axis=1)
 data3['target'] = data4.iloc[:, 1:ncol].median(axis=1)
<feature_engineering>
train_df, val_df = train_test_split(train_df, test_size=0.3, random_state=seed, stratify=train_df.label )
submission['target'] = 1/6 * data1['target'] + 1/6 * data2['target'] + 1/6 * data3['target'] + 1/6 *data4['target']+ 1/6 *data5['target']+ 1/6 *data6['target']<save_to_csv>
train_weight = val_weight = torch.tensor([
[1., 1.],
[1.5, 2.],
[1., 1.],
[1., 1.]
] ).to(device )
submission.to_csv('submission.csv', index=False, float_format='%.6f' )<load_from_csv>
class_weights = torch.tensor([1., 1.3, 1.1, 1.])
 sampler_weights = class_weights[train_df.label.values]
s1= pd.read_csv('.. /input/siimmy-submissions/submission(6 ).csv')
 s2= pd.read_csv('.. /input/siimmy-submissions/submission(7 ).csv')
 s3= pd.read_csv('.. /input/siimmy-submissions/b3submission.csv')
 submission= s1.copy()
submission['target'] = 1/3 * s1['target'] + 1/3 * s2['target'] + 1/3 * s3['target']
submission.to_csv('b3submission.csv', index=False, float_format='%.6f' )<import_modules>
def stack_image(image, **kwargs):
to_tensor = albumentations.pytorch.ToTensor()
vflip = albumentations.VerticalFlip(always_apply=True)
 hflip = albumentations.HorizontalFlip(always_apply=True)
 return torch.stack([
to_tensor(image=vflip(image=image)['image'])['image'],
to_tensor(image=hflip(image=image)['image'])['image'],
to_tensor(image=vflip(image=hflip(image=image)['image'])['image'])['image'],
to_tensor(image=image)['image']
])
 def get_transform(img_size, crop_size):
train_transform = albumentations.Compose([
albumentations.Resize(img_size, img_size, always_apply=True),
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.ShiftScaleRotate(rotate_limit=30.0, scale_limit=0.2, shift_limit=0.15, p=0.7),
albumentations.CenterCrop(crop_size, crop_size, always_apply=True),
albumentations.Normalize(mean, std),
albumentations.pytorch.ToTensor()
])
 transform = albumentations.Compose([
albumentations.Resize(img_size, img_size, always_apply=True),
albumentations.CenterCrop(crop_size, crop_size, always_apply=True),
albumentations.Normalize(mean, std),
albumentations.pytorch.ToTensor()
])
 tta_transform = albumentations.Compose([
albumentations.Resize(img_size, img_size, always_apply=True),
albumentations.CenterCrop(crop_size, crop_size, always_apply=True),
albumentations.Normalize(mean, std),
albumentations.Lambda(stack_image, always_apply=True)
 ])
 lambda_train = lambda image: train_transform(image=image)['image']
lambda_valid = lambda image: transform(image=image)['image']
lambda_tta = lambda image: tta_transform(image=image)['image']
return lambda_train, lambda_valid, lambda_tta
warnings.filterwarnings("ignore" )<import_modules>
model = torchvision.models.resnet18(pretrained=True)
 num_ft = model.fc.in_features
model.fc = torch.nn.Linear(in_features=num_ft, out_features=4, bias=True)
 model = model.to(device )
l5kit.__version__<set_options>
class Softmaxer(torch.nn.Module):
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, x):
x = self.model(x)
 return F.softmax(x, dim=1 )
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 set_seed(42 )<init_hyperparams>
def bce(preds, targets, weight=None):
loss = F.binary_cross_entropy_with_logits(preds, targets.type(preds.dtype), reduction='none')
 if weight is not None:
weight = weight[:, targets.T.long() ]
idx = np.diag_indices(weight.shape[0])
 weight = weight[idx[0], idx[1], :]
loss *= weight.T
return loss.mean()
cfg = {
'format_version': 4,
'data_path': "/kaggle/input/lyft-motion-prediction-autonomous-vehicles",
'model_params': {
'model_architecture': 'resnet34',
'history_num_frames': 10,
'history_step_size': 1,
'history_delta_time': 0.1,
'future_num_frames': 50,
'future_step_size': 1,
'future_delta_time': 0.1,
'model_name': "model_resnet34_output",
'lr': 1e-3,
'weight_path': "/kaggle/input/lyft-pretrained-model-hv/model_multi_update_lyft_public.pth",
'train': False,
'predict': True
},
'raster_params': {
'raster_size': [224, 224],
'pixel_size': [0.5, 0.5],
'ego_center': [0.25, 0.5],
'map_type': 'py_semantic',
'satellite_map_key': 'aerial_map/aerial_map.png',
'semantic_map_key': 'semantic_map/semantic_map.pb',
'dataset_meta_key': 'meta.json',
'filter_agents_threshold': 0.5
},
'train_data_loader': {
'key': 'scenes/train.zarr',
'batch_size': 16,
'shuffle': True,
'num_workers': 4
},
'test_data_loader': {
'key': 'scenes/test.zarr',
'batch_size': 32,
'shuffle': False,
'num_workers': 4
},
'train_params': {
'max_num_steps': 101,
'checkpoint_every_n_steps': 20,
}
}<feature_engineering>
img_size = 250
crop_size = 224
name = f'{arch}-{crop_size}'
print(f'{name} image size: {img_size}, crop size: {crop_size}' )
DIR_INPUT = cfg["data_path"]
os.environ["L5KIT_DATA_FOLDER"] = DIR_INPUT
dm = LocalDataManager(None )<create_dataframe>
train_transform, valid_transform, tta_transform = get_transform(img_size=img_size, crop_size=crop_size)
 train_dataset = PlantDataset(train_df, dataset_path, train_transform)
 train_sampler = torch.utils.data.sampler.WeightedRandomSampler(sampler_weights, len(train_df))
train_loader = torch.utils.data.DataLoader(
 train_dataset, batch_size=batch_size, num_workers=4,
sampler=train_sampler, shuffle=False 
)
 val_dataset = PlantDataset(val_df, dataset_path, valid_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False )
train_cfg = cfg["train_data_loader"]
rasterizer = build_rasterizer(cfg, dm)
 train_zarr = ChunkedDataset(dm.require(train_cfg["key"])).open()
train_dataset = AgentDataset(cfg, train_zarr, rasterizer)
 train_dataloader = DataLoader(train_dataset, shuffle=train_cfg["shuffle"], batch_size=train_cfg["batch_size"],
num_workers=train_cfg["num_workers"])
 print("==================================TRAIN DATA==================================")
 print(train_dataset )<create_dataframe>
criterion = bce
optimizer = torch.optim.SGD(model.parameters() , lr=lr, weight_decay=1e-5)
 lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True )
test_cfg = cfg["test_data_loader"]
rasterizer = build_rasterizer(cfg, dm)
 test_zarr = ChunkedDataset(dm.require(test_cfg["key"])).open()
test_mask = np.load(f"{DIR_INPUT}/scenes/mask.npz")["arr_0"]
test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
 test_dataloader = DataLoader(test_dataset,shuffle=test_cfg["shuffle"],batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"])
 print("==================================TEST DATA==================================")
 print(test_dataset )<choose_model_class>
tracked_metrics = [
metrics.MultilabelAccuracy() ,
metrics.MultilabelRocAuc() ,
metrics.MultilabelConfusionMatrix()
]
best_model = trainer.fit(
 model, train_dataloader=train_loader, val_dataloader=val_loader,
test_dataloader=None, test_every=0, criterion=criterion,
optimizer=optimizer, scheduler=lr_scheduler, metrics=tracked_metrics, n_epochs=n_epochs_224,
metric_choice='col-auc', mode='max',
name=name, device=device, weight={'train': train_weight, 'val': val_weight}
)
class LyftMultiModel(nn.Module):
def __init__(self, cfg: Dict, num_modes=3):
super().__init__()
architecture = cfg["model_params"]["model_architecture"]
backbone = eval(architecture )(pretrained=True, progress=True)
 self.backbone = backbone
num_history_channels =(cfg["model_params"]["history_num_frames"] + 1)* 2
num_in_channels = 3 + num_history_channels
self.backbone.conv1 = nn.Conv2d(
 num_in_channels,
self.backbone.conv1.out_channels,
kernel_size=self.backbone.conv1.kernel_size,
stride=self.backbone.conv1.stride,
padding=self.backbone.conv1.padding,
bias=False, 
)
 if architecture == "resnet50":
backbone_out_features = 2048
else:
backbone_out_features = 512
self.future_len = cfg["model_params"]["future_num_frames"]
num_targets = 2 * self.future_len
self.head = nn.Sequential(
 nn.Linear(in_features=backbone_out_features, out_features=4096), 
)
 self.num_preds = num_targets * num_modes
self.num_modes = num_modes
self.logit = nn.Linear(4096, out_features=self.num_preds + num_modes)
 def forward(self, x):
x = self.backbone.conv1(x)
 x = self.backbone.bn1(x)
 x = self.backbone.relu(x)
 x = self.backbone.maxpool(x)
 x = self.backbone.layer1(x)
 x = self.backbone.layer2(x)
 x = self.backbone.layer3(x)
 x = self.backbone.layer4(x)
 x = self.backbone.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.head(x)
 x = self.logit(x)
 bs, _ = x.shape
pred, confidences = torch.split(x, self.num_preds, dim=1)
 pred = pred.view(bs, self.num_modes, self.future_len, 2)
 assert confidences.shape ==(bs, self.num_modes)
 confidences = torch.softmax(confidences, dim=1)
 return pred, confidences<compute_train_metric>
tracked_metrics = [
metrics.MultilabelAccuracy(metric='top1-acc', apply_sigmoid=False),
metrics.MultilabelRocAuc(apply_sigmoid=False),
metrics.MultilabelConfusionMatrix()
]
softmaxer = Softmaxer(model)
 softmaxer_best = Softmaxer(best_model)
 val_dataset = PlantDataset(val_df, dataset_path, tta_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False)
 val_logs = trainer.test(softmaxer, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True)
 best_val_logs = trainer.test(softmaxer_best, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True )
def forward(data, model, device, criterion = pytorch_neg_multi_log_likelihood_batch):
inputs = data["image"].to(device)
 target_availabilities = data["target_availabilities"].to(device)
 targets = data["target_positions"].to(device)
 preds, confidences = model(inputs)
 loss = criterion(targets, preds, confidences, target_availabilities)
 return loss, preds, confidences<choose_model_class>
print(f'Final {name} val:', trainer.summarize_metrics(val_logs))
print(f'Best {name} val:', trainer.summarize_metrics(best_val_logs))
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = LyftMultiModel(cfg)
 weight_path = cfg["model_params"]["weight_path"]
if weight_path:
model.load_state_dict(torch.load(weight_path))
model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg["model_params"]["lr"])
 print(f'device {device}' )<train_model>
img_size *= 2
crop_size *= 2
name = f'{arch}-{crop_size}'
print(f'{name} image size: {img_size}, crop size: {crop_size}' )
print(model )<init_hyperparams>
train_transform, valid_transform, tta_transform = get_transform(img_size=img_size, crop_size=crop_size)
 train_dataset = PlantDataset(train_df, dataset_path, train_transform)
 train_sampler = torch.utils.data.sampler.WeightedRandomSampler(sampler_weights, len(train_df))
train_loader = torch.utils.data.DataLoader(
 train_dataset, batch_size=batch_size, num_workers=4,
sampler=train_sampler, shuffle=False 
)
 val_dataset = PlantDataset(val_df, dataset_path, valid_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False )
if cfg["model_params"]["train"]:
tr_it = iter(train_dataloader)
 progress_bar = tqdm(range(cfg["train_params"]["max_num_steps"]))
num_iter = cfg["train_params"]["max_num_steps"]
losses_train = []
iterations = []
metrics = []
times = []
model_name = cfg["model_params"]["model_name"]
start = time.time()
for i in progress_bar:
try:
data = next(tr_it)
 except StopIteration:
tr_it = iter(train_dataloader)
 data = next(tr_it)
 model.train()
torch.set_grad_enabled(True)
 loss, _, _ = forward(data, model, device)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
losses_train.append(loss.item())
 progress_bar.set_description(f"loss: {loss.item() } loss(avg): {np.mean(losses_train)}")
 if i % cfg['train_params']['checkpoint_every_n_steps'] == 0:
torch.save(model.state_dict() , f'{model_name}_{i}.pth')
 iterations.append(i)
 metrics.append(np.mean(losses_train))
times.append(( time.time() -start)/60)
 results = pd.DataFrame({'iterations': iterations, 'metrics(avg)': metrics, 'elapsed_time(mins)': times})
 results.to_csv(f"train_metrics_{model_name}_{num_iter}.csv", index = False)
 print(f"Total training time is {(time.time() -start)/60} mins")
 print(results.head() )<save_to_csv>
criterion = bce
optimizer = torch.optim.SGD(model.parameters() , lr=lr, weight_decay=1e-5)
 lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True )
pred_path = 'submission.csv'
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs = np.concatenate(confidences_list)
 )<import_modules>
tracked_metrics = [
metrics.MultilabelAccuracy(apply_sigmoid=True),
metrics.MultilabelRocAuc(apply_sigmoid=True),
metrics.MultilabelConfusionMatrix()
]
best_model = trainer.fit(
 model, train_dataloader=train_loader, val_dataloader=val_loader,
test_dataloader=None, test_every=0, criterion=criterion,
optimizer=optimizer, scheduler=lr_scheduler, metrics=tracked_metrics, n_epochs=n_epochs_448,
metric_choice='col-auc', mode='max',
name=name, device=device, weight={'train': train_weight, 'val': val_weight}
)
import shutil<save_to_csv>
tracked_metrics = [
metrics.MultilabelAccuracy(metric='top1-acc', apply_sigmoid=False),
metrics.MultilabelRocAuc(apply_sigmoid=False),
metrics.MultilabelConfusionMatrix()
]
softmaxer = Softmaxer(model)
 softmaxer_best = Softmaxer(best_model)
 val_dataset = PlantDataset(val_df, dataset_path, tta_transform)
 val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=10, num_workers=4, shuffle=False)
 val_logs = trainer.test(softmaxer, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True)
 best_val_logs = trainer.test(softmaxer_best, criterion=criterion, test_dataloader=val_loader, metrics=tracked_metrics, weight=val_weight, device=device, tta=True )
src = ".. /input/submission6/lyft-prediction-with-multi-mode-confidence.csv"
dst = "submission.csv"
shutil.copyfile(src, dst )<load_from_csv>
print(f'Final {name} val:', trainer.summarize_metrics(val_logs))
print(f'Best {name} val:', trainer.summarize_metrics(best_val_logs))
sns.set_style('whitegrid')
 %matplotlib inline
train_df = pd.read_csv('.. /input/train.csv', nrows=10000)
 test_df = pd.read_csv('.. /input/test.csv', nrows=10000)
 destination = pd.read_csv('.. /input/destinations.csv', nrows=100000)
 train_df.info()
print("----------------------------")
 test_df.info()
train_df.head()<data_type_conversions>
test_dataset = PlantDataset(test_df, dataset_path, tta_transform)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=10, shuffle=False )
def convert_date_into_days(df):
df['srch_ci'] = pd.to_datetime(df['srch_ci'])
 df['srch_co'] = pd.to_datetime(df['srch_co'])
 df['date_time'] = pd.to_datetime(df['date_time'])
 df['stay_dur'] =(df['srch_co'] - df['srch_ci'] ).astype('timedelta64[D]')
 df['no_of_days_bet_booking'] =(df['srch_ci'] - df['date_time'] ).astype('timedelta64[D]')
 df['Cin_day'] = df["srch_ci"].apply(lambda x: x.day)
 df['Cin_month'] = df["srch_ci"].apply(lambda x: x.month)
 df['Cin_year'] = df["srch_ci"].apply(lambda x: x.year)
 convert_date_into_days(train_df)
 convert_date_into_days(test_df)
 fig, ax = plt.subplots()
fig.set_size_inches(13, 8)
 sns.countplot('Cin_month',data=train_df[train_df["is_booking"] == 1],order=list(range(1,13)) ,ax=ax)
 fig, ax = plt.subplots()
fig.set_size_inches(13, 8)
 sns.countplot('Cin_day',data=train_df[train_df["is_booking"] == 1],order=list(range(1,32)) ,ax=ax)
 fig, ax = plt.subplots()
fig.set_size_inches(13, 8)
 sns.countplot('stay_dur',data=train_df[train_df["is_booking"] == 1],ax=ax)
 test_user_id = test_df['user_id']
columns = ['date_time', 'srch_ci','user_id','srch_destination_type_id','srch_destination_id', 'site_name', 'user_location_region', 'user_location_city',
'user_id', 'srch_co', 'srch_adults_cnt', 'srch_children_cnt', 'srch_rm_cnt']
train_df.drop(columns=columns,axis=1,inplace=True)
 test_df.drop(columns=columns,axis=1,inplace=True )<feature_engineering>
def get_test_preds(model, test_loader):
outputs = []
for batch_idx,(data, labels)in enumerate(tqdm(test_loader)) :
batch_size, n_crops, c, h, w = data.size()
data = data.view(-1, c, h, w)
 with torch.no_grad() :
output = model(data.to(device))
output = output.view(batch_size, n_crops, -1 ).mean(1)
 outputs.append(output.cpu())
 return torch.cat(outputs, dim=0 ).numpy()
total = train_df.isnull().sum().sort_values(ascending=False)
 percent =(train_df.isnull().sum() /train_df['hotel_cluster'].count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data
train_df['Cin_day'] = train_df['Cin_day'].fillna(26.0)
 train_df['Cin_month'] = train_df['Cin_month'].fillna(8.0)
 train_df['Cin_year'] = train_df['Cin_year'].fillna(2014.0)
 train_df['stay_dur'] = train_df['stay_dur'].fillna(1.0)
 train_df['no_of_days_bet_booking'] = train_df['no_of_days_bet_booking'].fillna(0.0)
 train_df['orig_destination_distance'].fillna(train_df['orig_destination_distance'].mean() , inplace=True )<load_from_csv>
def make_submission_df(df, preds):
classes = ['healthy', 'multiple_diseases', 'rust', 'scab']
for idx, name in enumerate(classes):
df[name] = pd.Series(preds[:, idx])
 return df
train = pd.read_csv('.. /input/train.csv',
dtype={'is_booking':bool,'srch_destination_id':np.int32, 'hotel_cluster':np.int32},
usecols=['srch_destination_id','is_booking','hotel_cluster'],
chunksize=1000000)
 aggs = []
print('-'*38)
 for chunk in train:
agg = chunk.groupby(['srch_destination_id',
'hotel_cluster'])['is_booking'].agg(['sum','count'])
 agg.reset_index(inplace=True)
 aggs.append(agg)
 print('.',end='')
 print('')
 aggs = pd.concat(aggs, axis=0)
 aggs.head()
CLICK_WEIGHT = 0.05
agg = aggs.groupby(['srch_destination_id','hotel_cluster'] ).sum().reset_index()
agg['count'] -= agg['sum']
agg = agg.rename(columns={'sum':'bookings','count':'clicks'})
 agg['relevance'] = agg['bookings'] + CLICK_WEIGHT * agg['clicks']
agg.head()
def most_popular(group, n_max=5):
relevance = group['relevance'].values
hotel_cluster = group['hotel_cluster'].values
most_popular = hotel_cluster[np.argsort(relevance)[::-1]][:n_max]
return np.array_str(most_popular)[1:-1]
most_pop = agg.groupby(['srch_destination_id'] ).apply(most_popular)
 most_pop = pd.DataFrame(most_pop ).rename(columns={0:'hotel_cluster'})
 most_pop.head()
test = pd.read_csv('.. /input/test.csv',
dtype={'srch_destination_id':np.int32},
usecols=['srch_destination_id'],)
 test = test.merge(most_pop, how='left',left_on='srch_destination_id',right_index=True)
 test.head()
test.hotel_cluster.isnull().sum()
most_pop_all = agg.groupby('hotel_cluster')['relevance'].sum().nlargest(5 ).index
most_pop_all = np.array_str(most_pop_all)[1:-1]
print(most_pop_all)
 test.hotel_cluster.fillna(most_pop_all,inplace=True)
 test.hotel_cluster.to_csv('submission.csv',header=True, index_label='id' )<define_variables>
softmaxer.eval()
test_preds = get_test_preds(softmaxer, test_loader)
 test_submission = make_submission_df(test_df, test_preds)
 test_submission.to_csv(f'submission-{name}.csv', index=False )
<load_from_csv><EOS> 
softmaxer_best.eval()
test_preds = get_test_preds(softmaxer_best, test_loader)
 test_submission = make_submission_df(test_df, test_preds)
 test_submission.to_csv(f'submission-best-{name}.csv', index=False )