Kaggle competition: RANZCR CLiP - Catheter and Line Position Challenge
<SOS> metric: MCAUC Kaggle data source: ranzcr-clip-catheter-and-line-position-challenge<define_variables>
effnet_path = '.. /input/efficientnet-pytorch/'
iterstrat_path = '.. /input/iterative-stratification/iterative-stratification-master'
sys.path.append(effnet_path)
 sys.path.append(iterstrat_path )
sys.path.insert(0, ".. /input/weightedboxesfusion")
 iou_thr = 0.44
skip_box_thr = 0.00001
pred_boxes_ensemble = []
pred_scores_ensemble = []
for(b00, b01, b02, b03, b04, b05, b06, b07, b08,
b10, b11, b12, b13, b14, b15, b16, b17, b18,
b20, b21, b22, b23, b24, b25, b26, b27, b28,
s00, s01, s02, s03, s04, s05, s06, s07, s08,
s10, s11, s12, s13, s14, s15, s16, s17, s18,
s20, s21, s22, s23, s24, s25, s26, s27, s28,
h0, w0)in zip(
 tqdm(bifpn0_pred_boxes_0),
bifpn0_pred_boxes_0_lr,
bifpn0_pred_boxes_0_ud,
bifpn0_pred_boxes_l,
bifpn0_pred_boxes_l_lr,
bifpn0_pred_boxes_l_ud,
bifpn0_pred_boxes_x,
bifpn0_pred_boxes_x_lr,
bifpn0_pred_boxes_x_ud,
bifpn1_pred_boxes_0,
bifpn1_pred_boxes_0_lr,
bifpn1_pred_boxes_0_ud,
bifpn1_pred_boxes_l,
bifpn1_pred_boxes_l_lr,
bifpn1_pred_boxes_l_ud,
bifpn1_pred_boxes_x,
bifpn1_pred_boxes_x_lr,
bifpn1_pred_boxes_x_ud,
bifpn3_pred_boxes_0,
bifpn3_pred_boxes_0_lr,
bifpn3_pred_boxes_0_ud,
bifpn3_pred_boxes_l,
bifpn3_pred_boxes_l_lr,
bifpn3_pred_boxes_l_ud,
bifpn3_pred_boxes_x,
bifpn3_pred_boxes_x_lr,
bifpn3_pred_boxes_x_ud,
bifpn0_pred_scores_0,
bifpn0_pred_scores_0_lr,
bifpn0_pred_scores_0_ud,
bifpn0_pred_scores_l,
bifpn0_pred_scores_l_lr,
bifpn0_pred_scores_l_ud,
bifpn0_pred_scores_x,
bifpn0_pred_scores_x_lr,
bifpn0_pred_scores_x_ud,
bifpn1_pred_scores_0,
bifpn1_pred_scores_0_lr,
bifpn1_pred_scores_0_ud,
bifpn1_pred_scores_l,
bifpn1_pred_scores_l_lr,
bifpn1_pred_scores_l_ud,
bifpn1_pred_scores_x,
bifpn1_pred_scores_x_lr,
bifpn1_pred_scores_x_ud,
bifpn3_pred_scores_0,
bifpn3_pred_scores_0_lr,
bifpn3_pred_scores_0_ud,
bifpn3_pred_scores_l,
bifpn3_pred_scores_l_lr,
bifpn3_pred_scores_l_ud,
bifpn3_pred_scores_x,
bifpn3_pred_scores_x_lr,
bifpn3_pred_scores_x_ud,
h0_list,
w0_list):
boxes_list = [
normalize_boxes(b00, h0, w0 ).tolist() ,
normalize_boxes(b01, h0, w0 ).tolist() ,
normalize_boxes(b02, h0, w0 ).tolist() ,
normalize_boxes(b03, h0, w0 ).tolist() ,
normalize_boxes(b04, h0, w0 ).tolist() ,
normalize_boxes(b05, h0, w0 ).tolist() ,
normalize_boxes(b06, h0, w0 ).tolist() ,
normalize_boxes(b07, h0, w0 ).tolist() ,
normalize_boxes(b08, h0, w0 ).tolist() ,
normalize_boxes(b10, h0, w0 ).tolist() ,
normalize_boxes(b11, h0, w0 ).tolist() ,
normalize_boxes(b12, h0, w0 ).tolist() ,
normalize_boxes(b13, h0, w0 ).tolist() ,
normalize_boxes(b14, h0, w0 ).tolist() ,
normalize_boxes(b15, h0, w0 ).tolist() ,
normalize_boxes(b16, h0, w0 ).tolist() ,
normalize_boxes(b17, h0, w0 ).tolist() ,
normalize_boxes(b18, h0, w0 ).tolist() ,
normalize_boxes(b20, h0, w0 ).tolist() ,
normalize_boxes(b21, h0, w0 ).tolist() ,
normalize_boxes(b22, h0, w0 ).tolist() ,
normalize_boxes(b23, h0, w0 ).tolist() ,
normalize_boxes(b24, h0, w0 ).tolist() ,
normalize_boxes(b25, h0, w0 ).tolist() ,
normalize_boxes(b26, h0, w0 ).tolist() ,
normalize_boxes(b27, h0, w0 ).tolist() ,
normalize_boxes(b28, h0, w0 ).tolist()
]
scores_list = [
s00.tolist() ,
s01.tolist() ,
s02.tolist() ,
s03.tolist() ,
s04.tolist() ,
s05.tolist() ,
s06.tolist() ,
s07.tolist() ,
s08.tolist() ,
s10.tolist() ,
s11.tolist() ,
s12.tolist() ,
s13.tolist() ,
s14.tolist() ,
s15.tolist() ,
s16.tolist() ,
s17.tolist() ,
s18.tolist() ,
s20.tolist() ,
s21.tolist() ,
s22.tolist() ,
s23.tolist() ,
s24.tolist() ,
s25.tolist() ,
s26.tolist() ,
s27.tolist() ,
s28.tolist()
]
labels_list = [
[0] * len(b00),
[0] * len(b01),
[0] * len(b02),
[0] * len(b03),
[0] * len(b04),
[0] * len(b05),
[0] * len(b06),
[0] * len(b07),
[0] * len(b08),
[0] * len(b10),
[0] * len(b11),
[0] * len(b12),
[0] * len(b13),
[0] * len(b14),
[0] * len(b15),
[0] * len(b16),
[0] * len(b17),
[0] * len(b18),
[0] * len(b20),
[0] * len(b21),
[0] * len(b22),
[0] * len(b23),
[0] * len(b24),
[0] * len(b25),
[0] * len(b26),
[0] * len(b27),
[0] * len(b28)
 ]
boxes, scores, _ = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes_list, scores_list, labels_list, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes_ensemble.append(boxes)
 pred_scores_ensemble.append(scores )<concatenate>
warnings.filterwarnings("ignore")
pred_boxes_ensemble = [denormalize_clip_boxes(a, h0, w0)for a, h0, w0 in zip(pred_boxes_ensemble, h0_list, w0_list)]
pred_scores_ensemble = [a for a in pred_scores_ensemble]<categorify>
IMAGE_SIZE =(512, 512)
 PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True
IMAGE_BACKEND = 'cv2'
FOLD = 0
def set_seed(seed=0):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
set_seed()
def format_prediction_string(boxes, scores):
pred_strings = []
for s, b in zip(scores, boxes.astype(int)) :
pred_strings.append(f'{s:.4f} {b[0]} {b[1]} {b[2]} {b[3]}')
 return " ".join(pred_strings )<compute_test_metric>
data_dir = ".. /input/ranzcr-clip-catheter-line-classification/"
path_checkpoints_dir = "./checkpoints"
path_submissions_dir = "./"
path_trained_models = ".. /input/000-010-sgdr-ensemble-4"
path_test_dir= os.path.join(data_dir, 'test')
 path_sample_submission_file= os.path.join(data_dir, 'sample_submission.csv')
 submission_file = pd.read_csv(path_sample_submission_file)
 path_test_images = [os.path.join(path_test_dir, i + ".jpg")for i in submission_file.StudyInstanceUID.values]
pred_strs = []
for bboxes, scores in zip(pred_boxes_ensemble, pred_scores_ensemble):
if len(bboxes)> 0:
bboxes[:, 2] -= bboxes[:, 0]
bboxes[:, 3] -= bboxes[:, 1]
bboxes = bboxes.round()
pred_strs.append(format_prediction_string(bboxes, scores))
else:
pred_strs.append('' )<create_dataframe>
def resize_one_image(input_path, output_path, image_size):
image = cv2.imread(input_path)
 image = cv2.resize(image, image_size)
 cv2.imwrite(output_path, image)
 def resize_image_batch(input_dir, output_dir, image_size):

 if not os.path.isdir(output_dir):
os.mkdir(output_dir)
 input_paths = [os.path.join(input_dir, image_name)for image_name in os.listdir(input_dir)]
output_paths = [os.path.join(output_dir, image_name)for image_name in os.listdir(input_dir)]
image_sizes = [image_size]*len(input_paths)
 _ = Parallel(n_jobs=-1, verbose=3 )(delayed(resize_one_image )(ipath, opath, img_size)for ipath, opath, img_size in zip(input_paths, output_paths, image_sizes))
test_df = pd.DataFrame({'image_id': img_ids, 'PredictionString':pred_strs})
 test_df<save_to_csv>
path_resized_test_image_dir = os.path.join('./', "test_resized")
 print(path_resized_test_image_dir )
test_df.to_csv('submission.csv', index=False )<set_options>
submission_file = pd.read_csv(path_sample_submission_file)
 path_test_images = [os.path.join(path_resized_test_image_dir, i + ".jpg")for i in submission_file.StudyInstanceUID.values]
sns.set(context="notebook", style="darkgrid", palette="deep", font="sans-serif", font_scale=1, color_codes=True)
<load_from_csv>
path_test_dir = ".. /input/ranzcr-clip-catheter-line-classification/test"
resize_image_batch(path_test_dir, path_resized_test_image_dir, IMAGE_SIZE )
data= pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 data["Date"] = data["Date"].apply(lambda x: x.replace("-",""))
data["Date"] = data["Date"].astype(int)
<count_missing_values>
class ImageDataset:
def __init__(
 self,
image_paths,
targets=None,
augmentations=None,
backend="cv2",
channel_first=True,
grayscale=False,
grayscale_as_rgb=False, 
):

 if grayscale is False and grayscale_as_rgb is True:
raise Exception("Invalid combination of " "arguments 'grayscale=False' and 'grayscale_as_rgb=True'")
 self.image_paths = image_paths
self.targets = targets
self.augmentations = augmentations
self.backend = backend
self.channel_first = channel_first
self.grayscale = grayscale
self.grayscale_as_rgb = grayscale_as_rgb
def __len__(self):
return len(self.image_paths)
 def __getitem__(self, item):
if self.backend == "pil":
image = Image.open(self.image_paths[item])
 if self.grayscale is True and self.grayscale_as_rgb is True:
image = image.convert('RGB')
 image = np.array(image)
 if self.augmentations is not None:
augmented = self.augmentations(image=image)
 image = augmented["image"]
elif self.backend == "cv2":
if self.grayscale is False or self.grayscale_as_rgb is True:
image = cv2.imread(self.image_paths[item])
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 else:
image = cv2.imread(self.image_paths[item], cv2.IMREAD_GRAYSCALE)
 if self.augmentations is not None:
augmented = self.augmentations(image=image)
 image = augmented["image"]
else:
raise Exception("Backend not implemented")
 if not isinstance(image, torch.Tensor):
if self.channel_first is True and image.ndim == 3:
image = np.transpose(image,(2, 0, 1)).astype(np.float32)
 image = torch.tensor(image)
 if len(image.size())== 2:
image = image.unsqueeze(0)
 if self.targets is not None:
targets = self.targets[item]
targets = torch.tensor(targets)
 else:
targets = torch.tensor([])
 return {
"image": image,
"targets": targets,
}
data = data.drop(['Province/State'],axis=1)
 data = data.dropna()
data.isnull().sum() 
<load_from_csv>
test_dataset = ImageDataset(
 path_test_images,
None,
augmentations=test_augmentation,
backend=IMAGE_BACKEND,
channel_first=True,
grayscale=True,
grayscale_as_rgb=True,
)
test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv")
 test["Date"] = test["Date"].apply(lambda x: x.replace("-",""))
test["Date"] = test["Date"].astype(int)
 test["Lat"] = test["Lat"].fillna(12.5211)
 test["Long"] = test["Long"].fillna(69.9683)
 test.isnull().sum() 
<prepare_x_and_y>
class DataModule(object):
def __init__(self, train_dataset, valid_dataset, test_dataset):
self.train_dataset = train_dataset
self.valid_dataset = valid_dataset
self.test_dataset = test_dataset
def get_train_dataloader(self, **kwargs):
return torch.utils.data.DataLoader(self.train_dataset, **kwargs)
 def get_valid_dataloader(self, **kwargs):
return torch.utils.data.DataLoader(self.valid_dataset, **kwargs)
 def get_test_dataloader(self, **kwargs):
return torch.utils.data.DataLoader(self.test_dataset, **kwargs )
x =data[['Lat', 'Long', 'Date']]
y1 = data[['ConfirmedCases']]
y2 = data[['Fatalities']]
x_test = test[['Lat', 'Long', 'Date']] 
<choose_model_class>
class EfficientNetModel(torch.nn.Module):
def __init__(self, num_labels=11, pretrained=True):
super().__init__()
self.num_labels = num_labels
if pretrained:
self.backbone = EfficientNet.from_pretrained("efficientnet-b5",)
 else:
self.backbone = EfficientNet.from_name("efficientnet-b5",)
 self.dropout = torch.nn.Dropout(p=0.5)
 self.avgpool = torch.nn.AdaptiveAvgPool2d(1)
 self.fc = torch.nn.Linear(2048, num_labels)
 self.sigmoid = torch.nn.Sigmoid()
def forward(self, image, targets=None):
image = self.backbone.extract_features(image)
 image = self.avgpool(image ).squeeze(-1 ).squeeze(-1)
 image = self.dropout(image)
 image = self.fc(image)
 loss = None
if targets is not None and targets.size(1)== self.num_labels:
loss = torch.nn.BCEWithLogitsLoss()(image, targets.type_as(image))
with torch.no_grad() :
y_pred = self.sigmoid(image)
 return y_pred, loss
model = RandomForestClassifier(n_estimators=200 )<predict_on_test>
class Trainer:
def __init__(self, model, data_module, experiment_id, optimizer=None, scheduler=None, device='cuda'):
self.model = model
self.data_module = data_module
self.optimizer = optimizer
self.scheduler = scheduler
self.device = device
self.fp16 = False
self.step_scheduler_after = None
self.n_epochs = None
self.metrics = {}
self.metrics['train'] = []
self.metrics['valid'] = []
self.current_epoch = 0
self.current_batch = 0
self.scaler = None
self._best_score = -np.inf
self._delta = None
self._current_score = None
self._counter = 0
self._patience = None
self.experiment_id=experiment_id
def configure_trainer(self):
if self.optimizer is None:
self.configure_optimizers()
if self.scheduler is None:
self.configure_schedulers()
if next(self.model.parameters() ).device != self.device:
self.model.to(self.device)
 if self.fp16:
self.scaler = torch.cuda.amp.GradScaler()
def configure_optimizers(self):
self.optimizer = torch.optim.Adam(self.model.parameters() , lr=self.lr)
 def configure_schedulers(self, **kwargs):
self.scheduler = None
def set_params(self, **kwargs):
for parameter, value in kwargs.items() :
setattr(self, parameter, value)
 def compute_metrics(self, batch_outputs_collection, batch_targets_collection):
y_pred = np.concatenate(batch_outputs_collection, axis=0)
 y_true = np.concatenate(batch_targets_collection, axis=0)
 assert y_pred.shape == y_true.shape, "shape mismatch"
scores = []
for i in range(y_true.shape[1]):
try:
score = roc_auc_score(y_true[:,i], y_pred[:,i])
 scores.append(score)
 except ValueError:
raise("Behaviour Not expected: FIXME")
 avg_score = np.mean(scores)
 return avg_score
def model_forward_pass(self, data):

 for key, value in data.items() :
data[key] = value.to(self.device)
 if self.fp16:
with torch.cuda.amp.autocast() :
output, loss = self.model(**data)
 else:
output, loss = self.model(**data)
 return output, loss
def train_one_batch(self, data):
self.optimizer.zero_grad()
output, loss = self.model_forward_pass(data)
 with torch.set_grad_enabled(True):
if self.fp16:
with torch.cuda.amp.autocast() :
self.scaler.scale(loss ).backward()
self.scaler.step(self.optimizer)
 self.scaler.update()
else:
loss.backward()
self.optimizer.step()
if self.scheduler:
if self.step_scheduler_after == "batch":
if self.step_scheduler_metric is None:
self.scheduler.step()
else:
pass
return output, loss
def train_one_epoch(self, dataloader):
self.model.train()
all_outputs = []
all_targets = []
all_losses = []
tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))
for batch_id, data in tk0:
batch_outputs, batch_loss= self.train_one_batch(data)
 all_outputs.append(batch_outputs.detach().cpu().numpy())
 all_targets.append(data['targets'].detach().cpu().numpy())
 all_losses.append(batch_loss.detach().cpu().item())
 tk0.set_postfix(loss=np.array(all_losses ).mean() , stage="train", epoch=self.current_epoch)
 tk0.close()
avg_auc = self.compute_metrics(all_outputs, all_targets)
 avg_loss = np.array(all_losses ).mean()
self.metrics['train'].append({'epoch': self.current_epoch,
'avg_loss': avg_loss, 'auc_score': avg_auc})
 print(self.metrics['train'][self.current_epoch -1])
 def validate_one_batch(self, data):
output, loss = self.model_forward_pass(data)
 return output, loss
def validate_one_epoch(self, dataloader):
self.model.eval()
all_outputs = []
all_targets = []
all_losses = []
tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))
for batch_id, data in tk0:
batch_outputs, batch_loss= self.validate_one_batch(data)
 all_outputs.append(batch_outputs.detach().cpu().numpy())
 all_targets.append(data['targets'].detach().cpu().numpy())
 all_losses.append(batch_loss.detach().cpu().item())
 tk0.set_postfix(loss=np.array(all_losses ).mean() , stage="validate", epoch=self.current_epoch)
 tk0.close()
avg_auc = self.compute_metrics(all_outputs, all_targets)
 avg_loss = np.array(all_losses ).mean()
self.metrics['valid'].append({'epoch': self.current_epoch,
'avg_loss': avg_loss, 'auc_score': avg_auc})
 print(self.metrics['valid'][-1])
 def early_stoping(self):

 self._current_score = self.metrics['valid'][-1]['auc_score']
if(self._current_score - self._best_score)> self._delta:
self._best_score = self._current_score
self._counter = 0
self.save_checkpoint()
print("early stopping counter reset to 0")
 else:
self._counter += 1
print(f"early stopping counter {self._counter} out of {self._patience}")
 if self._counter == self._patience:
return True
return False
def save_checkpoint(self):

 if not os.path.isdir(path_checkpoints_dir):
os.mkdir(path_checkpoints_dir)
 model_path = os.path.join(path_checkpoints_dir, f"{self.experiment_id}.pth")
 print(f"saved the model at {model_path}")
 model_state_dict = self.model.state_dict()
if self.optimizer is not None:
opt_state_dict = self.optimizer.state_dict()
else:
opt_state_dict = None
if self.scheduler is not None:
sch_state_dict = self.scheduler.state_dict()
else:
sch_state_dict = None
model_dict = {}
model_dict["state_dict"] = model_state_dict
model_dict["optimizer"] = opt_state_dict
model_dict["scheduler"] = sch_state_dict
model_dict["epoch"] = self.current_epoch
model_dict["fp16"] = self.fp16
model_dict['lr'] = self.lr
model_dict['metrics'] = self.metrics
model_dict['best_score'] = self._best_score
model_dict['patience'] = self._patience
model_dict['delta'] = self._delta
model_dict['train_batch_size'] = self.train_batch_size
model_dict['validation_batch_size'] = self.validation_batch_size
torch.save(model_dict, model_path)
 def load(self, model_path, device=None):

 if device:
self.device = device
checkpoint = torch.load(model_path)
 if self.model:
self.model.load_state_dict(checkpoint['state_dict'])
 self.model.to(self.device)
 if self.optimizer:
self.optimizer.load_state_dict(checkpoint['optimizer'])
 if self.scheduler:
self.scheduler.load_state_dict(checkpoint['scheduler'])
 self.fp16 = True
def predict(self, test_batch_size=64, device='cuda', load=False, model_path=None, dataloader_num_workers=4, save_prediction=True):

 self.model.eval()
self.device = device
self.test_batch_size = test_batch_size
if load:
if model_path:
self.load(model_path, device=self.device)
 else:
model_path = os.path.join(path_checkpoints_dir, f"{self.experiment_id}.pth")
 print(f"loaded model={model_path}")
 self.load(model_path, device=self.device)
 if self.model is None:
raise Exception("model cannot be None.Load or train the model before inference")
 dataloader = self.data_module.get_test_dataloader(batch_size=self.test_batch_size, shuffle=False, num_workers=dataloader_num_workers)
 all_outputs = []
tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))
for batch_id, data in tk0:
for key, value in data.items() :
data[key] = value.to(self.device)
 batch_outputs, batch_loss= self.validate_one_batch(data)
 all_outputs.append(batch_outputs.detach().cpu().numpy())
 predictions = np.concatenate(all_outputs, axis=0)
 if save_prediction:
submission = pd.read_csv(path_sample_submission_file)
 assert submission.shape[0] == predictions.shape[0], "unexpected behavior.code fix required"
submission.iloc[:, 1:] = predictions
if not os.path.isdir(path_submissions_dir):
os.mkdir(path_submissions_dir)
 submission.to_csv(os.path.join(path_submissions_dir, f"{self.experiment_id}.csv"), index=False)
 tk0.close()
return predictions
def fit(self,
n_epochs=100,
lr=1e-3,
step_scheduler_after='epoch',
device='cuda',
fp16=False,
train_batch_size = 64,
validation_batch_size=64,
dataloader_shuffle=True,
dataloader_num_workers=4,
tensorboard_writer = None,
es_delta=1e-4,
es_patience=3, 
):

 self.n_epochs = n_epochs
self.step_scheduler_after = step_scheduler_after
self.device = device
self.fp16 = fp16
self.lr = lr
self._delta = es_delta
self._patience = es_patience
self.train_batch_size = train_batch_size
self.validation_batch_size = validation_batch_size
self.configure_trainer()
for i in range(1, self.n_epochs+1):
self.current_epoch = i
train_dataloader = self.data_module.get_train_dataloader(
 batch_size=train_batch_size,
shuffle=dataloader_shuffle,
num_workers=dataloader_num_workers,
pin_memory=True)
 self.train_one_epoch(train_dataloader)
 validation_dataloader = self.data_module.get_valid_dataloader(
 batch_size=validation_batch_size,
shuffle=dataloader_shuffle,
num_workers=dataloader_num_workers,
pin_memory=True 
)
 self.validate_one_epoch(validation_dataloader)
 es_flag = self.early_stoping()
if es_flag:
print(f"early stopping at epoch={i} out of {n_epochs}")
 break
model.fit(x,y1)
 pred1 = model.predict(x_test)
 pred1 = pd.DataFrame(pred1)
 pred1.columns = ["ConfirmedCases_prediction"]<choose_model_class>
def ensemble_models(model_paths, output_file, **kwargs):

 model = EfficientNetModel(pretrained=False)
 data_module = DataModule(None, None, test_dataset)
 preds_list = []
num_models = len(model_paths)
 print(f"number of models to ensemble={num_models}")
 for mpath in model_paths:
print(mpath)
 trainer = Trainer(model, data_module, None)
 prediction = trainer.predict(load=True, model_path=mpath, save_prediction=False, **kwargs)
 preds_list.append(prediction)
 mean_prediction = np.stack(preds_list, axis=-1 ).mean(axis=-1)
 print(f"mean prediction array shape={mean_prediction.shape}")
 submission = pd.read_csv(path_sample_submission_file)
 assert submission.shape[0] == mean_prediction.shape[0], "unexpected behavior.code fix required"
submission.iloc[:, 1:] = mean_prediction
if not os.path.isdir(path_submissions_dir):
os.mkdir(path_submissions_dir)
 submission.to_csv(os.path.join(path_submissions_dir, f"{output_file}.csv"), index=False )
<predict_on_test><EOS> 
model_paths = os.listdir(path_trained_models,)
 model_paths = [os.path.join(path_trained_models, mpath)for mpath in model_paths]
ensemble_models(model_paths, "submission", test_batch_size=16)
 print("done" )