Kaggle competition: ALASKA2 Image Steganalysis
<SOS> metric: WeightedAUC Kaggle data source: alaska2-image-steganalysis<feature_engineering>
!pip install -q efficientnet_pytorch > /dev/null
power_six = train.id[train.budget > 1000][train.revenue < 100]
for k in power_six :
train.loc[train['id'] == k,'revenue'] = train.loc[train['id'] == k,'revenue'] * 1000000<categorify>
SEED = 512
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
def multi_hot_encode(df, column_name, mlb=MultiLabelBinarizer()):
encoded = pd.DataFrame(mlb.fit_transform(df[column_name]))
encoded.columns = [f'{column_name}_{i}'.format(i)for i in mlb.classes_]
return mlb, encoded<create_dataframe>
def conv_column_to_SVD(df, column_name, size=10, sparse_mlb=MultiLabelBinarizer(sparse_output=True)) :
sparse_matrix = sparse_mlb.fit_transform(df[column_name])
 sparse_SVD = TruncatedSVD(size)
 sparse_TSVD = pd.DataFrame(sparse_SVD.fit_transform(sparse_matrix))
sparse_TSVD.columns = [f'{column_name}_{i}' for i in range(size)]
return sparse_mlb, sparse_TSVD<categorify>
def get_train_transforms() :
return A.Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
genre_mlb, genre_encoded = multi_hot_encode(train,'genres')
 cast_sparse_mlb, cast_TSVD = conv_column_to_SVD(train, 'cast', 10)
 keywords_sparse_mlb, keywords_TSVD = conv_column_to_SVD(train, 'Keywords', 10)
 languages_mlb, languages_TSVD = conv_column_to_SVD(train, 'spoken_languages', 5)
 prodcomp_mlb, prodcomp_TSVD = conv_column_to_SVD(train, 'production_companies', 5)
 prodcountries_mlb, prodcountries_TSVD = conv_column_to_SVD(train, 'production_countries', 3 )<categorify>
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, kinds, image_names, labels, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
target = onehot(4, label)
 return image, target
def __len__(self)-> int:
return self.image_names.shape[0]
def get_labels(self):
return list(self.labels )
_,test_genre= multi_hot_encode(test,'genres',genre_mlb)
 _,test_cast = conv_column_to_SVD(test, 'cast', 10)
 _,test_keywords = conv_column_to_SVD(test, 'Keywords', 10)
 _,test_languages = conv_column_to_SVD(test, 'spoken_languages', 5)
 _,test_prodcomp = conv_column_to_SVD(test, 'production_companies', 5)
 _,test_prodcountries = conv_column_to_SVD(test, 'production_countries', 3 )<merge>
fold_number = 0
train_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] != fold_number].kind.values,
image_names=dataset[dataset['fold'] != fold_number].image_name.values,
labels=dataset[dataset['fold'] != fold_number].label.values,
transforms=get_train_transforms() , 
)
 validation_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] == fold_number].kind.values,
image_names=dataset[dataset['fold'] == fold_number].image_name.values,
labels=dataset[dataset['fold'] == fold_number].label.values,
transforms=get_valid_transforms() ,
)
train = train.join(genre_encoded)
 train = train.join(cast_TSVD)
 train = train.join(keywords_TSVD)
 train = train.join(languages_TSVD)
 train = train.join(prodcomp_TSVD)
 train = train.join(prodcountries_TSVD )<merge>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def alaska_weighted_auc(y_true, y_valid):

 tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1 ).clip(min=0, max=1 ).astype(int)
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = alaska_weighted_auc(self.y_true, self.y_pred)
 @property
def avg(self):
return self.score
test = test.join(test_genre)
 test = test.join(test_cast)
 test = test.join(test_keywords)
 test = test.join(test_languages)
 test = test.join(test_prodcomp)
 test = test.join(test_prodcountries )<feature_engineering>
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.05):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )
add_datepart(train, 'release_date')
 add_datepart(test, 'release_date' )<drop_column>
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = './'
self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = LabelSmoothing().to(self.device)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss, final_scores = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss, final_scores = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
targets = targets.to(self.device ).float()
batch_size = images.shape[0]
images = images.to(self.device ).float()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 targets = targets.to(self.device ).float()
images = images.to(self.device ).float()
batch_size = images.shape[0]
self.optimizer.zero_grad()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 loss.backward()
final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss, final_scores
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )
cats_to_drop=['id','title','genres', 'cast', 'crew','Keywords','spoken_languages','production_companies','production_countries','homepage', 'belongs_to_collection','poster_path','imdb_id','original_language', 'original_title', 'overview','tagline','status']
train = train.drop(cats_to_drop, axis=1)
 train = train.drop('genres_tv_movie', axis=1)
 test = test.drop(cats_to_drop,axis=1 )<data_type_conversions>
def get_net() :
net = EfficientNet.from_pretrained('efficientnet-b2')
 net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)
 return net
net = get_net().cuda()
train["revenue"]=np.log(train["revenue"] ).astype('float')
 train["budget"]=np.log(train["budget"]+0.1 ).astype('float')
 test["budget"]=np.log(test["budget"]+0.1 ).astype('float' )<feature_engineering>
class TrainGlobalConfig:
num_workers = 4
batch_size = 22
n_epochs = 3
lr = 0.001
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.6,
patience=1,
verbose=False,
threshold=0.001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-9,
eps=1e-09 
)
train.loc[train["runtime"].isnull() ,"runtime"]=train["runtime"].mode() [0]
train.loc[train["rating"].isnull() ,"rating"]=train["rating"].mode() [0]
train.loc[train["totalVotes"].isnull() ,"totalVotes"]=train["totalVotes"].mode() [0]
test.loc[test["runtime"].isnull() ,"runtime"]=train["runtime"].mode() [0]
test.loc[test["rating"].isnull() ,"rating"]=train["runtime"].mode() [0]
test.loc[test["totalVotes"].isnull() ,"totalVotes"]=train["totalVotes"].mode() [0]
test.loc[test["release_Year"].isnull() ,"release_Year"]=train["release_Year"].mode() [0]
test.loc[test["release_Month"].isnull() ,"release_Month"]=train["release_Month"].mode() [0]
test.loc[test["release_Week"].isnull() ,"release_Week"]=train["release_Week"].mode() [0]
test.loc[test["release_Day"].isnull() ,"release_Day"]=train["release_Day"].mode() [0]
test.loc[test["release_Dayofweek"].isnull() ,"release_Dayofweek"]=train["release_Dayofweek"].mode() [0]
test.loc[test["release_Dayofyear"].isnull() ,"release_Dayofyear"]=train["release_Dayofyear"].mode() [0]<normalization>
def run_training() :
device = torch.device('cuda:0')
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
batch_size=TrainGlobalConfig.batch_size,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.load(".. /input/alaska2-checkpoint/last-checkpoint.bin")
 fitter.fit(train_loader, val_loader )
X = train.drop(['revenue'], axis=1)
 y = train['revenue']
X_test = test
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 repeated_folds = RepeatedStratifiedKFold(n_splits=10, n_repeats=20, random_state=42)
<split>
!nvidia-smi
def train_model(X, X_test, y, params, folds, model_type='lgb', plot_feature_importance=False, averaging='usual', model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.loc[train_index], X.loc[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,
train_data,
num_boost_round=20000,
valid_sets = [train_data, valid_data],
verbose_eval=1000,
early_stopping_rounds = 200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_train.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_train.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict_proba(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict_proba(X_test)[:, 1]
if model_type == 'glm':
model = sm.GLM(y_train, X_train, family=sm.families.Binomial())
 model_results = model.fit()
model_results.predict(X_test)
 y_pred_valid = model_results.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model_results.predict(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=20000, learning_rate=0.1, loss_function='Logloss', eval_metric='AUC', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict_proba(X_valid)
 y_pred = model.predict_proba(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid))
if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importance()
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction, scores
else:
return oof, prediction, scores<init_hyperparams>
run_training()
params = {'num_leaves': 16,
'min_data_in_leaf': 2,
'objective': 'regression',
'max_depth': 20,
'learning_rate': 0.008,
'boosting': 'gbdt',
'bagging_freq': 5,
'feature_fraction': 0.82,
'bagging_seed': 11,
'reg_alpha': 1.7,
'reg_lambda': 6,
'random_state': 42,
'metric': 'mse',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01,
'min_child_weight': 10,
'num_threads': 8}
oof_lgb, prediction_lgb, scores = train_model(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True )<train_model>
file = open('.. /input/alaska2-checkpoint/log.txt', 'r')
 for line in file.readlines() :
print(line[:-1])
 file.close()
xgb_params = {'eta': 0.1, 'max_depth': 3, 'subsample': 0.9, 'colsample_bytree': 0.9,
'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True, 'nthread': 8}
oof_xgb, prediction_xgb, scores = train_model(X, X_test, y, params=xgb_params, folds=folds, model_type='xgb' )<save_to_csv>
checkpoint = torch.load('.. /input/alaska2-checkpoint/last-checkpoint.bin')
 net.load_state_dict(checkpoint['model_state_dict']);
net.eval() ;
submission = pd.read_csv(f'{PATH}sample_submission.csv')
 submission['revenue'] = np.exp(0.50*(prediction_lgb+prediction_xgb))
submission.to_csv(f'submission.csv', index=False )<set_options>
class DatasetSubmissionRetriever(Dataset):
def __init__(self, image_names, transforms=None):
super().__init__()
self.image_names = image_names
self.transforms = transforms
def __getitem__(self, index: int):
image_name = self.image_names[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/Test/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image_name, image
def __len__(self)-> int:
return self.image_names.shape[0]
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
py.init_notebook_mode(connected=True)
<load_from_csv>
results = []
for mode in range(0, 4):
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')]),
transforms=get_test_transforms(mode), 
)
 data_loader = DataLoader(
 dataset,
batch_size=8,
shuffle=False,
num_workers=2,
drop_last=False, 
)
 result = {'Id': [], 'Label': []}
for step,(image_names, images)in enumerate(data_loader):
print(step, end='\r')
 y_pred = net(images.cuda())
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
result['Id'].extend(image_names)
 result['Label'].extend(y_pred)
 results.append(result )
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )<define_variables>
submissions = []
for mode in range(0,4):
submission = pd.DataFrame(results[mode])
 submissions.append(submission )
for i, e in enumerate(train['belongs_to_collection'][:5]):
print(i, e )<count_values>
for mode in range(0,4):
submissions[mode].to_csv(f'submission_{mode}.csv', index=False )
<feature_engineering><EOS> 
submissions[0]['Label'] =(submissions[0]['Label']*3 + submissions[1]['Label'] + submissions[2]['Label'] + submissions[3]['Label'])/ 6
submissions[0].to_csv(f'submission.csv', index=False )