Kaggle competition: Deepfake Detection Challenge
<SOS> metric: LogLoss Kaggle data source: deepfake-detection-challenge<load_from_csv>
%matplotlib inline
warnings.filterwarnings("ignore" )
train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv' )<define_variables>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 frame_h = 5
frame_l = 5
len(test_videos )
columns = [col for col in train.columns.to_list() if col not in ['id','target']]<prepare_x_and_y>
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )
data=train[columns]
target=train['target']<split>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008,0.009,0.01,0.012,0.014,0.016,0.018, 0.02]),
'n_estimators': 4000,
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20]),
'random_state': trial.suggest_categorical('random_state', [24, 48,2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),
}
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<init_hyperparams>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
Best_trial= {'lambda': 0.0042687338951820425,
'alpha': 6.2637008222060935,
'colsample_bytree': 0.4,
'subsample': 0.6,
'n_estimators': 4000,
'learning_rate': 0.01,
'max_depth': 11,
'random_state': 2020,
'min_child_weight': 171,
'tree_method':'gpu_hist'
}<train_model>
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
preds = np.zeros(test.shape[0])
 kf = KFold(n_splits=5,random_state=48,shuffle=True)
 rmse=[]
models = []
n=0
for trn_idx, test_idx in kf.split(train[columns],train['target']):
X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]
y_tr,y_val=train['target'].iloc[trn_idx],train['target'].iloc[test_idx]
model = xgb.XGBRegressor(**Best_trial)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100,verbose=False)
 preds+=model.predict(test[columns])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
models.append(model)
 print(n+1,rmse[n])
 n+=1<save_to_csv>
input_size = 224
sub['target']=preds
sub.to_csv('submission.csv', index=False )<import_modules>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
import matplotlib.pyplot as plt<load_from_csv>
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )
train_df = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv", index_col=["id"])
 test_df = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 X = train_df.iloc[:, :-1].to_numpy()
y = train_df.iloc[:, -1].to_numpy()
X_test = test_df.to_numpy()<train_model>
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint
def train(model):
X_train, X_test, y_train, y_test = train_test_split(X, y.flatten() , test_size=0.1, random_state=156)
 y_train = y_train.reshape(-1, 1)
 y_test = y_test.reshape(-1, 1)
 model = model.fit(X_train, y_train, early_stopping_rounds=100, verbose=False, eval_set=[(X_test, y_test)])
 score = mean_squared_error(model.predict(X_train), y_train, squared=False)
 print(score)
 return model<compute_train_metric>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481
def objectiveXGB(trial: Trial, X, y, test):
param = {
"n_estimators" : trial.suggest_int('n_estimators', 500, 4000),
'max_depth':trial.suggest_int('max_depth', 8, 16),
'min_child_weight':trial.suggest_int('min_child_weight', 1, 300),
'gamma':trial.suggest_int('gamma', 1, 3),
'learning_rate': 0.01,
'colsample_bytree':trial.suggest_discrete_uniform('colsample_bytree',0.5, 1, 0.1),
'nthread' : -1,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'subsample': trial.suggest_categorical('subsample', [0.6,0.7,0.8,1.0]),
'random_state': 42
}
X_train, X_test, y_train, y_test = train_test_split(X, y.flatten() , test_size=0.1)
 y_train = y_train.reshape(-1, 1)
 y_test = y_test.reshape(-1, 1)
 model = xgb.XGBRegressor(**param)
 xgb_model = model.fit(X_train, y_train, verbose=False, eval_set=[(X_test, y_test)])
 score = mean_squared_error(xgb_model.predict(X_test), y_test, squared=False)
 return score<train_on_grid>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
study = optuna.create_study(direction='minimize',sampler=TPESampler())
 study.optimize(lambda trial : objectiveXGB(trial, X, y, X_test), n_trials=50)
 print('Best trial: score {},
params {}'.format(study.best_trial.value,study.best_trial.params))
best_param = study.best_trial.params
xgbReg = train(xgb.XGBRegressor(**best_param, tree_method='gpu_hist', random_state=42, predictor='gpu_predictor', learning_rate=0.01, nthread=-1)) 
<compute_train_metric>
predictions = predict_on_video_set(test_videos, num_workers=4 )
def objectiveLGBM(trial: Trial, X, y, test):
param = {
'objective': 'regression',
'metric': 'root_mean_squared_error',
'verbosity': -1,
'boosting_type': 'gbdt',
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'num_leaves': trial.suggest_int('num_leaves', 2, 512),
'learning_rate': 0.01,
'n_estimators': trial.suggest_int('n_estimators', 700, 3000),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'device':"gpu",
'gpu_use_dp':True
}
X_train, X_test, y_train, y_test = train_test_split(X, y.flatten() , test_size=0.1)
 lgbm_regr = LGBMRegressor(**param)
 lgbm_regr = lgbm_regr.fit(X_train, y_train, verbose=False)
 score = mean_squared_error(lgbm_regr.predict(X_test), y_test, squared=False)
 return score<train_on_grid>
submission_df_resnext = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_resnext.to_csv("submission_resnext.csv", index=False )
study = optuna.create_study(direction='minimize',sampler=TPESampler())
 study.optimize(lambda trial : objectiveLGBM(trial, X, y, X_test), n_trials=20)
 print('Best trial: score {},
params {}'.format(study.best_trial.value,study.best_trial.params))
best_param2 = study.best_trial.params
lgbm = LGBMRegressor(**best_param2, device="gpu",gpu_use_dp=True, objective='regression', metric='root_mean_squared_error', learning_rate= 0.01, boosting_type='gbdt')
<train_on_grid>
!pip install.. /input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet
final_model = xgb.XGBRegressor(n_estimators= 2000, max_depth= 16,tree_method='gpu_hist', predictor='gpu_predictor')
 sgd = SGDRegressor(max_iter=1000)
 hgb = HistGradientBoostingRegressor(max_depth=3, min_samples_leaf=1)
 cat = CatBoostRegressor(task_type="GPU", verbose=False)
 estimators = [
lgbm, cat, sgd, hgb, xgbReg
]
S_train, S_test = stacking(estimators, X, y, X_test, regression=True, metric=mean_squared_error, n_folds=5,
shuffle=False, random_state=0, verbose=2)
 final_model.fit(S_train, y)
 print(mean_squared_error(final_model.predict(S_train), y, squared=False))<save_to_csv>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )
submission = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 y_hat = final_model.predict(S_test)
 submission["target"] = y_hat
submission[["target"]].to_csv("/kaggle/working/submission_stacking.csv")
 joblib.dump(final_model, '/kaggle/working/skacking.pkl' )<save_to_csv>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu
submission = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 lgbm = LGBMRegressor(**best_param2, device="gpu",gpu_use_dp=True, objective='regression', learning_rate= 0.01, metric='root_mean_squared_error', boosting_type='gbdt')
 lgbm = lgbm.fit(X, y, verbose=False)
 y_hat = lgbm.predict(submission.to_numpy())
 print(mean_squared_error(lgbm.predict(X), y, squared=False))
submission["target"] = y_hat
submission[["target"]].to_csv("/kaggle/working/submission_lgbm.csv")
 joblib.dump(lgbm, '/kaggle/working/lgbm.pkl' )<save_to_csv>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
submission = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 params = {'n_estimators': 3520, 'max_depth': 11, 'min_child_weight': 231, 'gamma': 2, 'colsample_bytree': 0.7, 'lambda': 0.014950936465569798, 'alpha': 0.28520156840812494, 'subsample': 0.6}
xgbReg = train(xgb.XGBRegressor(**params, tree_method='gpu_hist', random_state=42, predictor='gpu_predictor', learning_rate=0.01, nthread=-1))
y_hat = xgbReg.predict(submission.to_numpy())
 print(mean_squared_error(xgbReg.predict(X), y, squared=False))
submission["target"] = y_hat
submission[["target"]].to_csv("/kaggle/working/submission_xgb.csv")
 joblib.dump(xgbReg, '/kaggle/working/xgb_reg.pkl' )<set_options>
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
sns.set(font_scale=1.4)
 warnings.filterwarnings("ignore")
<compute_test_metric>
input_size = 150
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )<categorify>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
def add_pca(train_df, test_df, cols, n_comp=20, fit_test = True, prefix='pca_', fit_test_first=False):
pca = PCA(n_components=n_comp, random_state=42)
 pca_titles = [prefix+'_pca_'+str(x)for x in range(n_comp)]
temp_train = train_df.copy()
temp_test = test_df.copy()
for c in cols:
fv = temp_train[c].mean()
temp_train[c] = temp_train[c].fillna(value=fv)
 fv = temp_test[c].mean()
temp_test[c] = temp_test[c].fillna(value=fv)
 for p in pca_titles:
train_df[p] = 0.0
test_df[p] = 0.0
if fit_test==True:
pca_data = pd.concat([temp_train[cols], temp_test[cols]], axis=0)
 if fit_test_first==True:
pca.fit(pca_data[len(train_df):])
 pca_data = pca.transform(pca_data)
 else:
pca_data = pca.fit_transform(pca_data)
 train_df.loc[:, pca_titles] = pca_data[0:len(train_df)]
test_df.loc[:, pca_titles] = pca_data[len(train_df):]
return train_df, test_df, pca_titles
def split_distributions(train_df, test_df, cols, target, n_comp=20, prefix='gm', add_labels=False):
gm = GaussianMixture(n_components=n_comp,covariance_type='full', tol=0.001, reg_covar=1e-06, max_iter=100, n_init=1,
init_params='kmeans', weights_init=None, means_init=None, precisions_init=None,
random_state=42, warm_start=False, verbose=0, verbose_interval=10)
 temp_train = train_df.copy()
temp_test = test_df.copy()
gm_data = pd.concat([temp_train[[cols]], temp_test[[cols]]], axis=0 ).reset_index(drop=True)
 gm.fit(gm_data)
 gm_data['labels'] = gm.predict(gm_data)
 if add_labels:
gm_titles = [prefix+str(x)for x in range(n_comp)]
train_df[gm_titles]=0
test_df[gm_titles]=0
dummies = pd.get_dummies(gm_data['labels'], prefix=prefix)
 train_df.loc[:, gm_titles] = dummies[0:len(train_df)].values
test_df.loc[:, gm_titles] = dummies[len(train_df):].values
else:
gm_titles = []
train_df[prefix+'_label'] = gm.labels_[0:len(train_df)]
test_df[prefix+'_label'] = gm.labels_[len(train_df):]
means = train_df.groupby([prefix+'_label'])[target].mean()
train_df['menc_'+prefix+'_label'] = train_df[prefix+'_label'].map(means)
 test_df['menc_'+prefix+'_label'] = test_df[prefix+'_label'].map(means)
 gm_titles+=[prefix+'_label', 'menc_'+prefix+'_label']
return train_df, test_df, gm_titles
def assign_fold(df, label_column,fold_column, NFOLDS=5):
skf = StratifiedKFold(n_splits=NFOLDS)
 df[fold_column]=0
f=0
for trn_idx, val_idx in skf.split(df, df[label_column]):
df.loc[val_idx, 'fold']=f
f+=1
df[fold_column].value_counts()
return df
def st_scale(train_df, test_df, cols):
StSc = StandardScaler()
train_df[cols] = StSc.fit_transform(train_df[cols])
 test_df[cols] = StSc.transform(test_df[cols])
 return train_df, test_df
def seed_everything(seed=1234):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed )<load_from_csv>
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-kernel-data/model_50epochs_lr0001_patience5_factor01_batchsize32.pth'))
net.append(model )
PATH = '/kaggle/input/tabular-playground-series-jan-2021/'
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv')
 submission = pd.read_csv(PATH+'sample_submission.csv')
 FT_COLS = [x for x in train.columns if 'cont' in x]
TARGET='target'
print(train.shape)
 train.head(10 )<define_variables>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481
original_feature_dict = {'cont1': 5,
'cont2': 10,
'cont3': 8,
'cont4': 8,
'cont5': 5,
'cont6': 6,
'cont7': 4,
'cont8': 3,
'cont9': 7,
'cont10':3,
'cont11': 4,
'cont12': 2,
'cont13': 3,
'cont14': 5,}
pca_dict = {
'extra_pca_0': 4,
'extra_pca_1': 1,
'extra_pca_2': 1,
'extra_pca_3': 2,
'extra_pca_4': 1,
'extra_pca_5': 1 }
MULTIPLIER = 3.0
original_feature_dict = {a:int(b*MULTIPLIER)for(a,b)in original_feature_dict.items() }
ft_dict = {**original_feature_dict, **pca_dict}
ft_dict<split>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
mixture_title_cols=[]
for f in FT_COLS+pca_titles2:
train, test, titles = split_distributions(train, test,f,TARGET, n_comp=ft_dict[f], prefix=f+'_dim', add_labels=True)
 mixture_title_cols+=titles
mixture_value_cols = []
for count,f in enumerate(FT_COLS+pca_titles2):
for d in range(ft_dict[f]):
t_median = train[f][train[f+'_dim'+str(d)]==1].median()
t_std = train[f][train[f+'_dim'+str(d)]==1].std()
train[f+'_dim'+str(d)+'_dist'] = np.where(train[f+'_dim'+str(d)]==1, 
(train[f] - t_median)/t_std, np.nan)
 test[f+'_dim'+str(d)+'_dist'] = np.where(test[f+'_dim'+str(d)]==1, 
(test[f] - t_median)/t_std, np.nan)
 mixture_value_cols+=[f+'_dim'+str(d)+'_dist']<define_variables>
speed_test = False
print('Total Original Features', len(FT_COLS))
print('Total Submixtures Labels', len(mixture_title_cols))
print('Total Submixtures Values', len(mixture_value_cols))
print('Total Feature Columns', len(FT_COLS)+len(mixture_title_cols)+len(mixture_value_cols))<data_type_conversions>
%%time
model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )
NAN_VALUE = 0.0
for d in mixture_value_cols:
train[d] = train[d].fillna(value=NAN_VALUE)
 test[d] = test[d].fillna(value=NAN_VALUE )<normalization>
submission_df_xception = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_xception.to_csv("submission_xception.csv", index=False )
SCALE = True
if SCALE:
train, test = st_scale(train, test, FT_COLS)
 for f in FT_COLS:
train[f] = np.clip(train[f], -2, 2)
 test[f] = np.clip(test[f], -2, 2)
 SCALE_DISTS = True
if SCALE_DISTS:
for d in mixture_value_cols:
TEMP_MAX = np.abs(train[d] ).max()
train[d] = train[d] / TEMP_MAX
test[d] = test[d] / TEMP_MAX<feature_engineering>
submission_df = pd.DataFrame({"filename": test_videos})
 submission_df["label"] = 0.53*submission_df_resnext["label"] + 0.49*submission_df_xception["label"]
<compute_train_metric><EOS> 
submission_df.to_csv("submission.csv", index=False )