Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<predict_on_test>
!pip install xgbfir
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5<predict_on_test>
warnings.filterwarnings('ignore' )
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<define_variables>
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 gc.collect()
speed_test = False<predict_on_test>
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True )
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )<predict_on_test>
y_train = train['isFraud'].copy()
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
gc.collect()
submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df.to_csv("submission.csv", index=False )<install_modules>
del train_transaction, train_identity, test_transaction, test_identity
gc.collect()
!pip install.. /input/pytorchcv/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet<set_options>
train['nulls'] = train.isna().sum(axis=1)
 test['nulls'] = test.isna().sum(axis=1 )
%matplotlib inline
warnings.filterwarnings("ignore" )<define_variables>
train['D9'] =(train['TransactionDT']%(3600*24)/3600//1)/24.0
test['D9'] =(test['TransactionDT']%(3600*24)/3600//1)/24.0
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<set_options>
train_cols = [c for c in train.columns if c not in list(set(['TransactionDT', 'id_30','id_31','id_33', 'id_35', "id_32", "D15", "D10", "D4", "C13", "D11", "C9", "C11", 'V300', 'V309', 'V111', 'C3', 'V124', 'V106', 'V125', 'V315', 'V134', 'V102', 'V123', 'V316', 'V113', 'V136', 'V305', 'V110', 'V299', 'V289', 'V286', 'V318', 'V103', 'V304', 'V116', 'V298', 'V284', 'V293', 'V137', 'V295', 'V301', 'V104', 'V311', 'V115', 'V109', 'V119', 'V321', 'V114', 'V133', 'V122', 'V319', 'V105', 'V112', 'V118', 'V117', 'V121', 'V108', 'V135', 'V320', 'V303', 'V297', 'V120',"isFraud", "id_32", "D15", 'D9','D15', 'V107', 'V28', 'V305', 'V68', 'V117', 'V119', 'V241', 'V27', 'V65', 'V89', 'V122', 'V88'])) ]
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<load_pretrained>
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 gc.collect()
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
cat_cols = train.select_dtypes("object" ).columns.tolist()
frames_per_video = 20
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )<define_variables>
%%time
for f in cat_cols:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))
gc.collect()
input_size = 150<normalization>
n_fold = 6
folds = KFold(n_fold, shuffle=False, random_state=11 )
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
params = {'tree_method': 'hist',
'silent': 1,
'colsample_bytree': 0.8999504397295506,
'subsample': 0.7446134812140273,
'learning_rate': 0.05,
'max_leaves': 72,
'objective': 'binary:logistic',
'max_depth': 0,
'reg_alpha': 0.7726783188295172,
'min_child_weight': 2,
'eval_metric': 'auc',
'grow_policy':'lossguide'}
params
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))
class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-xception/modelv2.pth'))
net.append(model )<predict_on_test>
%%time
oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance_df = pd.DataFrame()
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(train, y_train)) :
trn_x, trn_y = train[train_cols].iloc[trn_idx], y_train.iloc[trn_idx]
val_x, val_y = train[train_cols].iloc[val_idx], y_train.iloc[val_idx]
gc.collect()
dtrain = xgb.DMatrix(trn_x, trn_y, feature_names=trn_x.columns)
 dval = xgb.DMatrix(val_x, val_y, feature_names=val_x.columns)
 gc.collect()
clf = xgb.train(params=params, dtrain=dtrain, num_boost_round=1500, evals=[(dtrain, "Train"),(dval, "Val")],
verbose_eval= 250, early_stopping_rounds=100)
 gc.collect()
oof_preds[val_idx] = clf.predict(xgb.DMatrix(val_x))
sub_preds += clf.predict(xgb.DMatrix(test[train_cols])) / folds.n_splits
gc.collect()
xgbfir.saveXgbFI(clf, feature_names=trn_x.columns, OutputXlsxFile='ieee_xgbfir_%sFold.xlsx'%str(n_fold+1), MaxInteractionDepth=9, MaxHistograms=15)
 gc.collect()
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = pd.DataFrame.from_dict(data=clf.get_fscore() , orient="index", columns=["FScore"])["FScore"].index
fold_importance_df["fscore"] = pd.DataFrame.from_dict(data=clf.get_fscore() , orient="index", columns=["FScore"])["FScore"].values
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 gc.collect()
print('
Fold %2d AUC %.6f & std %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx]), np.std([oof_preds[val_idx]])))
 gc.collect()
print('
CV AUC score %.6f & std %.6f' %(roc_auc_score(y_train, oof_preds), np.std(( oof_preds))))
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5<predict_on_test>
gc.collect()
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<define_variables>
feature_importance_df.groupby(["feature"])["fscore",].mean().sort_values("fscore", ascending=False )
speed_test = False<predict_on_test>
sample_submission['isFraud'] = sub_preds
gc.collect()
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )<predict_on_test>
score = roc_auc_score(y_train, oof_preds)
 score
%%time
model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
sample_submission.to_csv('sub_ieee_sample_xgb_%sfold_%.6f.csv'%(n_fold, score))
submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df.to_csv("submission.csv", index=False )<set_options>
oof_train = pd.DataFrame(oof_preds, columns=["oof_preds"], index=train.index)
 oof_train.to_csv("ieee_xgb_%sfold_oof_%.6f.csv" %(n_fold, score))
oof_train.head()
%matplotlib inline 
<define_variables>
xl = pd.ExcelFile("ieee_xgbfir_4Fold.xlsx" )
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 frame_h = 5
frame_l = 5
len(test_videos )<import_modules>
xl.sheet_names