Kaggle competition: IEEE-CIS Fraud Detection
for df in [train, test]:
df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['date'].dt.dayofweek
df['sess_date_hours'] = df['date'].dt.hour
df['sess_date_dom'] = df['date'].dt.day<define_variables>
def memory_usage_mb(df, *args, **kwargs):

 return df.memory_usage(*args, **kwargs ).sum() / 1024**2
def reduce_memory_usage(df, deep=True, verbose=True):
numeric2reduce = ["int16", "int32", "int64", "float64"]
start_mem = 0
if verbose:
start_mem = memory_usage_mb(df, deep=deep)
 for col, col_type in df.dtypes.iteritems() :
best_type = None
if col_type in numeric2reduce:
downcast = "integer" if "int" in str(col_type)else "float"
df[col] = pd.to_numeric(df[col], downcast=downcast)
 best_type = df[col].dtype.name
if verbose and best_type is not None and best_type != str(col_type):
print(f"Column '{col}' converted from {col_type} to {best_type}")
 if verbose:
end_mem = memory_usage_mb(df, deep=deep)
 diff_mem = start_mem - end_mem
percent_mem = 100 * diff_mem / start_mem
print(f"Memory usage decreased from"
f" {start_mem:.2f}MB to {end_mem:.2f}MB"
f"({diff_mem:.2f}MB, {percent_mem:.2f}% reduction)")
 return df
excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]<feature_engineering>
train_identity_data = pd.read_csv('.. /input/train_identity.csv')
 train_transaction_data = pd.read_csv('.. /input/train_transaction.csv' )
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )<prepare_x_and_y>
pd.set_option('display.max_columns', 500)
 train_transaction_data.head(5 )
folds = get_folds(df=train, n_splits=5)
 train_features = [_f for _f in train.columns if _f not in excluded_features]
print(train_features)
 importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=100,
eval_metric='rmse' 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<prepare_output>
train_transaction_data['Transaction_dow'] = np.floor(( train_transaction_data['TransactionDT'] /(3600 * 24)- 1)% 7)
 train_transaction_data['Transaction_hour'] = np.floor(train_transaction_data['TransactionDT'] / 3600)% 24
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds<groupby>
na_columns = train_identity_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0]/train_identity_data.shape[0] )
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()<groupby>
import seaborn as sns
import matplotlib.pyplot as plt
%%time
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
pd.set_option('display.max_rows', 500)
 na_columns = train_transaction_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0]/train_transaction_data.shape[0] )
trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()
full_data.shape<groupby>
identity_data_columns = train_identity_data.columns
numericCols = train_identity_data._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
print('The categorical columns in identity data are: ',categoricalCols)
 train_identity_data[categoricalCols] = train_identity_data[categoricalCols].replace({ np.nan:'missing'})
 train_identity_data[numericCols] = train_identity_data[numericCols].replace({ np.nan:-1})
 colstoplot = ['id_01','id_02','id_03','id_04','id_05','id_06','id_07','id_08','id_09','id_10',
'id_11','id_13','id_14','id_17','id_18','id_19','id_20','id_21','id_22','id_24','id_25','id_26','id_32']
for i in colstoplot:
plt.figure(figsize=(16, 16))
ax = sns.distplot(eval('train_identity_data["'+i+'"]'))
ax.set(ylabel="Distribution")
 ax.set_title([i] )
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)} )<prepare_output>
train_transaction_data[categoricalCols] = train_transaction_data[categoricalCols].replace({ np.nan:'missing'})
 train_transaction_data[numericCols] = train_transaction_data[numericCols].replace({ np.nan:-1} )
sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
print('The average transaction amount for non fraudulent transactions is: ',
np.mean(train_transaction_data.loc[train_transaction_data["isFraud"]==0]['TransactionAmt']))
print('The average transaction amount for fraudulent transactions is: ',
np.mean(train_transaction_data.loc[train_transaction_data["isFraud"]==1]['TransactionAmt']))
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()<prepare_x_and_y>
maxCardData = {}
minCardData = {}
meanCardData = {}
for i,i_card in enumerate(cardTypes):
cardData = eval('train_transaction_data.loc[train_transaction_data["card4"]=="'+i_card+'"]')
 maxCardData[i_card] = np.max(cardData['TransactionAmt'])
 minCardData[i_card] = np.min(cardData['TransactionAmt'])
 meanCardData[i_card] = np.mean(cardData['TransactionAmt'] )
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 oof_preds = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=1000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds[oof_preds < 0] = 0
_preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_preds += _preds / len(folds)
 mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5<save_to_csv>
del fraudDataTransaction,normalDataTransaction,productData,cardData
sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('new_test.csv', index=True )<set_options>
train_identity_data.loc[train_identity_data['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
train_identity_data.loc[train_identity_data['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
train_identity_data.loc[train_identity_data['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
train_identity_data.loc[train_identity_data['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
train_identity_data.loc[train_identity_data['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'
%matplotlib inline 
<define_variables>
train_identity_data['device_name'] = train_identity_data['DeviceInfo'].str.split('/', expand=True)[0]
train_identity_data.loc[train_identity_data['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
train_identity_data.loc[train_identity_data['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
train_identity_data.loc[train_identity_data['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
train_identity_data.loc[train_identity_data['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
train_identity_data.loc[train_identity_data['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
train_identity_data.loc[train_identity_data['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
train_identity_data.loc[train_identity_data['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
train_identity_data.loc[train_identity_data['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
train_identity_data.loc[train_identity_data['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
train_identity_data.loc[train_identity_data['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
train_identity_data.loc[train_identity_data['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
train_identity_data.loc[train_identity_data['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
train_identity_data.loc[train_identity_data['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
train_identity_data.loc[train_identity_data.device_name.isin(train_identity_data.device_name.value_counts() [train_identity_data.device_name.value_counts() < 200].index), 'device_name'] = "Others"
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 frame_h = 5
frame_l = 5
len(test_videos )<import_modules>
raw_train_data = pd.merge(train_transaction_data, train_identity_data, on='TransactionID', how='left')
 del train_transaction_data,train_identity_data
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )<set_options>
raw_train_data_columns = raw_train_data.columns
numericCols = raw_train_data._get_numeric_data().columns
categoricalCols = list(set(raw_train_data_columns)- set(numericCols))
print('The categorical columns in training data are: ',categoricalCols)
 raw_train_data[categoricalCols] = raw_train_data[categoricalCols].replace({ np.nan:'missing'})
 raw_train_data[numericCols] = raw_train_data[numericCols].replace({ np.nan:-1} )
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu<load_pretrained>
for i_var in to_drop:
if i_var in goodNumericVars:
goodNumericVars.remove(i_var )
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
del to_drop,corrThresh,upper,correlationMatrix,na_vals
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )<define_variables>
variables = list(categoricalCols)
 na_vals = np.sum(raw_train_data.loc[:,variables]=='missing')/raw_train_data.shape[0]
goodCategoricalVars = []
for i_var in variables:
if na_vals[i_var] < 1:
goodCategoricalVars.append(i_var )
input_size = 224<normalization>
featuresToUse = goodNumericVars+goodCategoricalVars
train_data = raw_train_data.loc[:,featuresToUse]
target_data = raw_train_data['isFraud']
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
train_data['TransactionAmt_to_mean_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_mean_C13'] = train_data['TransactionAmt'] / train_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 train_data['TransactionAmt_to_std_card1'] = train_data['TransactionAmt'] / train_data.groupby(['card1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card4'] = train_data['TransactionAmt'] / train_data.groupby(['card4'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_card5'] = train_data['TransactionAmt'] / train_data.groupby(['card5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_addr1'] = train_data['TransactionAmt'] / train_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_id31'] = train_data['TransactionAmt'] / train_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_devicename'] = train_data['TransactionAmt'] / train_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C1'] = train_data['TransactionAmt'] / train_data.groupby(['C1'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C3'] = train_data['TransactionAmt'] / train_data.groupby(['C3'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_to_std_C5'] = train_data['TransactionAmt'] / train_data.groupby(['C5'])['TransactionAmt'].transform('std')
 train_data['TransactionAmt_decimal'] =(( train_data['TransactionAmt'] - train_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )<load_pretrained>
train_data['card1_card2'] = train_data['card1'].astype(str)+ '_' + train_data['card2'].astype(str)
 train_data['addr1_dist1'] = train_data['addr1'].astype(str)+ '_' + train_data['dist1'].astype(str)
 train_data['card1_addr1'] = train_data['card1'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card1_addr2'] = train_data['card1'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card2_addr1'] = train_data['card2'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card2_addr2'] = train_data['card2'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['card4_addr1'] = train_data['card4'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card4_addr2'] = train_data['card4'].astype(str)+ '_' + train_data['addr2'].astype(str)
 train_data['DeviceInfo_P_emaildomain'] = train_data['DeviceInfo'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['P_emaildomain_addr1'] = train_data['P_emaildomain'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['id01_addr1'] = train_data['id_01'].astype(str)+ '_' + train_data['addr1'].astype(str)
 train_data['card1_P_emaildomain'] = train_data['card1'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['card5_P_emaildomain'] = train_data['card5'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['card2_P_emaildomain'] = train_data['card2'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['card4_P_emaildomain'] = train_data['card4'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str)
 train_data['id_01_P_emaildomain'] = train_data['id_01'].astype(str)+ '_' + train_data['P_emaildomain'].astype(str )
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint<predict_on_test>
scale_pos_weight = np.sqrt(len(target_data.loc[target_data==0])/len(target_data.loc[target_data==1]))
del raw_train_data
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481<predict_on_test>
test_identity_data = pd.read_csv('.. /input/test_identity.csv')
 test_transaction_data = pd.read_csv('.. /input/test_transaction.csv' )
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<predict_on_test>
na_columns = test_transaction_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0]/test_transaction_data.shape[0] )
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
test_transaction_data['Transaction_dow'] = np.floor(( test_transaction_data['TransactionDT'] /(3600 * 24)- 1)% 7)
 test_transaction_data['Transaction_hour'] = np.floor(test_transaction_data['TransactionDT'] / 3600)% 24
transaction_data_columns = test_transaction_data.columns
numericCols = test_transaction_data._get_numeric_data().columns
categoricalCols = list(set(transaction_data_columns)- set(numericCols))
test_transaction_data[categoricalCols] = test_transaction_data[categoricalCols].replace({ np.nan:'missing'})
 test_transaction_data[numericCols] = test_transaction_data[numericCols].replace({ np.nan:-1} )
submission_df_resnext = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_resnext.to_csv("submission_resnext.csv", index=False )<install_modules>
identity_data_columns = test_identity_data.columns
numericCols = test_identity_data._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
test_identity_data[categoricalCols] = test_identity_data[categoricalCols].replace({ np.nan:'missing'})
 test_identity_data[numericCols] = test_identity_data[numericCols].replace({ np.nan:-1})
 test_identity_data['id_31'] = test_identity_data['id_31'].str.replace('\d+', '')
 test_identity_data.loc[test_identity_data['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
test_identity_data.loc[test_identity_data['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
test_identity_data.loc[test_identity_data['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'
test_identity_data['device_name'] = test_identity_data['DeviceInfo'].str.split('/', expand=True)[0]
test_identity_data.loc[test_identity_data['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
test_identity_data.loc[test_identity_data['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
test_identity_data.loc[test_identity_data['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
test_identity_data.loc[test_identity_data['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
test_identity_data.loc[test_identity_data['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
test_identity_data.loc[test_identity_data['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
test_identity_data.loc[test_identity_data['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
test_identity_data.loc[test_identity_data.device_name.isin(test_identity_data.device_name.value_counts() [test_identity_data.device_name.value_counts() < 200].index), 'device_name'] = "Others"
!pip install.. /input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet<set_options>
raw_test_data = pd.merge(test_transaction_data, test_identity_data, on='TransactionID', how='left')
 transactionID = raw_test_data.loc[:,'TransactionID']
del test_identity_data,test_transaction_data
%matplotlib inline
warnings.filterwarnings("ignore" )<define_variables>
raw_test_data_columns = raw_test_data.columns
numericCols = raw_test_data._get_numeric_data().columns
categoricalCols = list(set(raw_test_data_columns)- set(numericCols))
print('The categorical columns in training data are: ',categoricalCols)
 raw_test_data[categoricalCols] = raw_test_data[categoricalCols].replace({ np.nan:'missing'})
 raw_test_data[numericCols] = raw_test_data[numericCols].replace({ np.nan:-1})
 raw_test_data = reduce_memory_usage(raw_test_data, deep=True, verbose=True )
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<set_options>
test_data = raw_test_data.loc[:,featuresToUse]
test_data['TransactionAmt_to_mean_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_mean_C13'] = test_data['TransactionAmt'] / test_data.groupby(['C13'])['TransactionAmt'].transform('mean')
 test_data['TransactionAmt_to_std_card1'] = test_data['TransactionAmt'] / test_data.groupby(['card1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card4'] = test_data['TransactionAmt'] / test_data.groupby(['card4'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_card5'] = test_data['TransactionAmt'] / test_data.groupby(['card5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_addr1'] = test_data['TransactionAmt'] / test_data.groupby(['addr1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_id31'] = test_data['TransactionAmt'] / test_data.groupby(['id_31'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_devicename'] = test_data['TransactionAmt'] / test_data.groupby(['device_name'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C1'] = test_data['TransactionAmt'] / test_data.groupby(['C1'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C3'] = test_data['TransactionAmt'] / test_data.groupby(['C3'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_to_std_C5'] = test_data['TransactionAmt'] / test_data.groupby(['C5'])['TransactionAmt'].transform('std')
 test_data['TransactionAmt_decimal'] =(( test_data['TransactionAmt'] - test_data['TransactionAmt'].astype(int)) * 1000 ).astype(int )
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<load_pretrained>
test_data['card1_card2'] = test_data['card1'].astype(str)+ '_' + test_data['card2'].astype(str)
 test_data['addr1_dist1'] = test_data['addr1'].astype(str)+ '_' + test_data['dist1'].astype(str)
 test_data['card1_addr1'] = test_data['card1'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card1_addr2'] = test_data['card1'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card2_addr1'] = test_data['card2'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card2_addr2'] = test_data['card2'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['card4_addr1'] = test_data['card4'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card4_addr2'] = test_data['card4'].astype(str)+ '_' + test_data['addr2'].astype(str)
 test_data['DeviceInfo_P_emaildomain'] = test_data['DeviceInfo'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['P_emaildomain_addr1'] = test_data['P_emaildomain'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['id01_addr1'] = test_data['id_01'].astype(str)+ '_' + test_data['addr1'].astype(str)
 test_data['card1_P_emaildomain'] = test_data['card1'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['card5_P_emaildomain'] = test_data['card5'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['card2_P_emaildomain'] = test_data['card2'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['card4_P_emaildomain'] = test_data['card4'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str)
 test_data['id_01_P_emaildomain'] = test_data['id_01'].astype(str)+ '_' + test_data['P_emaildomain'].astype(str )
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
cv = StratifiedKFold(n_splits=5, random_state=123, shuffle=True )
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )<define_variables>
def compute_roc_auc(clf,index):
y_predict = clf.predict_proba(train_data.iloc[index])[:,1]
fpr, tpr, thresholds = roc_curve(target_data.iloc[index], y_predict)
 auc_score = auc(fpr, tpr)
 return fpr, tpr, auc_score
input_size = 150<normalization>
params = {'bagging_fraction': 0.7982116702024386,
'feature_fraction': 0.1785051643813966,
'max_depth': int(49.17611603427576),
'min_child_weight': 3.2852905549011155,
'min_data_in_leaf': int(31.03480802715621),
'n_estimators': 5000,
'num_leaves': int(52.851307790411965),
'reg_alpha': 0.45963319421692145,
'reg_lambda': 0.6591286807489907,
'metric':'auc',
'boosting_type': 'gbdt',
'colsample_bytree':.8,
'subsample':.9,
'min_split_gain':.01,
'max_bin':127,
'bagging_freq':5,
'learning_rate':0.01,
'early_stopping_rounds':100
}
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
fprs_lgb, tprs_lgb, scores_lgb = [], [], []
feature_importances = pd.DataFrame()
feature_importances['feature'] = train_data.columns
predictions = np.zeros(len(test_data))
for(train, test), i in zip(cv.split(train_data, target_data), range(5)) :
lgb_best = LGBMClassifier(boosting = params['boosting_type'],n_estimators = params['n_estimators'],
learning_rate = params['learning_rate'],num_leaves = params['num_leaves'],
colsample_bytree = params['colsample_bytree'],subsample = params['subsample'],
max_depth = params['max_depth'],reg_alpha = params['reg_alpha'],
reg_lambda = params['reg_lambda'],min_split_gain = params['min_split_gain'],
min_child_weight = params['min_child_weight'],max_bin = params['max_bin'],
bagging_freq = params['bagging_freq'],feature_fraction = params['feature_fraction'],
bagging_fraction = params['bagging_fraction'],min_data_in_leaf = params['min_data_in_leaf'],
early_stopping_rounds = params['early_stopping_rounds'])
 lgb_best.fit(train_data.iloc[train,:], target_data.iloc[train],
eval_set = [(train_data.iloc[train,:], target_data.iloc[train]), 
(train_data.iloc[test,:], target_data.iloc[test])],eval_metric='auc',verbose = 200)
 feature_importances['fold_{}'.format(i + 1)] = lgb_best.feature_importances_
_, _, auc_score_train = compute_roc_auc(lgb_best,train)
 fpr, tpr, auc_score = compute_roc_auc(lgb_best,test)
 scores_lgb.append(( auc_score_train, auc_score))
fprs_lgb.append(fpr)
 tprs_lgb.append(tpr)
 predictions += lgb_best.predict_proba(test_data, num_iteration=lgb_best.best_iteration_)[:,1]
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-xception-trained-model/model.pth'))
net.append(model )<predict_on_test>
predictions /= cv.n_splits
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481<predict_on_test>
print('Mean AUC:', np.mean(scores_lgb,axis = 0))
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<predict_on_test>
del train_data,target_data
%%time
model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
data = {'TransactionID':transactionID,'isFraud':predictions}
submissionDF = pd.DataFrame(data)
 submissionDF.to_csv('sample_submission.csv',index=False )