Kaggle competition: Deepfake Detection Challenge
<SOS> metric: LogLoss Kaggle data source: deepfake-detection-challenge<data_type_conversions>
from fastai.vision import *
for col in basement_num_cols:
train[col] = train[col].fillna(0)
 test[col] = test[col].fillna(0 )<define_variables>
train_sample_metadata = pd.read_json('.. /input/deepfake-detection-challenge/train_sample_videos/metadata.json' ).T.reset_index()
train_sample_metadata.columns = ['fname','label','split','original']
train_sample_metadata.head()
masvnr_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"MasVnr"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
masvnr_colums.append(col)
 masvnr_colums = np.array(masvnr_colums)
 masvnr_colums<filter>
fake_sample_df = train_sample_metadata[train_sample_metadata.label == 'FAKE']
real_sample_df = train_sample_metadata[train_sample_metadata.label == 'REAL']
all(train.loc[train['MasVnrArea'].isnull() ].index == train.loc[train['MasVnrType'].isnull() ].index )<data_type_conversions>
train_dir = Path('/kaggle/input/deepfake-detection-challenge/train_sample_videos/')
 test_dir = Path('/kaggle/input/deepfake-detection-challenge/test_videos/')
 train_video_files = get_files(train_dir, extensions=['.mp4'])
 test_video_files = get_files(test_dir, extensions=['.mp4'] )
train['MasVnrArea'] = train['MasVnrArea'].fillna(0)
 test['MasVnrArea'] = test['MasVnrArea'].fillna(0 )<data_type_conversions>
dummy_video_file = train_video_files[0]
train['MasVnrType'] = train['MasVnrType'].fillna('None')
 test['MasVnrType'] = test['MasVnrType'].fillna('None' )<define_variables>
sys.path.insert(0,'/kaggle/working/reader/python')
 set_bridge('torch')
 device = torch.device("cuda" )
pool_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Pool"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
pool_colums.append(col)
 pool_colums = np.array(pool_colums)
 pool_colums<data_type_conversions>
retinaface_stats = tensor([123,117,104] ).to(device)
 def decord_cpu_video_reader(path, freq=None):
video = VideoReader(str(path), ctx=cpu())
 len_video = len(video)
 if freq: t = video.get_batch(range(0, len(video), freq)).permute(0,3,1,2)
 else: t = video.get_batch(range(len_video))
return t, len_video
def get_decord_video_batch_cpu(path, freq=10, sz=640, stats:Tensor=None, device=defaults.device):
"get resized and mean substracted batch tensor of a sampled video(scale of 255)"
t_raw, len_video = decord_cpu_video_reader(path, freq)
 H,W = t_raw.size(2), t_raw.size(3)
 t = F.interpolate(t_raw.to(device ).to(torch.float32),(sz,sz))
if stats is not None: t -= stats[...,None,None]
return t, t_raw,(H, W )
train['PoolQC'] = train['PoolQC'].fillna('NA')
 test['PoolQC'] = test['PoolQC'].fillna('NA' )<data_type_conversions>
sys.path.insert(0,"/kaggle/input/retina-face-2/Pytorch_Retinaface_2/" )
train['Alley'] = train['Alley'].fillna('NA')
 test['Alley'] = test['Alley'].fillna('NA' )<define_variables>
import os
import torch
import torch.backends.cudnn as cudnn
import numpy as np
from data import cfg_mnet, cfg_re50
from layers.functions.prior_box import PriorBox
from utils.nms.py_cpu_nms import py_cpu_nms
import cv2
from models.retinaface import RetinaFace
from utils.box_utils import decode, decode_landm
import time
fireplace_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Fireplace"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
fireplace_colums.append(col)
 fireplace_colums = np.array(fireplace_colums)
 fireplace_colums<data_type_conversions>
def check_keys(model, pretrained_state_dict):
ckpt_keys = set(pretrained_state_dict.keys())
 model_keys = set(model.state_dict().keys())
 used_pretrained_keys = model_keys & ckpt_keys
unused_pretrained_keys = ckpt_keys - model_keys
missing_keys = model_keys - ckpt_keys
print('Missing keys:{}'.format(len(missing_keys)))
 print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))
 print('Used keys:{}'.format(len(used_pretrained_keys)))
 assert len(used_pretrained_keys)> 0, 'load NONE from pretrained checkpoint'
return True
def remove_prefix(state_dict, prefix):

 print('remove prefix '{}''.format(prefix))
f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix)else x
return {f(key): value for key, value in state_dict.items() }
def load_model(model, pretrained_path, load_to_cpu):
print('Loading pretrained model from {}'.format(pretrained_path))
if load_to_cpu:
pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)
 else:
device = torch.cuda.current_device()
pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))
if "state_dict" in pretrained_dict.keys() :
pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')
 else:
pretrained_dict = remove_prefix(pretrained_dict, 'module.')
 check_keys(model, pretrained_dict)
 model.load_state_dict(pretrained_dict, strict=False)
 return model
train['FireplaceQu'] = train['FireplaceQu'].fillna('NA')
 test['FireplaceQu'] = test['FireplaceQu'].fillna('NA' )<data_type_conversions>
cudnn.benchmark = True
train['Fence'] = train['Fence'].fillna('NA')
 test['Fence'] = test['Fence'].fillna('NA' )<define_variables>
def get_model(modelname="mobilenet"):
torch.set_grad_enabled(False)
 cfg = None
cfg_mnet['pretrain'] = False
cfg_re50['pretrain'] = False
if modelname == "mobilenet":
pretrained_path = ".. /input/retina-face-2/Pytorch_Retinaface_2/weights/mobilenet0.25_Final.pth"
cfg = cfg_mnet
if modelname == "resnet50":
pretrained_path = ".. /input/retina-face/Pytorch_Retinaface/weights/Resnet50_Final.pth"
cfg = cfg_re50
net = RetinaFace(cfg=cfg, phase='test')
 net = load_model(net, pretrained_path, False)
 net.eval().to(device)
 return net, cfg
misc_colums = []
regexp = re.compile(r"([-a-zA-Z]+)?"+r"Misc"+r"([-a-zA-Z]+)?")
 for col in train.columns:
if regexp.search(col):
misc_colums.append(col)
 misc_colums = np.array(misc_colums)
 misc_colums<data_type_conversions>
def predict(model, t, sz, cfg,
confidence_threshold = 0.5, top_k = 5, nms_threshold = 0.5, keep_top_k = 5):
"get prediction for a batch t by model with image sz"
resize = 1
scale_rate = 1
im_height, im_width = sz, sz
scale = torch.Tensor([sz, sz, sz, sz])
 scale = scale.to(device)
 locs, confs, landmss = torch.Tensor([]), torch.Tensor([]), torch.Tensor([])
 locs = locs.to(device)
 confs = confs.to(device)
 landmss = landmss.to(device)
 locs_, confs_, landmss_ = model(t)
 locs = torch.cat(( locs, locs_), 0)
 confs = torch.cat(( confs, confs_), 0)
 landmss = torch.cat(( landmss, landmss_), 0)
 bbox_result, landms_result = [], []
priorbox = PriorBox(cfg, image_size=(im_height, im_width))
priors = priorbox.forward()
priors = priors.to(device)
 prior_data = priors.data
for idx in range(t.size(0)) :
loc = locs[idx]
conf = confs[idx]
landms = landmss[idx]
boxes = decode(loc.data.squeeze(0), prior_data, cfg['variance'])
 boxes = boxes * scale / resize
boxes = boxes.cpu().numpy()
scores = conf.squeeze(0 ).data.cpu().numpy() [:, 1]
landms = decode_landm(landms.data.squeeze(0), prior_data, cfg['variance'])
 scale1 = torch.Tensor([t.shape[3], t.shape[2], t.shape[3], t.shape[2],
t.shape[3], t.shape[2], t.shape[3], t.shape[2],
t.shape[3], t.shape[2]])
 scale1 = scale1.to(device)
 landms = landms * scale1 / resize
landms = landms.cpu().numpy()
inds = np.where(scores > confidence_threshold)[0]
boxes = boxes[inds]
landms = landms[inds]
scores = scores[inds]
order = scores.argsort() [::-1][:top_k]
boxes = boxes[order]
landms = landms[order]
scores = scores[order]
dets = np.hstack(( boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)
 keep = py_cpu_nms(dets, nms_threshold)
 dets = dets[keep, :]
landms = landms[keep]
dets = dets[:keep_top_k, :]
landms = landms[:keep_top_k, :]
bbox_result.append(dets[:,:-1])
 landms_result.append(landms)
 return bbox_result, landms_result
train['MiscFeature'] = train['MiscFeature'].fillna('NA')
 test['MiscFeature'] = test['MiscFeature'].fillna('NA' )<drop_column>
%%time
model, cfg = get_model("mobilenet" )
train = train.drop(train.loc[train['Electrical'].isnull() ].index )<feature_engineering>
def bboxes_to_original_scale(bboxes, H, W, sz):

 res = []
for bb in bboxes:
h_scale, w_scale = H/sz, W/sz
orig_bboxes =(bb*array([w_scale, h_scale, w_scale, h_scale])[None,...] ).astype(int)
 res.append(orig_bboxes)
 return res
for i in test['Neighborhood'].unique() :
if test.MSZoning[test['Neighborhood'] == i].isnull().sum() > 0:
test.loc[test['Neighborhood'] == i,'MSZoning'] = \
test.loc[test['Neighborhood'] == i,'MSZoning'].fillna(test.loc[test['Neighborhood'] == i,'MSZoning'].mode() [0] )<data_type_conversions>
def landmarks_to_original_scale(landmarks, H, W, sz):

 res = []
for landms in landmarks:
h_scale, w_scale = H/sz, W/sz
orig_landms =(landms*array([w_scale, h_scale]*5)[None,...] ).astype(int)
 res.append(orig_landms)
 return res
train['LotFrontage'].fillna(train['LotFrontage'].median() , inplace=True)
 test['LotFrontage'].fillna(test['LotFrontage'].median() , inplace=True )<prepare_x_and_y>
from tqdm import tqdm
y_train = train['SalePrice']
train = train.drop('SalePrice',axis=1)
 train = train.drop('Id',axis=1 )<drop_column>
freq = 5
model_args = dict(confidence_threshold = 0.5, top_k = 5, nms_threshold = 0.5, keep_top_k = 5)
 sz = cfg['image_size']
imgnet_stats = [tensor(o)for o in imagenet_stats]
rescale_param = 1.3
test_id = test['Id']
test = test.drop('Id', axis=1 )<concatenate>
!pip install -q.. /input/efficientnetpytorchpip/efficientnet_pytorch-0.6.3/
df = train.append(test )<categorify>
from fastai.vision.models.efficientnet import *
df['GarageCond'] = df['GarageCond'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['GarageQual'] = df['GarageQual'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['BsmtCond'] = df['BsmtCond'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['BsmtExposure'] = df['BsmtExposure'].map({'NA':0, 'No':1, 'Mn':2, 'Av':3, 'Gd':4})
 df['BsmtFinType1'] = df['BsmtFinType1'].map({'NA':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})
 df['BsmtFinType2'] = df['BsmtFinType2'].map({'NA':0, 'Unf':1, 'LwQ':2, 'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})
 df['BsmtQual'] = df['BsmtQual'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['PoolQC'] = df['PoolQC'].map({'NA':0, 'Fa':1, 'TA':2, 'Gd':3, 'Ex':4})
 df['Alley'] = df['Alley'].map({'NA':0, 'Grvl':1, 'Pave':2})
 df['FireplaceQu'] = df['FireplaceQu'].map({'NA':0, 'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['ExterCond'] = df['ExterCond'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['ExterQual'] = df['ExterQual'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['KitchenQual'] = df['KitchenQual'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['LandSlope'] = df['LandSlope'].map({'Sev':1, 'Mod':2, 'Gtl':3})
 df['PavedDrive'] = df['PavedDrive'].map({'N':1, 'P':2, 'Y':3})
 df['Functional'] = df['Functional'].map({'Sal':1, 'Sev':2, 'Maj2':3, 'Maj1':4, 'Mod':5, 'Min2':6, 'Min1':7, 'Typ':8})
 df['HeatingQC'] = df['HeatingQC'].map({'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5})
 df['Street'] = df['Street'].map({'Grvl':1, 'Pave':2})
 df['Utilities'] = df['Utilities'].map({'ELO':1, 'NoSeWa':2, 'NoSewr':3, 'AllPub':4})
 df=df.drop('MoSold',axis=1)
 df['MSSubClass'] = df['MSSubClass'].map({20:'class1', 30:'class2', 40:'class3', 45:'class4',
50:'class5', 60:'class6', 70:'class7', 75:'class8',
80:'class9', 85:'class10', 90:'class11', 120:'class12',
150:'class13', 160:'class14', 180:'class15', 190:'class16'} )<drop_column>
class DummyDatabunch:
c = 2
path = '.'
device = defaults.device
loss_func = None
data = DummyDatabunch()
df = df.reset_index()
df = df.drop('index',axis = 1 )<filter>
effnet_model = EfficientNet.from_name("efficientnet-b5", override_params={'num_classes': 2})
 learner = Learner(data, effnet_model); learner.model_dir = '.'
learner.load('.. /input/deepfakerandmergeaugmodels/single_frame_effnetb5_randmerge')
 effnetb5_inference_model = learner.model.eval()
drop_id = df[df['LotArea'] > 100000].index<drop_column>
effnet_model = EfficientNet.from_name("efficientnet-b7", override_params={'num_classes': 2})
 learner = Learner(data, effnet_model); learner.model_dir = '.'
learner.load('.. /input/deepfakerandmergeaugmodels/single_frame_effnetb7_randmerge_fp16')
 effnetb7_inference_model = learner.model.float().eval()
drop_id = drop_id[drop_id < 1459]<feature_engineering>
learner = cnn_learner(data, models.resnet34, pretrained=False); learner.model_dir = '.'
learner.load('.. /input/deepfakerandmergeaugmodels/single_frame_resnet34_randmerge')
 resnet_inference_model = learner.model.eval()
df['MasVnrArea'][df[df['MasVnrArea'] > 1500].index] = df['MasVnrArea'].mean()
df['Utilities'][df[df['Utilities']==2].index] = df['Utilities'].mean()<drop_column>
predictions = []
video_fnames = []
df = df.drop(drop_id )<drop_column>
fname2pred = dict(zip(video_fnames, predictions))
y_train = y_train.drop(drop_id )<categorify>
submission_df = pd.read_csv("/kaggle/input/deepfake-detection-challenge/sample_submission.csv" )
dummy_drop = []
for i in cat_columns:
dummy_drop += [ i+'_'+str(df[i].unique() [-1])]
df = pd.get_dummies(df,columns=cat_columns)
 df = df.drop(dummy_drop,axis=1 )<normalization>
submission_df.label = submission_df.filename.map(fname2pred )
X_train = df[:-1459].drop(['index'], axis=1)
 X_test = df[-1459:].drop(['index'], axis=1)
 scaler = StandardScaler()
X_train[num_columns]= scaler.fit_transform(X_train[num_columns])
 X_test[num_columns]= scaler.transform(X_test[num_columns])
 X_train.shape, X_test.shape<feature_engineering>
submission_df['label'] = np.clip(submission_df['label'], 0.01, 0.99 )
<train_model><EOS> 
submission_df.to_csv("submission.csv",index=False )