Kaggle competition: Deepfake Detection Challenge
<SOS> metric: LogLoss Kaggle data source: deepfake-detection-challenge<import_modules>
!pip install /kaggle/input/deepfake-detection-jph/facenet_pytorch-2.0.1-py3-none-any.whl
!cp -r /kaggle/input/pretrainedmodels /kaggle/working/; cd /kaggle/working/pretrainedmodels/pretrained-models.pytorch ; pip install.
nltk.download('wordnet')
 nltk.download('punkt')
<load_from_csv>
import os
import math
import pickle
from functools import partial
from collections import defaultdict
from PIL import Image
from glob import glob
import cv2
import numpy as np
import skimage.measure
import albumentations as A
from tqdm.notebook import tqdm
from albumentations.pytorch import ToTensor
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torchvision.models.video import mc3_18, r2plus1d_18
from facenet_pytorch import MTCNN
df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 df.sample(10 )<train_model>
INPUT_DIR = "/kaggle/input/deepfake-detection-challenge/test_videos"
PRETRAINED_MODELS_3D = [{'type':'i3d',
'path':"/kaggle/input/deepfake-detection-jph/j3d_e1_l0.1374.model"},
{'type':'res34',
'path':"/kaggle/input/deepfake-detection-jph/res34_1cy_minaug_nonorm_e4_l0.1794.model"},
{'type':'mc3_112',
'path':"/kaggle/input/deepfake-detection-jph/mc3_18_112_1cy_lilaug_nonorm_e9_l0.1905.model"},
{'type':'mc3_224',
'path':"/kaggle/input/deepfake-detection-jph/mc3_18_112t224_1cy_lilaug_nonorm_e7_l0.1901.model"},
{'type':'r2p1_112',
'path':"/kaggle/input/deepfake-detection-jph/r2p1_18_8_112tr_112te_e12_l0.1741.model"},
{'type':'i3d',
'path':"/kaggle/input/deepfake-detection-jph/i3dcutmix_e11_l0.1612.model"},
{'type':'r2p1_112',
'path':"/kaggle/input/deepfake-detection-jph/r2plus1dcutmix_112_e10_l0.1608.model"}]
MAX_FRAMES_TO_LOAD = 100
MIN_FRAMES_FOR_FACE = 30
MAX_FRAMES_FOR_FACE = 100
FACE_FRAMES = 10
MAX_FACES_HIGHTHRESH = 5
MAX_FACES_LOWTHRESH = 1
FACEDETECTION_DOWNSAMPLE = 0.25
MTCNN_THRESHOLDS =(0.8, 0.8, 0.9)
 MTCNN_THRESHOLDS_RETRY =(0.5, 0.5, 0.5)
 MMTNN_FACTOR = 0.71
TWO_FRAME_OVERLAP = False
PROB_MIN, PROB_MAX = 0.001, 0.999
REVERSE_PROBS = True
DEFAULT_MISSING_PRED = 0.5
USE_FACE_FUNCTION = np.mean
RATIO_3D = 1
OUTPUT_FACE_SIZE =(256, 256)
 PRE_INFERENCE_CROP =(224, 224)
 RATIO_2D = 1
print("Training :")
 print("Length of the data :", len(df))
print(df.isnull().sum() )<count_missing_values>
def load_video(filename, every_n_frames=None, specific_frames=None, to_rgb=True, rescale=None, inc_pil=False, max_frames=None):

 assert every_n_frames or specific_frames, "Must supply either every n_frames or specific_frames"
assert bool(every_n_frames)!= bool(specific_frames), "Supply either 'every_n_frames' or 'specific_frames', not both"
cap = cv2.VideoCapture(filename)
 n_frames_in = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width_in = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height_in = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
if rescale:
rescale = rescale * 1920./np.max(( width_in, height_in))
width_out = int(width_in*rescale)if rescale else width_in
height_out = int(height_in*rescale)if rescale else height_in
if max_frames:
n_frames_in = min(n_frames_in, max_frames)
 if every_n_frames:
specific_frames = list(range(0,n_frames_in,every_n_frames))
n_frames_out = len(specific_frames)
 out_pil = []
out_video = np.empty(( n_frames_out, height_out, width_out, 3), np.dtype('uint8'))
i_frame_in = 0
i_frame_out = 0
ret = True
while(i_frame_in < n_frames_in and ret):
try:
try:
if every_n_frames == 1:
ret, frame_in = cap.read()
else:
ret = cap.grab()
if i_frame_in not in specific_frames:
i_frame_in += 1
continue
ret, frame_in = cap.retrieve()
if rescale:
frame_in = cv2.resize(frame_in,(width_out, height_out))
if to_rgb:
frame_in = cv2.cvtColor(frame_in, cv2.COLOR_BGR2RGB)
 except Exception as e:
print(f"Error for frame {i_frame_in} for video {filename}: {e}; using 0s")
 frame_in = np.zeros(( height_out, width_out, 3))
out_video[i_frame_out] = frame_in
i_frame_out += 1
if inc_pil:
try:
pil_img = Image.fromarray(frame_in)
 except Exception as e:
print(f"Using a blank frame for video {filename} frame {i_frame_in} as error {e}")
 pil_img = Image.fromarray(np.zeros(( 224,224,3), dtype=np.uint8))
out_pil.append(pil_img)
 i_frame_in += 1
except Exception as e:
print(f"Error for file {filename}: {e}")
 cap.release()
if inc_pil:
return out_video, out_pil, rescale
else:
return out_video, rescale
def get_roi_for_each_face(faces_by_frame, probs, video_shape, temporal_upsample, upsample=1):
frames_video, rows_video, cols_video, channels_video = video_shape
frames_video = math.ceil(frames_video)
 boolean_face_3d = np.zeros(( frames_video, rows_video, cols_video), dtype=np.bool)
 proba_face_3d = np.zeros(( frames_video, rows_video, cols_video)).astype('float32')
 for i_frame, faces in enumerate(faces_by_frame):
if faces is not None:
for i_face, face in enumerate(faces):
left, top, right, bottom = face
boolean_face_3d[i_frame, int(top):int(bottom), int(left):int(right)] = True
proba_face_3d[i_frame, int(top):int(bottom), int(left):int(right)] = probs[i_frame][i_face]
for i_frame, frame in enumerate(boolean_face_3d):
if i_frame == 0 or i_frame == frames_video-1:
continue
if True not in frame:
if TWO_FRAME_OVERLAP:
if i_frame > 1:
pre_overlap = boolean_face_3d[i_frame-1] | boolean_face_3d[i_frame-2]
else:
pre_overlap = boolean_face_3d[i_frame-1]
if i_frame < frames_video-2:
post_overlap = boolean_face_3d[i_frame+1] | boolean_face_3d[i_frame+2]
else:
post_overlap = boolean_face_3d[i_frame+1]
neighbour_overlap = pre_overlap & post_overlap
else:
neighbour_overlap = boolean_face_3d[i_frame-1] & boolean_face_3d[i_frame+1]
boolean_face_3d[i_frame] = neighbour_overlap
id_face_3d, n_faces = skimage.measure.label(boolean_face_3d, return_num=True)
 region_labels, counts = np.unique(id_face_3d, return_counts=True)
 region_labels, counts = region_labels[1:], counts[1:]
descending_size = np.argsort(counts)[::-1]
labels_by_size = region_labels[descending_size]
probs = [np.mean(proba_face_3d[id_face_3d == i_face])for i_face in region_labels]
descending_probs = np.argsort(probs)[::-1]
labels_by_probs = region_labels[descending_probs]
rois = [] ; face_maps = []
for i_face in labels_by_probs:
frames = np.where(np.any(id_face_3d == i_face, axis=(1, 2)) == True)
 starting_frame, ending_frame = frames[0].min() , frames[0].max()
cols, rows = [], []
for i_frame in range(starting_frame, ending_frame + 1):
rs = np.where(np.any(id_face_3d[i_frame] == i_face, axis=1)== True)
 rows.append(( rs[0].min() , rs[0].max()))
cs = np.where(np.any(id_face_3d[i_frame] == i_face, axis=0)== True)
 cols.append(( cs[0].min() , cs[0].max()))
frame_from, frame_to = starting_frame*temporal_upsample,(( ending_frame+1)*temporal_upsample)-1
rows_from, rows_to = np.array(rows)[:, 0].min() , np.array(rows)[:, 1].max()
cols_from, cols_to = np.array(cols)[:, 0].min() , np.array(cols)[:, 1].max()
frame_to = min(frame_to, frame_from + MAX_FRAMES_FOR_FACE)
 if frame_to - frame_from >= MIN_FRAMES_FOR_FACE:
tmp_face_map = id_face_3d.copy()
tmp_face_map[tmp_face_map != i_face] = 0
tmp_face_map[tmp_face_map == i_face] = 1
face_maps.append(tmp_face_map[frame_from//temporal_upsample:frame_to//temporal_upsample+1])
 rois.append(((frame_from, frame_to), 
(int(rows_from*upsample), int(rows_to*upsample)) , 
(int(cols_from*upsample), int(cols_to*upsample))))
return np.array(rois), face_maps
print("Test :")
 print("Length of the data :", len(df_test))
print(df_test.isnull().sum() )<string_transform>
def get_center(bbox):
x1, y1, x2, y2 = bbox
return(x1+x2)/2,(y1+y2)/2
def get_coords(faces_roi):
coords = np.argwhere(faces_roi == 1)
 if coords.shape[0] == 0:
return None
y1, x1 = coords[0]
y2, x2 = coords[-1]
return x1, y1, x2, y2
def interpolate_center(c1, c2, length):
x1, y1 = c1
x2, y2 = c2
xi, yi = np.linspace(x1, x2, length), np.linspace(y1, y2, length)
 return np.vstack([xi, yi] ).transpose(1,0)
 def get_faces(faces_roi, upsample):
all_faces = []
rows = faces_roi[0].shape[1]
cols = faces_roi[0].shape[2]
for i in range(len(faces_roi)) :
faces = np.asarray([get_coords(faces_roi[i][j])for j in range(len(faces_roi[i])) ])
 if faces[0] is None: faces[0] = faces[1]
if faces[-1] is None: faces[-1] = faces[-2]
if None in faces:
raise Exception('This should not have happened...')
 all_faces.append(faces)
 extracted_faces = []
for face in all_faces:
max_dim = np.concatenate([face[:,2]-face[:,0],face[:,3]-face[:,1]])
 max_dim = np.percentile(max_dim, 90)
 max_dim = int(max_dim * 1.2)
 centers = np.asarray([get_center(_)for _ in face])
 centers = np.vstack([interpolate_center(centers[i], centers[i+1], length=10)for i in range(len(centers)-1)] ).astype('int')
 x1y1 = centers - max_dim // 2
x2y2 = centers + max_dim // 2
x1, y1 = x1y1[:,0], x1y1[:,1]
x2, y2 = x2y2[:,0], x2y2[:,1]
x2[x1 < 0] -= x1[x1 < 0]
y2[y1 < 0] -= y1[y1 < 0]
x1[x1 < 0] = 0
y1[y1 < 0] = 0
y1[y2 > rows] += rows - y2[y2 > rows]
x1[x2 > cols] += cols - x2[x2 > cols]
y2[y2 > rows] = rows
x2[x2 > cols] = cols
vidface = np.asarray([[x1[_],y1[_],x2[_],y2[_]] for _,c in enumerate(centers)])
 vidface =(vidface*upsample ).astype('int')
 extracted_faces.append(vidface)
 return extracted_faces
def detect_face_with_mtcnn(mtcnn_model, pil_frames, facedetection_upsample, video_shape, face_frames):
boxes, _probs = mtcnn_model.detect(pil_frames, landmarks=False)
 faces, faces_roi = get_roi_for_each_face(faces_by_frame=boxes, probs=_probs, video_shape=video_shape, temporal_upsample=face_frames, upsample=facedetection_upsample)
 coords = [] if len(faces_roi)== 0 else get_faces(faces_roi, upsample=facedetection_upsample)
 return faces, coords
def face_detection_wrapper(mtcnn_model, videopath, every_n_frames, facedetection_downsample, max_frames_to_load):
video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample, inc_pil=True, max_frames=max_frames_to_load)
 if len(pil_frames):
try:
faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn,
pil_frames=pil_frames,
facedetection_upsample=1/rescale,
video_shape=video.shape,
face_frames=every_n_frames)
 except RuntimeError:
print(f"Failed to process {videopath} ! Downsampling x2...")
 video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample/2, inc_pil=True, max_frames=max_frames_to_load)
 try:
faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn,
pil_frames=pil_frames,
facedetection_upsample=1/rescale,
video_shape=video.shape,
face_frames=every_n_frames)
 except RuntimeError:
print(f"Failed on downsample ! Skipping...")
 return [], []
else:
print('Failed to fetch frames ! Skipping...')
 return [], []
if len(faces)== 0:
print('Failed to find faces ! Upsampling x2...')
 try:
video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample*2, inc_pil=True, max_frames=max_frames_to_load)
 faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn,
pil_frames=pil_frames,
facedetection_upsample=1/rescale,
video_shape=video.shape,
face_frames=every_n_frames)
 except Exception as e:
print(e)
 return [], []
return faces, coords
tokens = word_tokenize(df["text"][0])
 tokens = [word.lower() for word in tokens]
print(df["text"][0])
 print(tokens )<define_variables>
videopaths = sorted(glob(os.path.join(INPUT_DIR, "*.mp4")))
 print(f'Found {len(videopaths)} videos !')
 mtcnn = MTCNN(margin=0, keep_all=True, post_process=False, select_largest=False, device='cuda:0', thresholds=MTCNN_THRESHOLDS, factor=MMTNN_FACTOR)
 faces_by_videopath = {}
coords_by_videopath = {}
for i_video, videopath in enumerate(tqdm(videopaths)) :
faces, coords = face_detection_wrapper(mtcnn, videopath, every_n_frames=FACE_FRAMES, facedetection_downsample=FACEDETECTION_DOWNSAMPLE, max_frames_to_load=MAX_FRAMES_TO_LOAD)
 if len(faces):
faces_by_videopath[videopath] = faces[:MAX_FACES_HIGHTHRESH]
coords_by_videopath[videopath] = coords[:MAX_FACES_HIGHTHRESH]
else:
print(f"Found no faces for {videopath} !")
 del mtcnn
gc.collect()
videopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}
print(f"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing")
words = [word for word in tokens if word.isalpha() ]
print(words )<string_transform>
mtcnn = MTCNN(margin=0, keep_all=True, post_process=False, select_largest=False ,device='cuda:0', thresholds=MTCNN_THRESHOLDS_RETRY, factor=MMTNN_FACTOR)
 for i_video, videopath in enumerate(tqdm(videopaths_missing_faces)) :
faces, coords = face_detection_wrapper(mtcnn,
videopath,
every_n_frames=FACE_FRAMES,
facedetection_downsample=FACEDETECTION_DOWNSAMPLE,
max_frames_to_load=MAX_FRAMES_TO_LOAD)
 if len(faces):
faces_by_videopath[videopath] = faces[:MAX_FACES_LOWTHRESH]
coords_by_videopath[videopath] = coords[:MAX_FACES_LOWTHRESH]
else:
print(f"Found no faces for {videopath} !")
 del mtcnn
gc.collect()
faces_by_videopath = dict(sorted(faces_by_videopath.items() , key=lambda x: x[0]))
videopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}
print(f"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing" )
stop_words = set(stopwords.words("english"))
words = [word for word in words if not word in stop_words]
print(words )<string_transform>
faces_by_videopath = dict(sorted(faces_by_videopath.items() , key=lambda x: x[0]))
videopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}
print(f"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing" )
porter = PorterStemmer()
stemmed = [porter.stem(word)for word in words]
print(stemmed )<statistical_test>
class MaxPool3dSamePadding(nn.MaxPool3d):
def compute_pad(self, dim, s):
if s % self.stride[dim] == 0:
return max(self.kernel_size[dim] - self.stride[dim], 0)
 else:
return max(self.kernel_size[dim] -(s % self.stride[dim]), 0)
 def forward(self, x): 
(batch, channel, t, h, w)= x.size()
out_t = np.ceil(float(t)/ float(self.stride[0]))
out_h = np.ceil(float(h)/ float(self.stride[1]))
out_w = np.ceil(float(w)/ float(self.stride[2]))
pad_t = self.compute_pad(0, t)
 pad_h = self.compute_pad(1, h)
 pad_w = self.compute_pad(2, w)
 pad_t_f = pad_t // 2
pad_t_b = pad_t - pad_t_f
pad_h_f = pad_h // 2
pad_h_b = pad_h - pad_h_f
pad_w_f = pad_w // 2
pad_w_b = pad_w - pad_w_f
pad =(pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
 x = F.pad(x, pad)
 return super(MaxPool3dSamePadding, self ).forward(x)
 class Unit3D(nn.Module):
def __init__(self, in_channels,
output_channels,
kernel_shape=(1, 1, 1),
stride=(1, 1, 1),
padding=0,
activation_fn=F.relu,
use_batch_norm=True,
use_bias=False,
name='unit_3d'):

 super(Unit3D, self ).__init__()
self._output_channels = output_channels
self._kernel_shape = kernel_shape
self._stride = stride
self._use_batch_norm = use_batch_norm
self._activation_fn = activation_fn
self._use_bias = use_bias
self.name = name
self.padding = padding
self.conv3d = nn.Conv3d(in_channels=in_channels,
out_channels=self._output_channels,
kernel_size=self._kernel_shape,
stride=self._stride,
padding=0,
bias=self._use_bias)
 if self._use_batch_norm:
self.bn = nn.BatchNorm3d(self._output_channels, eps=0.001, momentum=0.01)
 def compute_pad(self, dim, s):
if s % self._stride[dim] == 0:
return max(self._kernel_shape[dim] - self._stride[dim], 0)
 else:
return max(self._kernel_shape[dim] -(s % self._stride[dim]), 0)
 def forward(self, x): 
(batch, channel, t, h, w)= x.size()
out_t = np.ceil(float(t)/ float(self._stride[0]))
out_h = np.ceil(float(h)/ float(self._stride[1]))
out_w = np.ceil(float(w)/ float(self._stride[2]))
pad_t = self.compute_pad(0, t)
 pad_h = self.compute_pad(1, h)
 pad_w = self.compute_pad(2, w)
 pad_t_f = pad_t // 2
pad_t_b = pad_t - pad_t_f
pad_h_f = pad_h // 2
pad_h_b = pad_h - pad_h_f
pad_w_f = pad_w // 2
pad_w_b = pad_w - pad_w_f
pad =(pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
 x = F.pad(x, pad)
 x = self.conv3d(x)
 if self._use_batch_norm:
x = self.bn(x)
 if self._activation_fn is not None:
x = self._activation_fn(x)
 return x
class InceptionModule(nn.Module):
def __init__(self, in_channels, out_channels, name):
super(InceptionModule, self ).__init__()
self.b0 = Unit3D(in_channels=in_channels, output_channels=out_channels[0], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_0/Conv3d_0a_1x1')
 self.b1a = Unit3D(in_channels=in_channels, output_channels=out_channels[1], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_1/Conv3d_0a_1x1')
 self.b1b = Unit3D(in_channels=out_channels[1], output_channels=out_channels[2], kernel_shape=[3, 3, 3],
name=name + '/Branch_1/Conv3d_0b_3x3')
 self.b2a = Unit3D(in_channels=in_channels, output_channels=out_channels[3], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_2/Conv3d_0a_1x1')
 self.b2b = Unit3D(in_channels=out_channels[3], output_channels=out_channels[4], kernel_shape=[3, 3, 3],
name=name + '/Branch_2/Conv3d_0b_3x3')
 self.b3a = MaxPool3dSamePadding(kernel_size=[3, 3, 3],
stride=(1, 1, 1), padding=0)
 self.b3b = Unit3D(in_channels=in_channels, output_channels=out_channels[5], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_3/Conv3d_0b_1x1')
 self.name = name
def forward(self, x):
b0 = self.b0(x)
 b1 = self.b1b(self.b1a(x))
b2 = self.b2b(self.b2a(x))
b3 = self.b3b(self.b3a(x))
return torch.cat([b0, b1, b2, b3], dim=1)
 class InceptionI3d(nn.Module):

 VALID_ENDPOINTS =(
 'Conv3d_1a_7x7',
'MaxPool3d_2a_3x3',
'Conv3d_2b_1x1',
'Conv3d_2c_3x3',
'MaxPool3d_3a_3x3',
'Mixed_3b',
'Mixed_3c',
'MaxPool3d_4a_3x3',
'Mixed_4b',
'Mixed_4c',
'Mixed_4d',
'Mixed_4e',
'Mixed_4f',
'MaxPool3d_5a_2x2',
'Mixed_5b',
'Mixed_5c',
'Logits',
'Predictions', 
)
 def __init__(self, num_classes=400, spatial_squeeze=True, output_method='per_frame',
final_endpoint='Logits', name='inception_i3d', in_channels=3, dropout_keep_prob=0.5):

 if final_endpoint not in self.VALID_ENDPOINTS:
raise ValueError('Unknown final endpoint %s' % final_endpoint)
 super(InceptionI3d, self ).__init__()
self.output_method = output_method
assert output_method in('per_frame', 'avg_pool', 'max_pool', 'dual_pool')
 self._num_classes = num_classes
self._spatial_squeeze = spatial_squeeze
self._final_endpoint = final_endpoint
self.logits = None
if self._final_endpoint not in self.VALID_ENDPOINTS:
raise ValueError('Unknown final endpoint %s' % self._final_endpoint)
 self.end_points = {}
end_point = 'Conv3d_1a_7x7'
self.end_points[end_point] = Unit3D(in_channels=in_channels, output_channels=64, kernel_shape=[7, 7, 7],
stride=(2, 2, 2), padding=(3, 3, 3), name=name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_2a_3x3'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Conv3d_2b_1x1'
self.end_points[end_point] = Unit3D(in_channels=64, output_channels=64, kernel_shape=[1, 1, 1], padding=0,
name=name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Conv3d_2c_3x3'
self.end_points[end_point] = Unit3D(in_channels=64, output_channels=192, kernel_shape=[3, 3, 3], padding=1,
name=name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_3a_3x3'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_3b'
self.end_points[end_point] = InceptionModule(192, [64, 96, 128, 16, 32, 32], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_3c'
self.end_points[end_point] = InceptionModule(256, [128, 128, 192, 32, 96, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_4a_3x3'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[3, 3, 3], stride=(2, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4b'
self.end_points[end_point] = InceptionModule(128 + 192 + 96 + 64, [192, 96, 208, 16, 48, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4c'
self.end_points[end_point] = InceptionModule(192 + 208 + 48 + 64, [160, 112, 224, 24, 64, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4d'
self.end_points[end_point] = InceptionModule(160 + 224 + 64 + 64, [128, 128, 256, 24, 64, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4e'
self.end_points[end_point] = InceptionModule(128 + 256 + 64 + 64, [112, 144, 288, 32, 64, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4f'
self.end_points[end_point] = InceptionModule(112 + 288 + 64 + 64, [256, 160, 320, 32, 128, 128],
name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_5a_2x2'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[2, 2, 2], stride=(2, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_5b'
self.end_points[end_point] = InceptionModule(256 + 320 + 128 + 128, [256, 160, 320, 32, 128, 128],
name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_5c'
self.end_points[end_point] = InceptionModule(256 + 320 + 128 + 128, [384, 192, 384, 48, 128, 128],
name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Logits'
self.avg_pool = nn.AvgPool3d(kernel_size=[2, 7, 7],
stride=(1, 1, 1))
self.dropout = nn.Dropout(dropout_keep_prob)
 self.logits = Unit3D(in_channels=384 + 384 + 128 + 128, output_channels=self._num_classes,
kernel_shape=[1, 1, 1],
padding=0,
activation_fn=None,
use_batch_norm=False,
use_bias=True,
name='logits')
 self.build()
def replace_logits(self, num_classes):
self._num_classes = num_classes
self.logits = Unit3D(in_channels=384 + 384 + 128 + 128, output_channels=self._num_classes,
kernel_shape=[1, 1, 1],
padding=0,
activation_fn=None,
use_batch_norm=False,
use_bias=True,
name='logits')
 def build(self):
for k in self.end_points.keys() :
self.add_module(k, self.end_points[k])
 def forward(self, x):
for end_point in self.VALID_ENDPOINTS:
if end_point in self.end_points:
x = self._modules[end_point](x)
 x = self.logits(self.dropout(self.avg_pool(x)))
 if self._spatial_squeeze:
logits = x.squeeze(3 ).squeeze(3)
 if self.output_method == 'per_frame':
return F.interpolate(logits, 64, mode='linear')
 elif self.output_method == 'avg_pool':
avg_all_frames = F.adaptive_avg_pool1d(logits, output_size=1)
 return avg_all_frames.squeeze(-1)
 def extract_features(self, x):
for end_point in self.VALID_ENDPOINTS:
if end_point in self.end_points:
x = self._modules[end_point](x)
 return self.avg_pool(x )
lemmatizer = WordNetLemmatizer()
lemmatized = [lemmatizer.lemmatize(word)for word in words]
print(lemmatized )<drop_column>
def conv3x3x3(in_planes, out_planes, stride=1):
return nn.Conv3d(
 in_planes,
out_planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
 def downsample_basic_block(x, planes, stride):
out = F.avg_pool3d(x, kernel_size=1, stride=stride)
 zero_pads = torch.Tensor(
 out.size(0), planes - out.size(1), out.size(2), out.size(3),
out.size(4)).zero_()
if isinstance(out.data, torch.cuda.FloatTensor):
zero_pads = zero_pads.cuda()
out = Variable(torch.cat([out.data, zero_pads], dim=1))
return out
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3x3(inplanes, planes, stride)
 self.bn1 = nn.BatchNorm3d(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3x3(planes, planes)
 self.bn2 = nn.BatchNorm3d(planes)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm3d(planes)
 self.conv2 = nn.Conv3d(
 planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
 self.bn2 = nn.BatchNorm3d(planes)
 self.conv3 = nn.Conv3d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm3d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self,
block,
layers,
sample_size,
sample_duration,
shortcut_type='B',
num_classes=400):
self.inplanes = 64
super(ResNet, self ).__init__()
self.conv1 = nn.Conv3d(
 3,
64,
kernel_size=7,
stride=(1, 2, 2),
padding=(3, 3, 3),
bias=False)
 self.bn1 = nn.BatchNorm3d(64)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool3d(kernel_size=(3, 3, 3), stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0], shortcut_type)
 self.layer2 = self._make_layer(
 block, 128, layers[1], shortcut_type, stride=2)
 self.layer3 = self._make_layer(
 block, 256, layers[2], shortcut_type, stride=2)
 self.layer4 = self._make_layer(
 block, 512, layers[3], shortcut_type, stride=2)
 self.avgpool = nn.AdaptiveAvgPool3d(1)
 self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv3d):
m.weight = nn.init.kaiming_normal(m.weight, mode='fan_out')
 elif isinstance(m, nn.BatchNorm3d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, shortcut_type, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
if shortcut_type == 'A':
downsample = partial(
 downsample_basic_block,
planes=planes * block.expansion,
stride=stride)
 else:
downsample = nn.Sequential(
 nn.Conv3d(
 self.inplanes,
planes * block.expansion,
kernel_size=1,
stride=stride,
bias=False), nn.BatchNorm3d(planes * block.expansion))
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return x
def get_fine_tuning_parameters(model, ft_begin_index):
if ft_begin_index == 0:
return model.parameters()
ft_module_names = []
for i in range(ft_begin_index, 5):
ft_module_names.append('layer{}'.format(i))
ft_module_names.append('fc')
 parameters = []
for k, v in model.named_parameters() :
for ft_module in ft_module_names:
if ft_module in k:
parameters.append({'params': v})
 break
else:
parameters.append({'params': v, 'lr': 0.0})
 return parameters
def resnet18(**kwargs):

 model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
 return model
def resnet34(**kwargs):

 model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)
 return model
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 stop_words = set(stopwords.words("english"))
def mytokenize(text, normalize=None, stop_words=stop_words):
text = remove_html(text)
 text = remove_URL(text)
 text = remove_emoji(text)
 text = text.lower()
text = word_tokenize(text)
 text = [word for word in text if word.isalpha() ]
text = [word for word in text if not word in stop_words]
if normalize == "s":
porter = PorterStemmer()
text = [porter.stem(word)for word in text]
if normalize == "l":
lemmatizer = WordNetLemmatizer()
text = [lemmatizer.lemmatize(word)for word in text]
return text
mytokenize_s = partial(mytokenize, normalize='s')
 mytokenize_l = partial(mytokenize, normalize='l')
 def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 df['tokens'] = df['text'].apply(mytokenize_l)
 df_test['tokens'] = df_test['text'].apply(mytokenize_l )<feature_engineering>
class HFlipWrapper(nn.Module):
def __init__(self, model, flip_dim=(-1,)) :
super(HFlipWrapper, self ).__init__()
self.model = model
self.flip_dim = flip_dim
def forward(self, x):
with torch.no_grad() :
xf = torch.flip(x, self.flip_dim)
 pred = self.model(torch.stack([x, xf]))
return pred
def join_list(tab):
return " ".join(tab)
 df["text_preprocessed"] = df["tokens"].apply(join_list)
 df_test["text_preprocessed"] = df_test["tokens"].apply(join_list)
 def transform_keyword(word):
return word.split('%20')
 df["keyword"] = df.keyword.fillna(" ")
 df_test["keyword"] = df_test.keyword.fillna(" ")
 df["keyword"] = df["keyword"].apply(transform_keyword ).apply(join_list)
 df_test["keyword"] = df_test["keyword"].apply(transform_keyword ).apply(join_list)
 df["text_preprocessed"] = df["keyword"] + " " + df["text_preprocessed"]
df_test["text_preprocessed"] = df_test["keyword"] + " " + df_test["text_preprocessed"]
df["tokens"] = df["text_preprocessed"].apply(str.split)
 df_test["tokens"] = df_test["text_preprocessed"].apply(str.split)
 df.sample()<split>
models_3d = []
for modeldict in PRETRAINED_MODELS_3D:
if modeldict['type'] == 'i3d':
model = InceptionI3d(157, in_channels=3, output_method='avg_pool')
 model.replace_logits(2)
 model = model.cuda()
model.eval()
model.load_state_dict(torch.load(modeldict['path']))
models_3d.append({'norm':'i3d', 'model':model})
 elif modeldict['type'] == 'res18':
model = resnet18(num_classes=2, shortcut_type='A', sample_size=224, sample_duration=32)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'nil', 'model':model})
 elif modeldict['type'] == 'res34':
model = resnet34(num_classes=2, shortcut_type='A', sample_size=224, sample_duration=32)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'nil', 'model':model})
 elif modeldict['type'] == 'mc3_112':
model = mc3_18(num_classes=2)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'112_imagenet', 'model':model})
 elif modeldict['type'] == 'mc3_224':
model = mc3_18(num_classes=2)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'224_imagenet', 'model':model})
 elif modeldict['type'] == 'r2p1_112':
model = r2plus1d_18(num_classes=2)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'112_imagenet', 'model':model})
 else:
raise ValueError(f"Unknown model type {modeldict['type']}" )
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 sk_doc2bow = CountVectorizer()
sk_doc2bow.fit(X_all)
 del X_all
X = sk_doc2bow.transform(df["text_preprocessed"])
 X_test = sk_doc2bow.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )<save_to_csv>
with open('/kaggle/input/deepfake/deepfake/deepfake/skp/configs/experiments/experiment001.yaml')as f:
CFG = yaml.load(f, Loader=yaml.FullLoader)
 CFG['model']['params']['pretrained'] = None
model2d = build_model(CFG['model']['name'], CFG['model']['params'])
 model2d.load_state_dict(torch.load('/kaggle/input/deepfake/SRXT50_094_VM-0.2504.PTH'))
model2d = model2d.eval().cuda()
loader = build_dataloader(CFG, data_info={'vidfiles': [], 'labels': []}, mode='predict' )
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_d2b.csv",index=False )<split>
def resize_and_square_face(video, output_size):
input_size = max(video.shape[1], video.shape[2])
 out_video = np.empty(( len(video), output_size[0], output_size[1], 3), np.dtype('uint8'))
for i_frame, frame in enumerate(video):
padded_image = np.zeros(( input_size, input_size, 3))
padded_image[0:frame.shape[0], 0:frame.shape[1]] = frame
if(input_size, input_size)!= output_size:
frame = cv2.resize(padded_image,(output_size[0], output_size[1])).astype(np.uint8)
 else:
frame = padded_image.astype(np.uint8)
 out_video[i_frame] = frame
return out_video
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 tfidf = TfidfVectorizer(stop_words = 'english')
 tfidf.fit(X_all)
 del X_all
X = tfidf.transform(df["text_preprocessed"])
 X_test = tfidf.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )<save_to_csv>
def get_last_frame_needed_across_faces(faces):
last_frame = 0
for face in faces: 
(frame_from, frame_to),(row_from, row_to),(col_from, col_to)= face
last_frame = max(frame_to, last_frame)
 return last_frame
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_tf.csv",index=False )<split>
predictions = defaultdict(list )
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 tfidf = TfidfVectorizer(stop_words = 'english', min_df=10)
 tfidf.fit(X_all)
 del X_all
X = tfidf.transform(df["text_preprocessed"])
 X_test = tfidf.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42 )<save_to_csv>
test_transforms_114_imagenet = A.Compose([A.Resize(height=112, width=112),
A.Normalize() ])
 test_transforms_224_imagenet = A.Compose([A.Normalize() ] )
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_tf.csv",index=False )<prepare_x_and_y>
del model
for modeldict in models_3d:
del modeldict['model']
del modeldict
del models_3d
gc.collect()
X_all = pd.concat([df["text_preprocessed"], df_test["text_preprocessed"]])
 tfidf = TfidfVectorizer(stop_words = 'english', min_df=5, ngram_range=(1, 3))
tfidf.fit(X_all)
 del X_all
X = tfidf.transform(df["text_preprocessed"])
 X_test = tfidf.transform(df_test["text_preprocessed"])
 X_train, X_val, y_train, y_val = train_test_split(X, df["target"], test_size=0.1, random_state=42)
 row_tf = X_train[0].toarray() [0]
print(row_tf.shape)
 print(row_tf)
 row_tf[row_tf != 0]<save_to_csv>
for videopath in videopaths:
videoname = os.path.basename(videopath)
 if videoname not in predictions:
predictions[videoname] = [DEFAULT_MISSING_PRED]
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X_train, y_train)
 y_val_pred = model.predict(X_val)
 print(accuracy_score(y_val, y_val_pred), f1_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
model = SVC(C=2, gamma=0.4, kernel='rbf' ).fit(X, df.target)
 y_test_pred = model.predict(X_test)
 sub_df = pd.read_csv(os.path.join('.. /input/nlp-getting-started/', 'sample_submission.csv'))
sub_df["target"] = y_test_pred
sub_df.to_csv("submission_tf.csv",index=False )<categorify>
preds_ensembled = {}
for videopath, pred in predictions.items() :
preds_ensembled[videopath] = np.clip(np.mean(pred), PROB_MIN, PROB_MAX )
<count_unique_values><EOS> 
print(f"{len(preds_ensembled)} predictions")
 with open("submission.csv", 'w')as f:
f.write("filename,label
")
 for filename, label in preds_ensembled.items() :
f.write(f"{filename},{label}
" )