Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<predict_on_test>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
!pip install imgaug
speed_test = False
if speed_test:
start = time.perf_counter()
for vid in tqdm(test_videos[:10]):
print(predict(test_dir+vid))
print(( time.perf_counter() -start)/10 )<define_variables>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from tqdm import tqdm_notebook
import matplotlib.pyplot as plt
import random
import math
import imgaug as ia
from imgaug import augmenters as iaa
start = time.perf_counter()
predictions = []
for i, vid in tqdm(enumerate(test_videos)) :
if i%20==19:
os.system(f'echo {str(i)} {predictions[-1]:.2f}')
 try:
pred = []
x = np.array(get_batch(test_dir+vid))
for i in x:
i = np.reshape(i,(1,224,224,3))
predictions.append(np.mean(np.array(model.predict(x))))
except Exception as e:
print(vid+' error:'+str(e))
predictions.append(0.5)
 print(( time.perf_counter() -start)/len(test_videos))<save_to_csv>
PATH = './'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
LABELS = '.. /input/humpback-whale-identification/train.csv'
BOXES = '.. /input/generating-whale-bounding-boxes/bounding_boxes.csv'
MODLE_INIT = '.. /input/pytorch-pretrained-models/'
n_embedding = 512
bs = 32
sz = 224
nw = 2
submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df['label'] = np.clip(submission_df['label'],0.05,0.95)
 submission_df.to_csv("submission.csv", index=False )<install_modules>
def open_image(fn):
flags = cv2.IMREAD_UNCHANGED+cv2.IMREAD_ANYDEPTH+cv2.IMREAD_ANYCOLOR
if not os.path.exists(fn):
raise OSError('No such file or directory: {}'.format(fn))
elif os.path.isdir(fn):
raise OSError('Is a directory: {}'.format(fn))
else:
try:
im = cv2.imread(str(fn), flags)
 if im is None: raise OSError(f'File not recognized by opencv: {fn}')
 return cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
 except Exception as e:
raise OSError('Error handling image at: {}'.format(fn)) from e
class Loader() :
def __init__(self,path,tfms_g=None, tfms_px=None):
self.boxes = pd.read_csv(BOXES ).set_index('Image')
 self.path = path
self.tfms_g = iaa.Sequential(tfms_g,random_order=False)\
if tfms_g is not None else None
self.tfms_px = iaa.Sequential(tfms_px,random_order=False)\
if tfms_px is not None else None
def __call__(self, fname):
fname = os.path.basename(fname)
 x0,y0,x1,y1 = tuple(self.boxes.loc[fname,['x0','y0','x1','y1']].tolist())
 img = open_image(os.path.join(self.path,fname))
l1,l0,_ = img.shape
b0,b1 = x1-x0, y1-y0
x0n,x1n = max(int(x0 - b0*0.05),0), min(int(x1 + b0*0.05),l0-1)
 y0n,y1n = max(int(y0 - b1*0.05),0), min(int(y1 + b1*0.05),l1-1)
 if self.tfms_g != None: img = self.tfms_g.augment_image(img)
 img = cv2.resize(img[y0n:y1n,x0n:x1n,:],(sz,sz))
if self.tfms_px != None: img = self.tfms_px.augment_image(img)
 return img.astype(np.float)/255
!pip install /kaggle/input/deepfake-detection-jph/facenet_pytorch-2.0.1-py3-none-any.whl
!cp -r /kaggle/input/pretrainedmodels /kaggle/working/; cd /kaggle/working/pretrainedmodels/pretrained-models.pytorch ; pip install.
<import_modules>
def get_idxs0(names, df, n=64):
idxs = []
for name in names:
label = df[df.Image == name].Id.values[0]
idxs.append(df[df.Id != label].Image.sample(n ).values)
 return idxs
class Image_selection:
def __init__(self,fnames,data,emb_df=None,model=None):
if emb_df is None or model is None:
df = data.copy()
counts = Counter(df.Id.values)
 df['c'] = df['Id'].apply(lambda x: counts[x])
 df['label'] = df.Id
df.loc[df.c == 1,'label'] = 'new_whale'
df = df.sort_values(by=['c'])
 df.label = pd.factorize(df.label)[0]
l1 = 1 + df.label.max()
l2 = len(df[df.label==0])
 df.loc[df.label==0,'label'] = range(l1, l1+l2)
 df = df.set_index('label')
 l = len(fnames)
 idxs = Parallel(n_jobs=nw )(delayed(get_idxs0)\ 
(fnames[int(i*l/nw):int(( i+1)*l/nw)], df)for i in range(nw))
idxs = [y for x in idxs for y in x]
else:
data = data.copy().set_index('Image')
 trn_emb = emb_df.copy()
trn_emb.set_index('files',inplace=True)
 trn_emb['emb'] = [[float(i)for i in s.split() ] for s in trn_emb['emb']]
trn_emb = data.join(trn_emb)
 trn_emb = trn_emb.reset_index()
trn_emb['idx'] = np.arange(len(trn_emb))
trn_emb = trn_emb.set_index('Id')
 emb = np.array(trn_emb.emb.tolist())
 l = len(fnames)
 idxs = []
model.eval()
with torch.no_grad() :
m = model.module if isinstance(model,FP16)else model
emb = torch.from_numpy(emb ).half().cuda()
for name in tqdm_notebook(fnames):
label = trn_emb.loc[trn_emb.Image == name].index[0]
v0 = np.array(trn_emb.loc[trn_emb.Image == name,'emb'].tolist() [0])
 v0 = torch.from_numpy(v0 ).half().cuda()
d = m.get_d(v0,emb)
 ids = trn_emb.loc[trn_emb.index!=label].idx.tolist()
sorted, indices = torch.sort(d[ids])
 idxs.append([ids[i] for i in indices[:64]])
 trn_emb = trn_emb.set_index('idx')
 idxs = [trn_emb.loc[idx,'Image'] for idx in idxs]
self.df = pd.DataFrame({'Image':fnames,'idxs':idxs} ).set_index('Image')
 def get(self,name):
return np.random.choice(self.df.loc[name].values[0],1)[0]
import os
import math
import pickle
from functools import partial
from collections import defaultdict
from PIL import Image
from glob import glob
import cv2
import numpy as np
import skimage.measure
import albumentations as A
from tqdm.notebook import tqdm
from albumentations.pytorch import ToTensor
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torchvision.models.video import mc3_18, r2plus1d_18
from facenet_pytorch import MTCNN<define_variables>
class pdFilesDataset(FilesDataset):
def __init__(self, data, path, transform):
df = data.copy()
counts = Counter(df.Id.values)
 df['c'] = df['Id'].apply(lambda x: counts[x])
 fnames = df[(df.c>2)&(df.Id != 'new_whale')].Image.tolist()
df['label'] = df.Id
df.loc[df.c == 1,'label'] = 'new_whale'
df = df.sort_values(by=['c'])
 df.label = pd.factorize(df.label)[0]
l1 = 1 + df.label.max()
l2 = len(df[df.label==0])
 df.loc[df.label==0,'label'] = range(l1, l1+l2)
 self.labels = df.copy().set_index('Image')
 self.names = df.copy().set_index('label')
 if path == TRAIN:
tfms_g = [iaa.Affine(rotate=(-8, 8),mode='reflect',
scale={"x":(0.9, 1.1), "y":(0.9, 1.1)}, shear=(-16,16)) ]
tfms_px = [iaa.Fliplr(0.5), iaa.AddToHueAndSaturation(( -20, 20)) ,
iaa.Grayscale(alpha=(0.0, 1.0)) ,iaa.GaussianBlur(( 0, 1.0)) ]
self.loader = Loader(path,tfms_g,tfms_px)
 else: self.loader = Loader(path)
 self.selection = None
super().__init__(fnames, transform, path)
 def get_x(self, i):
label = self.labels.loc[self.fnames[i],'label']
for j in range(10):
try:
names = self.names.loc[label].Image
break
except: None
name_p = names if isinstance(names,str)else \
random.sample(set(names)- set([self.fnames[i]]),1)[0]
if(self.selection == None):
for j in range(10):
try:
names = self.names.loc[self.names.index!=label].Image
break
except: names = self.fnames[i]
name_n = names if isinstance(names,str)else names.sample(1 ).values[0]
else:
name_n = self.selection.get(self.fnames[i])
 imgs = [self.loader(os.path.join(self.path,self.fnames[i])) ,
self.loader(os.path.join(self.path,name_p)) ,
self.loader(os.path.join(self.path,name_n)) ,
label,label,self.labels.loc[name_n,'label']]
return imgs
def get_y(self, i):
return 0
def get(self, tfm, x, y):
if tfm is None:
return(*x,0)
 else:
x1, y1 = tfm(x[0],x[3])
 x2, y2 = tfm(x[1],x[4])
 x3, y3 = tfm(x[2],x[5])
 x = np.stack(( x1,x2,x3),0)
 return x,(y1,y2,y3)
 def get_names(self,label):
names = []
for j in range(10):
try:
names = self.names.loc[label].Image
break
except: None
return names
@property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return n_embedding
def get_n(self): return len(self.fnames)
 class FilesDataset_single(FilesDataset):
def __init__(self, data, path, transform):
self.loader = Loader(path)
 fnames = os.listdir(path)
 super().__init__(fnames, transform, path)
 def get_x(self, i):
return self.loader(os.path.join(self.path,self.fnames[i]))
def get_y(self, i):
return 0
@property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return n_embedding
def get_n(self): return len(self.fnames )
INPUT_DIR = "/kaggle/input/deepfake-detection-challenge/test_videos"
PRETRAINED_MODELS_3D = [{'type':'i3d',
'path':"/kaggle/input/deepfake-detection-jph/j3d_e1_l0.1374.model"},
{'type':'res34',
'path':"/kaggle/input/deepfake-detection-jph/res34_1cy_minaug_nonorm_e4_l0.1794.model"},
{'type':'mc3_112',
'path':"/kaggle/input/deepfake-detection-jph/mc3_18_112_1cy_lilaug_nonorm_e9_l0.1905.model"},
{'type':'mc3_224',
'path':"/kaggle/input/deepfake-detection-jph/mc3_18_112t224_1cy_lilaug_nonorm_e7_l0.1901.model"},
{'type':'r2p1_112',
'path':"/kaggle/input/deepfake-detection-jph/r2p1_18_8_112tr_112te_e12_l0.1741.model"},
{'type':'i3d',
'path':"/kaggle/input/deepfake-detection-jph/i3dcutmix_e11_l0.1612.model"},
{'type':'r2p1_112',
'path':"/kaggle/input/deepfake-detection-jph/r2plus1dcutmix_112_e10_l0.1608.model"}]
MAX_FRAMES_TO_LOAD = 100
MIN_FRAMES_FOR_FACE = 30
MAX_FRAMES_FOR_FACE = 100
FACE_FRAMES = 10
MAX_FACES_HIGHTHRESH = 5
MAX_FACES_LOWTHRESH = 1
FACEDETECTION_DOWNSAMPLE = 0.25
MTCNN_THRESHOLDS =(0.8, 0.8, 0.9)
 MTCNN_THRESHOLDS_RETRY =(0.5, 0.5, 0.5)
 MMTNN_FACTOR = 0.71
TWO_FRAME_OVERLAP = False
PROB_MIN, PROB_MAX = 0.001, 0.999
REVERSE_PROBS = True
DEFAULT_MISSING_PRED = 0.5
USE_FACE_FUNCTION = np.mean
RATIO_3D = 1
OUTPUT_FACE_SIZE =(256, 256)
 PRE_INFERENCE_CROP =(224, 224)
 RATIO_2D = 1<load_pretrained>
def get_data(sz,bs,fname_emb=None,model=None):
tfms = tfms_from_model(resnet34, sz, crop_type=CropType.NO)
 tfms[0].tfms = [tfms[0].tfms[2],tfms[0].tfms[3]]
tfms[1].tfms = [tfms[1].tfms[2],tfms[1].tfms[3]]
df = pd.read_csv(LABELS)
 trn_df, val_df = train_test_split(df,test_size=0.2, random_state=42)
 ds = ImageData.get_ds(pdFilesDataset,(trn_df,TRAIN),(val_df,TRAIN), tfms)
 md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 if fname_emb != None and model != None:
emb = pd.read_csv(fname_emb)
 md.trn_dl.dataset.selection = Image_selection(md.trn_dl.dataset.fnames,trn_df,
emb,model)
 md.val_dl.dataset.selection = Image_selection(md.val_dl.dataset.fnames,val_df,
emb,model)
 return md
def load_video(filename, every_n_frames=None, specific_frames=None, to_rgb=True, rescale=None, inc_pil=False, max_frames=None):

 assert every_n_frames or specific_frames, "Must supply either every n_frames or specific_frames"
assert bool(every_n_frames)!= bool(specific_frames), "Supply either 'every_n_frames' or 'specific_frames', not both"
cap = cv2.VideoCapture(filename)
 n_frames_in = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
width_in = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height_in = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
if rescale:
rescale = rescale * 1920./np.max(( width_in, height_in))
width_out = int(width_in*rescale)if rescale else width_in
height_out = int(height_in*rescale)if rescale else height_in
if max_frames:
n_frames_in = min(n_frames_in, max_frames)
 if every_n_frames:
specific_frames = list(range(0,n_frames_in,every_n_frames))
n_frames_out = len(specific_frames)
 out_pil = []
out_video = np.empty(( n_frames_out, height_out, width_out, 3), np.dtype('uint8'))
i_frame_in = 0
i_frame_out = 0
ret = True
while(i_frame_in < n_frames_in and ret):
try:
try:
if every_n_frames == 1:
ret, frame_in = cap.read()
else:
ret = cap.grab()
if i_frame_in not in specific_frames:
i_frame_in += 1
continue
ret, frame_in = cap.retrieve()
if rescale:
frame_in = cv2.resize(frame_in,(width_out, height_out))
if to_rgb:
frame_in = cv2.cvtColor(frame_in, cv2.COLOR_BGR2RGB)
 except Exception as e:
print(f"Error for frame {i_frame_in} for video {filename}: {e}; using 0s")
 frame_in = np.zeros(( height_out, width_out, 3))
out_video[i_frame_out] = frame_in
i_frame_out += 1
if inc_pil:
try:
pil_img = Image.fromarray(frame_in)
 except Exception as e:
print(f"Using a blank frame for video {filename} frame {i_frame_in} as error {e}")
 pil_img = Image.fromarray(np.zeros(( 224,224,3), dtype=np.uint8))
out_pil.append(pil_img)
 i_frame_in += 1
except Exception as e:
print(f"Error for file {filename}: {e}")
 cap.release()
if inc_pil:
return out_video, out_pil, rescale
else:
return out_video, rescale
def get_roi_for_each_face(faces_by_frame, probs, video_shape, temporal_upsample, upsample=1):
frames_video, rows_video, cols_video, channels_video = video_shape
frames_video = math.ceil(frames_video)
 boolean_face_3d = np.zeros(( frames_video, rows_video, cols_video), dtype=np.bool)
 proba_face_3d = np.zeros(( frames_video, rows_video, cols_video)).astype('float32')
 for i_frame, faces in enumerate(faces_by_frame):
if faces is not None:
for i_face, face in enumerate(faces):
left, top, right, bottom = face
boolean_face_3d[i_frame, int(top):int(bottom), int(left):int(right)] = True
proba_face_3d[i_frame, int(top):int(bottom), int(left):int(right)] = probs[i_frame][i_face]
for i_frame, frame in enumerate(boolean_face_3d):
if i_frame == 0 or i_frame == frames_video-1:
continue
if True not in frame:
if TWO_FRAME_OVERLAP:
if i_frame > 1:
pre_overlap = boolean_face_3d[i_frame-1] | boolean_face_3d[i_frame-2]
else:
pre_overlap = boolean_face_3d[i_frame-1]
if i_frame < frames_video-2:
post_overlap = boolean_face_3d[i_frame+1] | boolean_face_3d[i_frame+2]
else:
post_overlap = boolean_face_3d[i_frame+1]
neighbour_overlap = pre_overlap & post_overlap
else:
neighbour_overlap = boolean_face_3d[i_frame-1] & boolean_face_3d[i_frame+1]
boolean_face_3d[i_frame] = neighbour_overlap
id_face_3d, n_faces = skimage.measure.label(boolean_face_3d, return_num=True)
 region_labels, counts = np.unique(id_face_3d, return_counts=True)
 region_labels, counts = region_labels[1:], counts[1:]
descending_size = np.argsort(counts)[::-1]
labels_by_size = region_labels[descending_size]
probs = [np.mean(proba_face_3d[id_face_3d == i_face])for i_face in region_labels]
descending_probs = np.argsort(probs)[::-1]
labels_by_probs = region_labels[descending_probs]
rois = [] ; face_maps = []
for i_face in labels_by_probs:
frames = np.where(np.any(id_face_3d == i_face, axis=(1, 2)) == True)
 starting_frame, ending_frame = frames[0].min() , frames[0].max()
cols, rows = [], []
for i_frame in range(starting_frame, ending_frame + 1):
rs = np.where(np.any(id_face_3d[i_frame] == i_face, axis=1)== True)
 rows.append(( rs[0].min() , rs[0].max()))
cs = np.where(np.any(id_face_3d[i_frame] == i_face, axis=0)== True)
 cols.append(( cs[0].min() , cs[0].max()))
frame_from, frame_to = starting_frame*temporal_upsample,(( ending_frame+1)*temporal_upsample)-1
rows_from, rows_to = np.array(rows)[:, 0].min() , np.array(rows)[:, 1].max()
cols_from, cols_to = np.array(cols)[:, 0].min() , np.array(cols)[:, 1].max()
frame_to = min(frame_to, frame_from + MAX_FRAMES_FOR_FACE)
 if frame_to - frame_from >= MIN_FRAMES_FOR_FACE:
tmp_face_map = id_face_3d.copy()
tmp_face_map[tmp_face_map != i_face] = 0
tmp_face_map[tmp_face_map == i_face] = 1
face_maps.append(tmp_face_map[frame_from//temporal_upsample:frame_to//temporal_upsample+1])
 rois.append(((frame_from, frame_to), 
(int(rows_from*upsample), int(rows_to*upsample)) , 
(int(cols_from*upsample), int(cols_to*upsample))))
return np.array(rois), face_maps<train_model>
md = get_data(sz,bs)
 x,y = next(iter(md.trn_dl))
print(x.shape, y[0].shape)
 def display_imgs(x,lbs=None):
columns = 3
rows = min(bs,16)
 ig,ax = plt.subplots(rows, columns, figsize=(columns*5, rows*5))
for i in range(rows):
for j in range(columns):
idx = j+i*columns
ax[i,j].imshow(( x[j][i,:,:,:]*255 ).astype(np.int))
ax[i,j].axis('off')
 if lbs is not None:
ax[i,j].text(10, 25, lbs[j][i], size=20, color='red')
 plt.subplots_adjust(wspace=0, hspace=0)
 plt.show()
to_lb = md.trn_ds.names.Id.to_dict()
lbs = [[to_lb[idx] for idx in y_cur.tolist() ] for y_cur in y]
display_imgs(( md.trn_ds.denorm(x[:,0,:,:,:]),md.trn_ds.denorm(x[:,1,:,:,:]),\
md.trn_ds.denorm(x[:,2,:,:,:])) ,lbs )
def get_center(bbox):
x1, y1, x2, y2 = bbox
return(x1+x2)/2,(y1+y2)/2
def get_coords(faces_roi):
coords = np.argwhere(faces_roi == 1)
 if coords.shape[0] == 0:
return None
y1, x1 = coords[0]
y2, x2 = coords[-1]
return x1, y1, x2, y2
def interpolate_center(c1, c2, length):
x1, y1 = c1
x2, y2 = c2
xi, yi = np.linspace(x1, x2, length), np.linspace(y1, y2, length)
 return np.vstack([xi, yi] ).transpose(1,0)
 def get_faces(faces_roi, upsample):
all_faces = []
rows = faces_roi[0].shape[1]
cols = faces_roi[0].shape[2]
for i in range(len(faces_roi)) :
faces = np.asarray([get_coords(faces_roi[i][j])for j in range(len(faces_roi[i])) ])
 if faces[0] is None: faces[0] = faces[1]
if faces[-1] is None: faces[-1] = faces[-2]
if None in faces:
raise Exception('This should not have happened...')
 all_faces.append(faces)
 extracted_faces = []
for face in all_faces:
max_dim = np.concatenate([face[:,2]-face[:,0],face[:,3]-face[:,1]])
 max_dim = np.percentile(max_dim, 90)
 max_dim = int(max_dim * 1.2)
 centers = np.asarray([get_center(_)for _ in face])
 centers = np.vstack([interpolate_center(centers[i], centers[i+1], length=10)for i in range(len(centers)-1)] ).astype('int')
 x1y1 = centers - max_dim // 2
x2y2 = centers + max_dim // 2
x1, y1 = x1y1[:,0], x1y1[:,1]
x2, y2 = x2y2[:,0], x2y2[:,1]
x2[x1 < 0] -= x1[x1 < 0]
y2[y1 < 0] -= y1[y1 < 0]
x1[x1 < 0] = 0
y1[y1 < 0] = 0
y1[y2 > rows] += rows - y2[y2 > rows]
x1[x2 > cols] += cols - x2[x2 > cols]
y2[y2 > rows] = rows
x2[x2 > cols] = cols
vidface = np.asarray([[x1[_],y1[_],x2[_],y2[_]] for _,c in enumerate(centers)])
 vidface =(vidface*upsample ).astype('int')
 extracted_faces.append(vidface)
 return extracted_faces
def detect_face_with_mtcnn(mtcnn_model, pil_frames, facedetection_upsample, video_shape, face_frames):
boxes, _probs = mtcnn_model.detect(pil_frames, landmarks=False)
 faces, faces_roi = get_roi_for_each_face(faces_by_frame=boxes, probs=_probs, video_shape=video_shape, temporal_upsample=face_frames, upsample=facedetection_upsample)
 coords = [] if len(faces_roi)== 0 else get_faces(faces_roi, upsample=facedetection_upsample)
 return faces, coords
def face_detection_wrapper(mtcnn_model, videopath, every_n_frames, facedetection_downsample, max_frames_to_load):
video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample, inc_pil=True, max_frames=max_frames_to_load)
 if len(pil_frames):
try:
faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn,
pil_frames=pil_frames,
facedetection_upsample=1/rescale,
video_shape=video.shape,
face_frames=every_n_frames)
 except RuntimeError:
print(f"Failed to process {videopath} ! Downsampling x2...")
 video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample/2, inc_pil=True, max_frames=max_frames_to_load)
 try:
faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn,
pil_frames=pil_frames,
facedetection_upsample=1/rescale,
video_shape=video.shape,
face_frames=every_n_frames)
 except RuntimeError:
print(f"Failed on downsample ! Skipping...")
 return [], []
else:
print('Failed to fetch frames ! Skipping...')
 return [], []
if len(faces)== 0:
print('Failed to find faces ! Upsampling x2...')
 try:
video, pil_frames, rescale = load_video(videopath, every_n_frames=every_n_frames, to_rgb=True, rescale=facedetection_downsample*2, inc_pil=True, max_frames=max_frames_to_load)
 faces, coords = detect_face_with_mtcnn(mtcnn_model=mtcnn,
pil_frames=pil_frames,
facedetection_upsample=1/rescale,
video_shape=video.shape,
face_frames=every_n_frames)
 except Exception as e:
print(e)
 return [], []
return faces, coords<feature_engineering>
class Metric(nn.Module):
def __init__(self, emb_sz=64):
super().__init__()
self.l = nn.Linear(emb_sz*2,emb_sz*2,False)
 def forward(self,d):
d2 = d.pow(2)
 d = self.l(torch.cat(( d,d2),dim=-1))
x = d.pow(2 ).sum(dim=-1)
 return x.view(-1 )
videopaths = sorted(glob(os.path.join(INPUT_DIR, "*.mp4")))
 print(f'Found {len(videopaths)} videos !')
 mtcnn = MTCNN(margin=0, keep_all=True, post_process=False, select_largest=False, device='cuda:0', thresholds=MTCNN_THRESHOLDS, factor=MMTNN_FACTOR)
 faces_by_videopath = {}
coords_by_videopath = {}
for i_video, videopath in enumerate(tqdm(videopaths)) :
faces, coords = face_detection_wrapper(mtcnn, videopath, every_n_frames=FACE_FRAMES, facedetection_downsample=FACEDETECTION_DOWNSAMPLE, max_frames_to_load=MAX_FRAMES_TO_LOAD)
 if len(faces):
faces_by_videopath[videopath] = faces[:MAX_FACES_HIGHTHRESH]
coords_by_videopath[videopath] = coords[:MAX_FACES_HIGHTHRESH]
else:
print(f"Found no faces for {videopath} !")
 del mtcnn
gc.collect()
videopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}
print(f"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing")
<feature_engineering>
def resnext50(pretrained=True):
model = resnext_50_32x4d()
name = 'resnext_50_32x4d.pth'
if pretrained:
path = os.path.join(MODLE_INIT,name)
 load_model(model, path)
 return model
class TripletResneXt50(nn.Module):
def __init__(self, pre=True, emb_sz=64, ps=0.5):
super().__init__()
encoder = resnext50(pretrained=pre)
 self.cnn = nn.DataParallel(nn.Sequential(encoder[0],encoder[1],nn.ReLU() ,
encoder[3],encoder[4],encoder[5],encoder[6],encoder[7]))
self.head = nn.DataParallel(nn.Sequential(AdaptiveConcatPool2d() , Flatten() ,
nn.Dropout(ps),nn.Linear(4096, 512), nn.ReLU() ,
nn.BatchNorm1d(512), nn.Dropout(ps), nn.Linear(512, emb_sz)))
 self.metric = nn.DataParallel(Metric(emb_sz))
def forward(self,x):
x1,x2,x3 = x[:,0,:,:,:],x[:,1,:,:,:],x[:,2,:,:,:]
x1 = self.head(self.cnn(x1))
x2 = self.head(self.cnn(x2))
x3 = self.head(self.cnn(x3))
x = torch.cat(( x1,x2,x3))
sz = x.shape[0]
x1 = x.unsqueeze(1 ).expand(( sz,sz,-1))
x2 = x1.transpose(0,1)
 d =(x1 - x2 ).view(sz*sz,-1)
 return self.metric(d)
 def get_embedding(self, x):
return self.head(self.cnn(x))
def get_d(self, x0, x):
d =(x - x0)
 return self.metric(d)
 class ResNeXt50Model() :
def __init__(self,pre=True,name='TripletResneXt50',**kwargs):
self.model = to_gpu(TripletResneXt50(pre=True,**kwargs))
self.name = name
def get_layer_groups(self, precompute):
m = self.model.module if isinstance(self.model,FP16)else self.model
if precompute:
return [m.head] + [m.metric]
c = children(m.cnn.module)
 return list(split_by_idxs(c,[5])) + [m.head] + [m.metric]
mtcnn = MTCNN(margin=0, keep_all=True, post_process=False, select_largest=False ,device='cuda:0', thresholds=MTCNN_THRESHOLDS_RETRY, factor=MMTNN_FACTOR)
 for i_video, videopath in enumerate(tqdm(videopaths_missing_faces)) :
faces, coords = face_detection_wrapper(mtcnn,
videopath,
every_n_frames=FACE_FRAMES,
facedetection_downsample=FACEDETECTION_DOWNSAMPLE,
max_frames_to_load=MAX_FRAMES_TO_LOAD)
 if len(faces):
faces_by_videopath[videopath] = faces[:MAX_FACES_LOWTHRESH]
coords_by_videopath[videopath] = coords[:MAX_FACES_LOWTHRESH]
else:
print(f"Found no faces for {videopath} !")
 del mtcnn
gc.collect()
faces_by_videopath = dict(sorted(faces_by_videopath.items() , key=lambda x: x[0]))
videopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}
print(f"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing" )<sort_values>
def get_densenet169(pre=True):
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 layers = cut_model(cut_model(densenet169(pre), -1)[0],-1)
 return nn.Sequential(*layers)
 class TripletDenseNet169(nn.Module):
def __init__(self, pre=True, emb_sz=64, ps=0.5):
super().__init__()
encoder = get_densenet169(pre)
 self.cnn = nn.DataParallel(nn.Sequential(encoder[0],encoder[1],nn.ReLU() ,
encoder[3],encoder[4],encoder[5],encoder[6],encoder[7],
encoder[8],encoder[9],encoder[10]))
self.head = nn.DataParallel(nn.Sequential(AdaptiveConcatPool2d() , Flatten() ,
nn.Dropout(ps),nn.Linear(3328, 512), nn.ReLU() ,
nn.BatchNorm1d(512), nn.Dropout(ps), nn.Linear(512, emb_sz)))
 self.metric = nn.DataParallel(Metric(emb_sz))
def forward(self,x):
x1,x2,x3 = x[:,0,:,:,:],x[:,1,:,:,:],x[:,2,:,:,:]
x1 = self.head(self.cnn(x1))
x2 = self.head(self.cnn(x2))
x3 = self.head(self.cnn(x3))
x = torch.cat(( x1,x2,x3))
sz = x.shape[0]
x1 = x.unsqueeze(1 ).expand(( sz,sz,-1))
x2 = x1.transpose(0,1)
 d =(x1 - x2 ).view(sz*sz,-1)
 return self.metric(d)
 def get_embedding(self, x):
return self.head(self.cnn(x))
def get_d(self, x0, x):
d =(x - x0)
 return self.metric(d)
 class DenseNet169Model() :
def __init__(self,pre=True,name='TripletDenseNet169',**kwargs):
self.model = to_gpu(TripletDenseNet169(pre=True,**kwargs))
self.name = name
def get_layer_groups(self, precompute):
m = self.model.module if isinstance(self.model,FP16)else self.model
if precompute:
return [m.head] + [m.metric]
c = children(m.cnn.module)
 return list(split_by_idxs(c,[8])) + [m.head] + [m.metric]
faces_by_videopath = dict(sorted(faces_by_videopath.items() , key=lambda x: x[0]))
videopaths_missing_faces = {p for p in videopaths if p not in faces_by_videopath}
print(f"Found faces for {len(faces_by_videopath)} videos; {len(videopaths_missing_faces)} missing" )<categorify>
def get_densenet121(pre=True):
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 layers = cut_model(cut_model(densenet121(pre), -1)[0],-1)
 return nn.Sequential(*layers)
 class TripletDenseNet121(nn.Module):
def __init__(self, pre=True, emb_sz=64, ps=0.5):
super().__init__()
encoder = get_densenet121(pre)
 self.cnn = nn.DataParallel(nn.Sequential(encoder[0],encoder[1],nn.ReLU() ,
encoder[3],encoder[4],encoder[5],encoder[6],encoder[7],
encoder[8],encoder[9],encoder[10]))
self.head = nn.DataParallel(nn.Sequential(AdaptiveConcatPool2d() , Flatten() ,
nn.BatchNorm1d(2048),nn.Dropout(ps),nn.Linear(2048, 1024), nn.ReLU() ,
nn.BatchNorm1d(1024), nn.Dropout(ps), nn.Linear(1024, emb_sz)))
 self.metric = nn.DataParallel(Metric(emb_sz))
def forward(self,x):
x1,x2,x3 = x[:,0,:,:,:],x[:,1,:,:,:],x[:,2,:,:,:]
x1 = self.head(self.cnn(x1))
x2 = self.head(self.cnn(x2))
x3 = self.head(self.cnn(x3))
x = torch.cat(( x1,x2,x3))
sz = x.shape[0]
x1 = x.unsqueeze(1 ).expand(( sz,sz,-1))
x2 = x1.transpose(0,1)
 d =(x1 - x2 ).view(sz*sz,-1)
 return self.metric(d)
 def get_embedding(self, x):
return self.head(self.cnn(x))
def get_d(self, x0, x):
d =(x - x0)
 return self.metric(d)
 class DenseNet121Model() :
def __init__(self,pre=True,name='TripletDenseNet21',**kwargs):
self.model = to_gpu(TripletDenseNet121(pre=True,**kwargs))
self.name = name
def get_layer_groups(self, precompute):
m = self.model.module if isinstance(self.model,FP16)else self.model
if precompute:
return [m.head] + [m.metric]
c = children(m.cnn.module)
 return list(split_by_idxs(c,[8])) + [m.head] + [m.metric]
class MaxPool3dSamePadding(nn.MaxPool3d):
def compute_pad(self, dim, s):
if s % self.stride[dim] == 0:
return max(self.kernel_size[dim] - self.stride[dim], 0)
 else:
return max(self.kernel_size[dim] -(s % self.stride[dim]), 0)
 def forward(self, x): 
(batch, channel, t, h, w)= x.size()
out_t = np.ceil(float(t)/ float(self.stride[0]))
out_h = np.ceil(float(h)/ float(self.stride[1]))
out_w = np.ceil(float(w)/ float(self.stride[2]))
pad_t = self.compute_pad(0, t)
 pad_h = self.compute_pad(1, h)
 pad_w = self.compute_pad(2, w)
 pad_t_f = pad_t // 2
pad_t_b = pad_t - pad_t_f
pad_h_f = pad_h // 2
pad_h_b = pad_h - pad_h_f
pad_w_f = pad_w // 2
pad_w_b = pad_w - pad_w_f
pad =(pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
 x = F.pad(x, pad)
 return super(MaxPool3dSamePadding, self ).forward(x)
 class Unit3D(nn.Module):
def __init__(self, in_channels,
output_channels,
kernel_shape=(1, 1, 1),
stride=(1, 1, 1),
padding=0,
activation_fn=F.relu,
use_batch_norm=True,
use_bias=False,
name='unit_3d'):

 super(Unit3D, self ).__init__()
self._output_channels = output_channels
self._kernel_shape = kernel_shape
self._stride = stride
self._use_batch_norm = use_batch_norm
self._activation_fn = activation_fn
self._use_bias = use_bias
self.name = name
self.padding = padding
self.conv3d = nn.Conv3d(in_channels=in_channels,
out_channels=self._output_channels,
kernel_size=self._kernel_shape,
stride=self._stride,
padding=0,
bias=self._use_bias)
 if self._use_batch_norm:
self.bn = nn.BatchNorm3d(self._output_channels, eps=0.001, momentum=0.01)
 def compute_pad(self, dim, s):
if s % self._stride[dim] == 0:
return max(self._kernel_shape[dim] - self._stride[dim], 0)
 else:
return max(self._kernel_shape[dim] -(s % self._stride[dim]), 0)
 def forward(self, x): 
(batch, channel, t, h, w)= x.size()
out_t = np.ceil(float(t)/ float(self._stride[0]))
out_h = np.ceil(float(h)/ float(self._stride[1]))
out_w = np.ceil(float(w)/ float(self._stride[2]))
pad_t = self.compute_pad(0, t)
 pad_h = self.compute_pad(1, h)
 pad_w = self.compute_pad(2, w)
 pad_t_f = pad_t // 2
pad_t_b = pad_t - pad_t_f
pad_h_f = pad_h // 2
pad_h_b = pad_h - pad_h_f
pad_w_f = pad_w // 2
pad_w_b = pad_w - pad_w_f
pad =(pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
 x = F.pad(x, pad)
 x = self.conv3d(x)
 if self._use_batch_norm:
x = self.bn(x)
 if self._activation_fn is not None:
x = self._activation_fn(x)
 return x
class InceptionModule(nn.Module):
def __init__(self, in_channels, out_channels, name):
super(InceptionModule, self ).__init__()
self.b0 = Unit3D(in_channels=in_channels, output_channels=out_channels[0], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_0/Conv3d_0a_1x1')
 self.b1a = Unit3D(in_channels=in_channels, output_channels=out_channels[1], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_1/Conv3d_0a_1x1')
 self.b1b = Unit3D(in_channels=out_channels[1], output_channels=out_channels[2], kernel_shape=[3, 3, 3],
name=name + '/Branch_1/Conv3d_0b_3x3')
 self.b2a = Unit3D(in_channels=in_channels, output_channels=out_channels[3], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_2/Conv3d_0a_1x1')
 self.b2b = Unit3D(in_channels=out_channels[3], output_channels=out_channels[4], kernel_shape=[3, 3, 3],
name=name + '/Branch_2/Conv3d_0b_3x3')
 self.b3a = MaxPool3dSamePadding(kernel_size=[3, 3, 3],
stride=(1, 1, 1), padding=0)
 self.b3b = Unit3D(in_channels=in_channels, output_channels=out_channels[5], kernel_shape=[1, 1, 1], padding=0,
name=name + '/Branch_3/Conv3d_0b_1x1')
 self.name = name
def forward(self, x):
b0 = self.b0(x)
 b1 = self.b1b(self.b1a(x))
b2 = self.b2b(self.b2a(x))
b3 = self.b3b(self.b3a(x))
return torch.cat([b0, b1, b2, b3], dim=1)
 class InceptionI3d(nn.Module):

 VALID_ENDPOINTS =(
 'Conv3d_1a_7x7',
'MaxPool3d_2a_3x3',
'Conv3d_2b_1x1',
'Conv3d_2c_3x3',
'MaxPool3d_3a_3x3',
'Mixed_3b',
'Mixed_3c',
'MaxPool3d_4a_3x3',
'Mixed_4b',
'Mixed_4c',
'Mixed_4d',
'Mixed_4e',
'Mixed_4f',
'MaxPool3d_5a_2x2',
'Mixed_5b',
'Mixed_5c',
'Logits',
'Predictions', 
)
 def __init__(self, num_classes=400, spatial_squeeze=True, output_method='per_frame',
final_endpoint='Logits', name='inception_i3d', in_channels=3, dropout_keep_prob=0.5):

 if final_endpoint not in self.VALID_ENDPOINTS:
raise ValueError('Unknown final endpoint %s' % final_endpoint)
 super(InceptionI3d, self ).__init__()
self.output_method = output_method
assert output_method in('per_frame', 'avg_pool', 'max_pool', 'dual_pool')
 self._num_classes = num_classes
self._spatial_squeeze = spatial_squeeze
self._final_endpoint = final_endpoint
self.logits = None
if self._final_endpoint not in self.VALID_ENDPOINTS:
raise ValueError('Unknown final endpoint %s' % self._final_endpoint)
 self.end_points = {}
end_point = 'Conv3d_1a_7x7'
self.end_points[end_point] = Unit3D(in_channels=in_channels, output_channels=64, kernel_shape=[7, 7, 7],
stride=(2, 2, 2), padding=(3, 3, 3), name=name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_2a_3x3'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Conv3d_2b_1x1'
self.end_points[end_point] = Unit3D(in_channels=64, output_channels=64, kernel_shape=[1, 1, 1], padding=0,
name=name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Conv3d_2c_3x3'
self.end_points[end_point] = Unit3D(in_channels=64, output_channels=192, kernel_shape=[3, 3, 3], padding=1,
name=name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_3a_3x3'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_3b'
self.end_points[end_point] = InceptionModule(192, [64, 96, 128, 16, 32, 32], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_3c'
self.end_points[end_point] = InceptionModule(256, [128, 128, 192, 32, 96, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_4a_3x3'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[3, 3, 3], stride=(2, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4b'
self.end_points[end_point] = InceptionModule(128 + 192 + 96 + 64, [192, 96, 208, 16, 48, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4c'
self.end_points[end_point] = InceptionModule(192 + 208 + 48 + 64, [160, 112, 224, 24, 64, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4d'
self.end_points[end_point] = InceptionModule(160 + 224 + 64 + 64, [128, 128, 256, 24, 64, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4e'
self.end_points[end_point] = InceptionModule(128 + 256 + 64 + 64, [112, 144, 288, 32, 64, 64], name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_4f'
self.end_points[end_point] = InceptionModule(112 + 288 + 64 + 64, [256, 160, 320, 32, 128, 128],
name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'MaxPool3d_5a_2x2'
self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[2, 2, 2], stride=(2, 2, 2),
padding=0)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_5b'
self.end_points[end_point] = InceptionModule(256 + 320 + 128 + 128, [256, 160, 320, 32, 128, 128],
name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Mixed_5c'
self.end_points[end_point] = InceptionModule(256 + 320 + 128 + 128, [384, 192, 384, 48, 128, 128],
name + end_point)
 if self._final_endpoint == end_point: return
end_point = 'Logits'
self.avg_pool = nn.AvgPool3d(kernel_size=[2, 7, 7],
stride=(1, 1, 1))
self.dropout = nn.Dropout(dropout_keep_prob)
 self.logits = Unit3D(in_channels=384 + 384 + 128 + 128, output_channels=self._num_classes,
kernel_shape=[1, 1, 1],
padding=0,
activation_fn=None,
use_batch_norm=False,
use_bias=True,
name='logits')
 self.build()
def replace_logits(self, num_classes):
self._num_classes = num_classes
self.logits = Unit3D(in_channels=384 + 384 + 128 + 128, output_channels=self._num_classes,
kernel_shape=[1, 1, 1],
padding=0,
activation_fn=None,
use_batch_norm=False,
use_bias=True,
name='logits')
 def build(self):
for k in self.end_points.keys() :
self.add_module(k, self.end_points[k])
 def forward(self, x):
for end_point in self.VALID_ENDPOINTS:
if end_point in self.end_points:
x = self._modules[end_point](x)
 x = self.logits(self.dropout(self.avg_pool(x)))
 if self._spatial_squeeze:
logits = x.squeeze(3 ).squeeze(3)
 if self.output_method == 'per_frame':
return F.interpolate(logits, 64, mode='linear')
 elif self.output_method == 'avg_pool':
avg_all_frames = F.adaptive_avg_pool1d(logits, output_size=1)
 return avg_all_frames.squeeze(-1)
 def extract_features(self, x):
for end_point in self.VALID_ENDPOINTS:
if end_point in self.end_points:
x = self._modules[end_point](x)
 return self.avg_pool(x )<normalization>
class Contrastive_loss(nn.Module):
def __init__(self, m=10.0, wd=1e-4):
super().__init__()
self.m, self.wd = m, wd
def forward(self, d, target):
d = d.float()
t = torch.cat(target)
 sz = t.shape[0]
t1 = t.unsqueeze(1 ).expand(( sz,sz))
t2 = t1.transpose(0,1)
 y =(( t1==t2)+ to_gpu(torch.eye(sz ).byte())).view(-1)
 loss_p = d[y==1]
loss_n = F.relu(self.m - torch.sqrt(d[y==0])) **2
loss = torch.cat(( loss_p,loss_n),0)
 loss = loss[torch.nonzero(loss ).squeeze() ]
loss = loss.mean() if loss.shape[0] > 0 else loss.sum()
loss += self.wd*(d**2 ).mean()
return loss
def T_acc(d, target):
sz = target[0].shape[0]
lp = [3*sz*i + i+sz for i in range(sz)]
ln = [3*sz*i + i+2*sz for i in range(sz)]
dp, dn = d[lp], d[ln]
return(dp < dn ).float().mean()
def BH_acc(d, target):
t = torch.cat(target)
 sz = t.shape[0]
t1 = t.unsqueeze(1 ).expand(( sz,sz))
t2 = t1.transpose(0,1)
 y =(t1==t2)
 d = d.float().view(sz,sz)
 BH = []
for i in range(sz):
dp = d[i,y[i,:] == 1].max()
dn = d[i,y[i,:] == 0].min()
BH.append(dp < dn)
 return torch.FloatTensor(BH ).float().mean()
def conv3x3x3(in_planes, out_planes, stride=1):
return nn.Conv3d(
 in_planes,
out_planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
 def downsample_basic_block(x, planes, stride):
out = F.avg_pool3d(x, kernel_size=1, stride=stride)
 zero_pads = torch.Tensor(
 out.size(0), planes - out.size(1), out.size(2), out.size(3),
out.size(4)).zero_()
if isinstance(out.data, torch.cuda.FloatTensor):
zero_pads = zero_pads.cuda()
out = Variable(torch.cat([out.data, zero_pads], dim=1))
return out
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3x3(inplanes, planes, stride)
 self.bn1 = nn.BatchNorm3d(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3x3(planes, planes)
 self.bn2 = nn.BatchNorm3d(planes)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm3d(planes)
 self.conv2 = nn.Conv3d(
 planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
 self.bn2 = nn.BatchNorm3d(planes)
 self.conv3 = nn.Conv3d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm3d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self,
block,
layers,
sample_size,
sample_duration,
shortcut_type='B',
num_classes=400):
self.inplanes = 64
super(ResNet, self ).__init__()
self.conv1 = nn.Conv3d(
 3,
64,
kernel_size=7,
stride=(1, 2, 2),
padding=(3, 3, 3),
bias=False)
 self.bn1 = nn.BatchNorm3d(64)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool3d(kernel_size=(3, 3, 3), stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0], shortcut_type)
 self.layer2 = self._make_layer(
 block, 128, layers[1], shortcut_type, stride=2)
 self.layer3 = self._make_layer(
 block, 256, layers[2], shortcut_type, stride=2)
 self.layer4 = self._make_layer(
 block, 512, layers[3], shortcut_type, stride=2)
 self.avgpool = nn.AdaptiveAvgPool3d(1)
 self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv3d):
m.weight = nn.init.kaiming_normal(m.weight, mode='fan_out')
 elif isinstance(m, nn.BatchNorm3d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, shortcut_type, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
if shortcut_type == 'A':
downsample = partial(
 downsample_basic_block,
planes=planes * block.expansion,
stride=stride)
 else:
downsample = nn.Sequential(
 nn.Conv3d(
 self.inplanes,
planes * block.expansion,
kernel_size=1,
stride=stride,
bias=False), nn.BatchNorm3d(planes * block.expansion))
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return x
def get_fine_tuning_parameters(model, ft_begin_index):
if ft_begin_index == 0:
return model.parameters()
ft_module_names = []
for i in range(ft_begin_index, 5):
ft_module_names.append('layer{}'.format(i))
ft_module_names.append('fc')
 parameters = []
for k, v in model.named_parameters() :
for ft_module in ft_module_names:
if ft_module in k:
parameters.append({'params': v})
 break
else:
parameters.append({'params': v, 'lr': 0.0})
 return parameters
def resnet18(**kwargs):

 model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
 return model
def resnet34(**kwargs):

 model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)
 return model<train_model>
class BH_loss(nn.Module):
def __init__(self, n=1, m=0.0):
super().__init__()
self.n = n
self.m = m
def forward(self, input, target, size_average=True):
t = torch.cat(target)
 sz = t.shape[0]
t1 = t.unsqueeze(1 ).expand(( sz,sz))
t2 = t1.transpose(0,1)
 y =(( t1==t2)+ to_gpu(torch.eye(sz ).byte()))
d = input.float().view(sz,sz)
 D = []
for i in range(2*sz//3):
dp = d[i,y[i,:] == 1].max()
dn = d[i,y[i,:] == 0]
dist, idxs = torch.sort(dn)
 n = min(self.n,dn.shape[0])
 for j in range(n):
D.append(( self.m + dp - dn[idxs[j]] ).unsqueeze(0))
loss = torch.log1p(torch.exp(torch.cat(D)))
 loss = loss.mean() if size_average else loss.sum()
return loss
class HFlipWrapper(nn.Module):
def __init__(self, model, flip_dim=(-1,)) :
super(HFlipWrapper, self ).__init__()
self.model = model
self.flip_dim = flip_dim
def forward(self, x):
with torch.no_grad() :
xf = torch.flip(x, self.flip_dim)
 pred = self.model(torch.stack([x, xf]))
return pred<choose_model_class>
learner = ConvLearner(md,DenseNet121Model(ps=0.0,emb_sz=n_embedding))
learner.opt_fn = optim.Adam
learner.clip = 1.0
learner.crit = Contrastive_loss()
learner.metrics = [T_acc,BH_acc]
learner.freeze_to(-2)
 learner
models_3d = []
for modeldict in PRETRAINED_MODELS_3D:
if modeldict['type'] == 'i3d':
model = InceptionI3d(157, in_channels=3, output_method='avg_pool')
 model.replace_logits(2)
 model = model.cuda()
model.eval()
model.load_state_dict(torch.load(modeldict['path']))
models_3d.append({'norm':'i3d', 'model':model})
 elif modeldict['type'] == 'res18':
model = resnet18(num_classes=2, shortcut_type='A', sample_size=224, sample_duration=32)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'nil', 'model':model})
 elif modeldict['type'] == 'res34':
model = resnet34(num_classes=2, shortcut_type='A', sample_size=224, sample_duration=32)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'nil', 'model':model})
 elif modeldict['type'] == 'mc3_112':
model = mc3_18(num_classes=2)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'112_imagenet', 'model':model})
 elif modeldict['type'] == 'mc3_224':
model = mc3_18(num_classes=2)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'224_imagenet', 'model':model})
 elif modeldict['type'] == 'r2p1_112':
model = r2plus1d_18(num_classes=2)
 model.load_state_dict(torch.load(modeldict['path']))
model = model.cuda()
model.eval()
models_3d.append({'norm':'112_imagenet', 'model':model})
 else:
raise ValueError(f"Unknown model type {modeldict['type']}" )<load_pretrained>
lr = 1e-3
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lr,1 )
with open('/kaggle/input/deepfake/deepfake/deepfake/skp/configs/experiments/experiment001.yaml')as f:
CFG = yaml.load(f, Loader=yaml.FullLoader)
 CFG['model']['params']['pretrained'] = None
model2d = build_model(CFG['model']['name'], CFG['model']['params'])
 model2d.load_state_dict(torch.load('/kaggle/input/deepfake/SRXT50_094_VM-0.2504.PTH'))
model2d = model2d.eval().cuda()
loader = build_dataloader(CFG, data_info={'vidfiles': [], 'labels': []}, mode='predict' )<categorify>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/2,4,cycle_len=1,use_clr=(10,20))
learner.fit(lrs/4,4,cycle_len=2,use_clr=(10,20))
learner.save('model0' )
def resize_and_square_face(video, output_size):
input_size = max(video.shape[1], video.shape[2])
 out_video = np.empty(( len(video), output_size[0], output_size[1], 3), np.dtype('uint8'))
for i_frame, frame in enumerate(video):
padded_image = np.zeros(( input_size, input_size, 3))
padded_image[0:frame.shape[0], 0:frame.shape[1]] = frame
if(input_size, input_size)!= output_size:
frame = cv2.resize(padded_image,(output_size[0], output_size[1])).astype(np.uint8)
 else:
frame = padded_image.astype(np.uint8)
 out_video[i_frame] = frame
return out_video<define_variables>
def extract_embedding(model,path):
tfms = tfms_from_model(resnet34, sz, crop_type=CropType.NO)
 tfms[0].tfms = [tfms[0].tfms[2],tfms[0].tfms[3]]
tfms[1].tfms = [tfms[1].tfms[2],tfms[1].tfms[3]]
ds = ImageData.get_ds(FilesDataset_single,(None,TRAIN),(None,TRAIN),
tfms, test=(None,path))
md = ImageData(PATH, ds, 3*bs, num_workers=nw, classes=None)
 model.eval()
with torch.no_grad() :
preds = torch.zeros(( len(md.test_dl.dataset), n_embedding))
start=0
for i,(x,y)in tqdm_notebook(enumerate(md.test_dl,start=0),
total=len(md.test_dl)) :
size = x.shape[0]
m = model.module if isinstance(model,FP16)else model
preds[start:start+size,:] = m.get_embedding(x.half())
 start+= size
return preds, [os.path.basename(name)for name in md.test_dl.dataset.fnames]
def get_last_frame_needed_across_faces(faces):
last_frame = 0
for face in faces: 
(frame_from, frame_to),(row_from, row_to),(col_from, col_to)= face
last_frame = max(frame_to, last_frame)
 return last_frame<define_variables>
emb, names = extract_embedding(learner.model,TRAIN)
 df = pd.DataFrame({'files':names,'emb':emb.tolist() })
 df.emb = df.emb.map(lambda emb: ' '.join(list([str(i)for i in emb])))
 df.to_csv('train_emb.csv', header=True, index=False)
 emb, names = extract_embedding(learner.model,TEST)
 df_test = pd.DataFrame({'files':names,'emb':emb.tolist() })
 df_test.emb = df_test.emb.map(lambda emb: ' '.join(list([str(i)for i in emb])))
 df_test.to_csv('test_emb.csv', header=True, index=False )
predictions = defaultdict(list )<categorify>
dcut = 18.0
get_val_nbs(learner.model,df,dcut=dcut,out='val0.csv' )
test_transforms_114_imagenet = A.Compose([A.Resize(height=112, width=112),
A.Normalize() ])
 test_transforms_224_imagenet = A.Compose([A.Normalize() ] )<drop_column>
def get_test_nbs(model,trn_emb,test_emb,out='test.csv',\
submission='submission.csv',dcut=None):
trn_emb = trn_emb.copy()
data = pd.read_csv(LABELS ).set_index('Image')
 trn_emb['emb'] = [[float(i)for i in s.split() ] for s in trn_emb['emb']]
trn_emb.set_index('files',inplace=True)
 train_df = data.join(trn_emb)
 train_df = train_df.reset_index()
train_preds = np.array(train_df.emb.tolist())
 test_emb = test_emb.copy()
test_emb['emb'] = [[float(i)for i in s.split() ] for s in test_emb['emb']]
test_emb['Image'] = test_emb['files']
test_emb.set_index('files',inplace=True)
 test_df = test_emb.reset_index()
test_preds = np.array(test_df.emb.tolist())
 train_preds = torch.from_numpy(train_preds ).half().cuda()
test_preds = torch.from_numpy(test_preds ).half().cuda()
test_d,test_idxs = get_nbs(model,test_preds,train_preds)
 s = []
for l1 in test_d.tolist() :
s.append(' '.join([str(l2)for l2 in l1]))
test_df['d'] = s
test_df['nbs'] = [' '.join(train_df.loc[test_idxs[index]].Id.tolist())\
for index, row in test_df.iterrows() ]
test_df[['Image','nbs','d']].to_csv(out, header=True, index=False)
 if dcut is not None:
pred = []
for idx, row in test_df.iterrows() :
nbs = dict()
for i in range(0,16):
nb = test_idxs[idx,i]
l, s = train_df.loc[nb].Id, test_d[idx,i]
if s > dcut and 'new_whale' not in nbs: nbs['new_whale'] = dcut
if l not in nbs: nbs[l] = s
if len(nbs)>= 5: break
nbs_sorted = list(nbs.items())
 p = ' '.join([lb[0] for lb in nbs_sorted])
 pred.append({'Image':row.files,'Id':p})
 pd.DataFrame(pred ).to_csv(submission,index=False )
del model
for modeldict in models_3d:
del modeldict['model']
del modeldict
del models_3d
gc.collect()<feature_engineering>
get_test_nbs(learner.model,df,df_test,dcut=dcut,out='test0.csv',\
submission='submission0.csv' )
for videopath in videopaths:
videoname = os.path.basename(videopath)
 if videoname not in predictions:
predictions[videoname] = [DEFAULT_MISSING_PRED]<prepare_output>
md = get_data(sz,bs,'train_emb.csv',learner.model)
 learner.set_data(md)
 learner.unfreeze()
learner.half()
x,y = next(iter(md.trn_dl))
to_lb = md.trn_ds.names.Id.to_dict()
lbs = [[to_lb[idx] for idx in y_cur.tolist() ] for y_cur in y]
display_imgs(( md.trn_ds.denorm(x[:,0,:,:,:]),md.trn_ds.denorm(x[:,1,:,:,:]),\
md.trn_ds.denorm(x[:,2,:,:,:])) ,lbs )
preds_ensembled = {}
for videopath, pred in predictions.items() :
preds_ensembled[videopath] = np.clip(np.mean(pred), PROB_MIN, PROB_MAX )<save_to_csv>
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/16,5,cycle_len=4,use_clr=(10,20))
learner.save('model' )
print(f"{len(preds_ensembled)} predictions")
 with open("submission.csv", 'w')as f:
f.write("filename,label
")
 for filename, label in preds_ensembled.items() :
f.write(f"{filename},{label}
" )<import_modules>
emb, names = extract_embedding(learner.model,TRAIN)
 df = pd.DataFrame({'files':names,'emb':emb.tolist() })
 df.emb = df.emb.map(lambda emb: ' '.join(list([str(i)for i in emb])))
 df.to_csv('train_emb.csv', header=True, index=False)
 emb, names = extract_embedding(learner.model,TEST)
 df_test = pd.DataFrame({'files':names,'emb':emb.tolist() })
 df_test.emb = df_test.emb.map(lambda emb: ' '.join(list([str(i)for i in emb])))
 df_test.to_csv('test_emb.csv', header=True, index=False )
<install_modules><EOS> 
dcut = 23.0
get_val_nbs(learner.model,df,dcut=dcut,out='val.csv')
 get_test_nbs(learner.model,df,df_test,dcut=dcut,out='test.csv',\
submission='submission.csv' )