Kaggle competition: Deepfake Detection Challenge
<SOS> metric: LogLoss Kaggle data source: deepfake-detection-challenge<train_model>
from IPython.display import Image
fw='/tmp/checkpoint'
handle_base='mobilenet_v2_100_%d'%img_size2
mhandle='https://tfhub.dev/google/imagenet/{}/classification/4'\ 
.format(handle_base)
 hub_model=premodel(img_size2,1024,mhandle,num_classes,
'softmax','sparse_categorical_crossentropy')
 history=hub_model.fit(x=x_train3,y=y_train3,batch_size=128,
epochs=20,callbacks=cb(fw),verbose=0,
validation_data=(x_valid3,y_valid3))<train_model>
Image('.. /input/deepfake-kernel-data/google_cloud_compute_engine_launch_vm.png' )
hub_model.load_weights('/tmp/checkpoint')
 hub_model.evaluate(x_test3,y_test3,verbose=0 )<set_options>
Image('.. /input/deepfake-kernel-data/google_cloud_vm.png' )
%matplotlib inline
sns.set(style='white', context='notebook', palette='deep' )<load_from_csv>
Image('.. /input/deepfake-kernel-data/lr_15e-2_epochs_42_patience_5.png' )
df_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 df_test = pd.read_csv(".. /input/digit-recognizer/test.csv" )<prepare_x_and_y>
Image('.. /input/deepfake-kernel-data/lr_2e-3_epochs_10_patience_5.png' )
X_train = np.array(df_train.drop(['label'], axis=1), dtype="float32")/ 255.0
X_train = X_train.reshape(-1, 28, 28, 1)
 Y_train = to_categorical(df_train['label'], num_classes = 10)
 X_test = np.array(df_test, dtype="float32")/ 255.0
X_test = X_test.reshape(-1, 28, 28, 1 )<split>
Image('.. /input/deepfake-kernel-data/lr_2e-3_epochs_20_patience_5.png' )
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.05, random_state=34 )<choose_model_class>
Image('.. /input/deepfake-kernel-data/lr_4e-3_epochs_12_patience_2.png' )
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(10, activation = "softmax")) 
<define_variables>
Image('.. /input/deepfake-kernel-data/lr_4e-3_epochs_30_patience_2.png' )
datagen = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<choose_model_class>
Image('.. /input/deepfake-kernel-data/google_cloud_vm_deepfake_training_screenshot.png' )
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
%matplotlib inline
warnings.filterwarnings("ignore" )
epochs=50
batch_size=128
history = model.fit(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs,
validation_data =(X_val,Y_val),
verbose = 2,
steps_per_epoch=X_train.shape[0] // batch_size,
callbacks=[learning_rate_reduction] )<predict_on_test>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 frame_h = 5
frame_l = 5
len(test_videos )
Y_pred = model.predict(X_val)
 Y_pred_classes = np.argmax(Y_pred,axis = 1)
 Y_true = np.argmax(Y_val,axis = 1 )<compute_test_metric>
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )
errors =(Y_pred_classes - Y_true != 0)
 Y_pred_classes_errors = Y_pred_classes[errors]
Y_pred_errors = Y_pred[errors]
Y_true_errors = Y_true[errors]
X_val_errors = X_val[errors]<compute_train_metric>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu
Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)
 true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))
delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors
sorted_dela_errors = np.argsort(delta_pred_true_errors)
 most_important_errors = sorted_dela_errors[-6:]
display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors )<predict_on_test>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
results = model.predict(X_test)
 results = np.argmax(results, axis = 1)
 results = pd.Series(results, name="Label" )<save_to_csv>
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"), results],axis = 1)
 submission.to_csv("submission.csv",index=False )<install_modules>
input_size = 224
!pip3 install --no-dependencies.. /input/efficientnetcassava/Keras_Applications-1.0.8-py3-none-any.whl
!pip3 install --no-dependencies.. /input/efficientnetcassava/efficientnet-1.1.1-py3-none-any.whl<import_modules>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
import re
import numpy as np
import pandas as pd
import os
import json<import_modules>
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras import models
from tensorflow.keras import layers
from tensorflow.keras import losses
from sklearn.model_selection import train_test_split
from efficientnet.keras import EfficientNetB3 as EfficientNet<define_variables>
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint
AUTO = tf.data.experimental.AUTOTUNE
SIZE = 600
ORIGINAL_WIDTH = 800
ORIGINAL_HEIGHT = 600
CHANNELS = 3
BATCH_SIZE = 32<normalization>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5
def decode_image(path):
image = tf.io.read_file(path)
 image = tf.image.decode_jpeg(image, channels=3)
 image =(tf.cast(image, tf.float32)/ 255.0)
 image = tf.image.resize(image, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH])
 image = tf.reshape(image, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH , CHANNELS])
 return image
def normalize(x):
x = tf.image.resize(x, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH])
 x = tf.reshape(x, [ORIGINAL_HEIGHT, ORIGINAL_WIDTH, CHANNELS])
 return x<load_from_csv>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
load_dir = "/kaggle/input/cassava-leaf-disease-classification"
sub_df = pd.read_csv(load_dir + '/sample_submission.csv')
 sub_df['paths'] = load_dir + "/test_images/" + sub_df.image_id<categorify>
speed_test = False
def load_dataset(augment=False):
test_dataset =tf.data.Dataset.from_tensor_slices(sub_df.paths.values ).map(decode_image, num_parallel_calls=AUTO)
 if augment:
test_dataset = test_dataset.map(lambda x: data_aug(x), num_parallel_calls=AUTO)
 else:
test_dataset = test_dataset.map(lambda x:normalize(x))
return test_dataset.batch(BATCH_SIZE ).prefetch(AUTO )<choose_model_class>
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )
def load_model(i):
inputs = layers.Input(shape=(ORIGINAL_HEIGHT, ORIGINAL_WIDTH, 3))
model = Sequential([
EfficientNet(include_top=False,weights=None, input_tensor=inputs),
layers.GlobalAveragePooling2D(name="avg_pool"),
layers.BatchNormalization() ,
layers.Dropout(0.3, name="top_dropout"),
layers.Dense(5, activation="softmax", name="pred")
 ])
 model.load_weights(F".. /input/efficientnetcassava/EfficientNetB3_tl_best_weights_{i}.h5")
 model.compile(loss=losses.SparseCategoricalCrossentropy() , optimizer=tf.optimizers.Adam(lr=0.001), metrics=['accuracy'])
 return model<load_pretrained>
predictions = predict_on_video_set(test_videos, num_workers=4 )
n_models = 5
models = []
for i in range(n_models):
models.append(load_model(i))<predict_on_test>
submission_df_resnext = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_resnext.to_csv("submission_resnext.csv", index=False )
preds = []
test_dataset = load_dataset()
for i in range(n_models):
preds.append(models[i].predict(test_dataset, verbose=1))
for i in range(10):
test_dataset_augmented = load_dataset(augment=True)
 for i in range(n_models):
preds.append(models[i].predict(test_dataset_augmented, verbose=1))
preds = np.mean(preds, axis=0)
 preds<save_to_csv>
!pip install.. /input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet
sub_df['label'] = preds.argmax(axis=1)
 sub_df.drop(columns='paths' ).to_csv('submission.csv', index=False)
 !head submission.csv<set_options>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )
warnings.filterwarnings('ignore')
<define_variables>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu
training_folder = '.. /input/cassava-leaf-disease-classification/train_images/'<load_from_csv>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
samples_df = pd.read_csv(".. /input/cassava-leaf-disease-classification/train.csv")
 samples_df = shuffle(samples_df, random_state=42)
 samples_df["filepath"] = training_folder+samples_df["image_id"]
samples_df.head() 
<define_variables>
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
training_percentage = 0.8
training_item_count = int(len(samples_df)*training_percentage)
 validation_item_count = len(samples_df)-int(len(samples_df)*training_percentage)
 training_df = samples_df[:training_item_count]
validation_df = samples_df[training_item_count:] 
<define_variables>
input_size = 150
batch_size = 8
image_size = 512
input_shape =(image_size, image_size, 3)
 dropout_rate = 0.4
classes_to_predict = sorted(training_df.label.unique() )<prepare_x_and_y>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
training_data = tf.data.Dataset.from_tensor_slices(( training_df.filepath.values, training_df.label.values))
validation_data = tf.data.Dataset.from_tensor_slices(( validation_df.filepath.values, validation_df.label.values))<categorify>
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-kernel-data/model_50epochs_lr0001_patience5_factor01_batchsize32.pth'))
net.append(model )
def load_image_and_label_from_path(image_path, label):
img = tf.io.read_file(image_path)
 img = tf.image.decode_jpeg(img, channels=3)
 return img, label
AUTOTUNE = tf.data.experimental.AUTOTUNE
training_data = training_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)
 validation_data = validation_data.map(load_image_and_label_from_path, num_parallel_calls=AUTOTUNE)
 training_data_batches = training_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE)
 validation_data_batches = validation_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE )<normalization>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5
adapt_data = tf.data.Dataset.from_tensor_slices(training_df.filepath.values)
 def adapt_mode(image_path):
img = tf.io.read_file(image_path)
 img = tf.image.decode_jpeg(img, channels=3)
 img = layers.experimental.preprocessing.Rescaling(1.0 / 255 )(img)
 return img
adapt_data = adapt_data.map(adapt_mode, num_parallel_calls=AUTOTUNE)
 adapt_data_batches = adapt_data.shuffle(buffer_size=1000 ).batch(batch_size ).prefetch(buffer_size=AUTOTUNE )<set_options>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
data_augmentation_layers = tf.keras.Sequential(
 [
layers.experimental.preprocessing.RandomCrop(height=image_size, width=image_size),
layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical"),
layers.experimental.preprocessing.RandomRotation(0.25),
layers.experimental.preprocessing.RandomZoom(( -0.2, 0)) ,
layers.experimental.preprocessing.RandomContrast(( 0.2,0.2))
]
)<choose_model_class>
speed_test = False
efficientnet = EfficientNetB3(weights=".. /input/keras-efficientnetb3-no-top-weights/efficientnetb3_notop.h5",
include_top=False,
input_shape=input_shape,
drop_connect_rate=dropout_rate)
 inputs = Input(shape=input_shape)
 augmented = data_augmentation_layers(inputs)
 efficientnet = efficientnet(augmented)
 pooling = layers.GlobalAveragePooling2D()(efficientnet)
 dropout = layers.Dropout(dropout_rate )(pooling)
 outputs = Dense(len(classes_to_predict), activation="softmax" )(dropout)
 model = Model(inputs=inputs, outputs=outputs)
 model.summary()<define_variables>
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )
epochs = 20<choose_model_class>
%%time
model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )
decay_steps = int(round(len(training_df)/batch_size)) *epochs
cosine_decay = CosineDecay(initial_learning_rate=1e-4, decay_steps=decay_steps, alpha=0.3)
 callbacks = [ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True)]
model.compile(loss="sparse_categorical_crossentropy", optimizer=tf.keras.optimizers.Adam(cosine_decay), metrics=["accuracy"] )<train_model>
submission_df_xception = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_xception.to_csv("submission_xception.csv", index=False )
history = model.fit(training_data_batches,
epochs = epochs,
validation_data=validation_data_batches,
callbacks=callbacks )<load_pretrained>
submission_df = pd.DataFrame({"filename": test_videos})
 submission_df["label"] = 0.5*submission_df_resnext["label"] + 0.5*submission_df_xception["label"]
model.load_weights("./best_model.h5" )<save_to_csv>
submission_df.to_csv("submission.csv", index=False )