Kaggle competition: Deepfake Detection Challenge
train_null_tidy = train_null.add_prefix('Train_');
test_null_tidy = test_null.add_prefix('Test_');
total_null_tidy = pd.concat([train_null_tidy, test_null_tidy], axis=1 ).fillna(0)
 total_null_tidy<train_model>
%matplotlib inline
print("Train features with Na = " + str(( train.isnull().sum() != 0 ).sum()))
print("Test features with Na = " + str(( test.isnull().sum() != 0 ).sum()))<data_type_conversions>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )
def handle_missing(data_sets):
for data_set in data_sets:
data_set['MSSubClass'] = data_set ['MSSubClass'].apply(str)
 data_set['YrSold'] = data_set['YrSold'].astype(str)
 data_set['MoSold'] = data_set['MoSold'].astype(str)
 for col in('PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu', 'MasVnrType', 'Utilities'):
data_set[col] = data_set[col].fillna('None')
 data_set['MasVnrArea'] = data_set['MasVnrArea'].fillna(0)
 for col in('Exterior1st', 'Exterior2nd', 'SaleType'):
data_set[col] = data_set[col].fillna(data_set[col].mode() [0])
 data_set['MSZoning'] = data_set.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
data_set['LotFrontage'] = data_set.groupby("Neighborhood")['LotFrontage'].transform(lambda x: x.fillna(x.median()))
data_set['Electrical'] = data_set['Electrical'].fillna("SBrkr")
 data_set['Functional'] = data_set['Functional'].fillna('Typ')
 data_set['KitchenQual'] = data_set['KitchenQual'].fillna('TA')
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
data_set[col] = data_set[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data_set[col] = data_set[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data_set[col] = data_set[col].fillna('None')
 for col in('BsmtFullBath', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtHalfBath'):
data_set[col] = data_set[col].fillna(0 )<train_model>
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )
print("Train features with Na = " + str(( train.isnull().sum() != 0 ).sum()))
print("Test features with Na = " + str(( test.isnull().sum() != 0 ).sum()))<feature_engineering>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu
train['SalePrice_norm'] = np.log1p(train['SalePrice'] )<prepare_x_and_y>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/helper/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/helper/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
train_Y = train['SalePrice_norm']
train.drop(['SalePrice', 'SalePrice_norm'], axis = 1, inplace = True)
 all_data = pd.concat([train, test], axis=0, ignore_index = True)
<sort_values>
frames_per_video = 17
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_data.columns:
if all_data[i].dtype in numeric_dtypes:
numeric.append(i)
 skew_features = all_data[numeric].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[abs(skew_features)> 0.5]
skew_index = high_skew.index
print("There are {} numerical features with Skew > 0.5 :".format(high_skew.shape[0]))
skewness = pd.DataFrame({'Skew' :high_skew})
 high_skew<compute_test_metric>
input_size = 224
for i in skew_index:
all_data[i] = boxcox1p(all_data[i], boxcox_normmax(all_data[i] + 1))<feature_engineering>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<create_dataframe>
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 23, 3],
groups=32,
width_per_group=8)
 self.fc = nn.Linear(2048, 1)
new_data = all_data.copy()<categorify>
checkpoint = torch.load("/kaggle/input/models-update/checkpoint_t.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint
ordinal_label=['LandSlope','YearBuilt','YearRemodAdd',
'CentralAir','GarageYrBlt','PavedDrive',
'YrSold']
for field in ordinal_label:
le = LabelEncoder()
new_data[field] = le.fit_transform(new_data[field].values )<define_variables>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5
ordinal_features = [
'MSSubClass','ExterQual','LotShape','BsmtQual','BsmtCond',
'BsmtExposure','BsmtFinType1', 'BsmtFinType2','HeatingQC',
'Functional', 'FireplaceQu','KitchenQual', 'GarageFinish',
'GarageQual','GarageCond', 'PoolQC', 'Fence'
]
orders=[
['20','30','40','45','50','60','70','75','80','85', '90','120','150','160','180','190'],
['Po','Fa','TA','Gd','Ex'],
['Reg','IR1' ,'IR2','IR3'],
['None','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','No','Mn','Av','Gd'],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['None','Unf','LwQ', 'Rec','BLQ','ALQ' , 'GLQ' ],
['Po','Fa','TA','Gd','Ex'],
['Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],
['None','Po','Fa','TA','Gd','Ex'],
['Fa','TA','Gd','Ex'],
['None','Unf','RFn','Fin'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Po','Fa','TA','Gd','Ex'],
['None','Fa','Gd','Ex'],
['None','MnWw','GdWo','MnPrv','GdPrv'] ]
for field in ordinal_features:
new_data[field] = new_data[field].astype(str)
 for i in range(len(orders)) :
ord_en = OrdinalEncoder(categories = [orders[i]])
 new_data[ordinal_features[i]] = pd.DataFrame(ord_en.fit_transform(new_data[[ordinal_features[i]]]))<normalization>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
scale = StandardScaler()
new_data.drop("Id", axis = 1, inplace = True)
 numerical_cols = new_data._get_numeric_data().columns
new_data[numerical_cols]= scale.fit_transform(new_data[numerical_cols] )<define_variables>
speed_test = False
categorical_cols = list(set(new_data.columns)- set(numerical_cols))
categorical_cols<categorify>
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )
new_data=pd.get_dummies(new_data )<import_modules>
predictions = predict_on_video_set(test_videos, num_workers=4 )
<compute_train_metric><EOS> 
submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df.to_csv("submission.csv", index=False )