Kaggle competition: Deepfake Detection Challenge
<SOS> metric: LogLoss Kaggle data source: deepfake-detection-challenge<create_dataframe>
%matplotlib inline
ft2 = pd.DataFrame({"Actual" : y_train,"predict" : lgb_pred} )<create_dataframe>
frames_per_vid = [17, 25, 30, 32, 35, 36, 40]
public_LB = [0.46788, 0.46776, 0.46611, 0.46542, 0.46643, 0.46484, 0.46635]
df_viz = pd.DataFrame({'frames_per_vid': frames_per_vid, 'public_LB':public_LB} )
submission = pd.DataFrame({"Id" : test_id, "SalePrice" : lgb_pred} )<save_to_csv>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 frame_h = 5
frame_l = 5
len(test_videos )
submission.to_csv("sub.csv",index = False )<import_modules>
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )
import numpy as np
import pandas as pd<load_from_csv>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu
train= pd.read_csv(".. /input/ames-housing-dataset/AmesHousing.csv")
 train.shape<drop_column>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
train.drop(['PID'], axis=1, inplace=True )<load_from_csv>
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
df_test=pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )<count_missing_values>
input_size = 224
df_test.isna().sum()<load_from_csv>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )<load_from_csv>
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )
df_train=pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv" )<drop_column>
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint
missing = df_test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 df_test.dropna(axis=1, inplace=True)
 df_test.drop(['Electrical'], axis=1, inplace=True )<count_duplicates>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.5
test = tqdm(range(0, len(df_test)) , desc='Matching')
 for i in test:
for j in range(0, len(train)) :
for k in range(1, len(df_test.columns)) :
if df_test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
test.close()<save_to_csv>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
submission.to_csv('enes_results.csv', index=False )<import_modules>
speed_test = False
!pip install contractions
nltk.download("stopwords")
 nltk.download('punkt')
 nltk.download('wordnet')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train = pd.read_csv('/kaggle/input/jigsaw-toxic-comment-classification-challenge/train.csv.zip')
 test = pd.read_csv('/kaggle/input/jigsaw-toxic-comment-classification-challenge/test.csv.zip')
 test_label = pd.read_csv('/kaggle/input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip')
 sub=pd.read_csv('/kaggle/input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip' )<concatenate>
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )
class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
train_text = train['comment_text']
test_text = test['comment_text']
all_text = pd.concat([train_text, test_text] )<string_transform>
predictions = predict_on_video_set(test_videos, num_workers=4 )
<feature_engineering><EOS> 
submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df.to_csv("submission.csv", index=False )