Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_variables>
%matplotlib inline
input_size = 150<normalization>
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
train_df['char'] = train_df['text'].str.len()
train_df['words'] = train_df['text'].str.split().map(lambda x: len(x))
train_df.head(3 )
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-xception-trained-model/model.pth'))
net.append(model )<predict_on_test>
print('There are {} tweets in total and {} tweets with keyword'.format(train_df['keyword'].shape[0], train_df['keyword'].notna().sum()))
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481<predict_on_test>
train_df['text'] = train_df['text'].str.replace("http\S+", " ")
 test_df['text'] = test_df['text'].str.replace("http\S+", " " )
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<predict_on_test>
train_df = train_df[['text', 'target']]
tweets = train_df.groupby('text' ).mean().reset_index()
print('There are {} tweets with different label in duplicates.'.format(tweets[(tweets['target']!=1)&(tweets['target']!=0)].shape[0]))
df_diff = tweets[(tweets['target']!=1)&(tweets['target']!=0)].reset_index(drop=True)
 tweets = tweets[(tweets['target']==1)|(tweets['target']==0)].reset_index()
%%time
model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
for i in range(64):
print(train_df[train_df['text']==df_diff.loc[i]['text']] )
submission_df_xception = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_xception.to_csv("submission_xception.csv", index=False )<create_dataframe>
nlp = spacy.load('en_core_web_lg' )
submission_df = pd.DataFrame({"filename": test_videos} )<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(vectors, tweets.target, test_size=0.2,
random_state=52, stratify = tweets.target)
 model = LinearSVC(random_state=1, dual=False)
 model.fit(X_train, y_train)
 print(f'Model test accuracy: {model.score(X_valid, y_valid)*100:.3f}%' )
submission_df["label"] = 0.70*submission_df_resnext["label"] + 0.30*submission_df_xception["label"]<save_to_csv>
second_model = LogisticRegression(solver='saga')
 second_model.fit(X_train, y_train)
 print(f'Model test accuracy: {second_model.score(X_valid, y_valid)*100:.3f}%' )
submission_df.to_csv("submission.csv", index=False )<define_variables>
third_model = linear_model.RidgeClassifier()
third_model.fit(X_train, y_train)
 print(f'Model test accuracy: {third_model.score(X_valid, y_valid)*100:.3f}%' )
TEST_DIR = "/kaggle/input/deepfake-detection-challenge/test_videos/"
CHECKPOINT = '/kaggle/input/kha-deepfake-dataset/checkpoint_mobilev3_alldata_1903_withfaceforensics_3epochs_.pth'
CHECKPOINT2 = '/kaggle/input/kha-deepfake-dataset/cpt_mbn_sqrimg_2503)2epochs_.pth'
CHECKPOINT3 = '/kaggle/input/kha-deepfake-dataset/checkpoint_b0_1803_0epochs_0.4498354019969702.pth'
CHECKPOINT4 = '/kaggle/input/kha-deepfake-dataset/cpt_xception_new_sqrimg_29030_epochs_3_moment.pth'
CHECKPOINT5 = '/kaggle/input/kha-deepfake-dataset/cpt_nasnet_bbd_29032_epochs_0_moment.pth'
CHECKPOINT6 = '/kaggle/input/kha-deepfake-dataset/cpt_effb1_sqrimg_25032_epochs_1_moment.pth'
CHECKPOINT7 = '/kaggle/input/kha-deepfake-dataset/cpt_mbn_LSTM16_300313epochs_0.21526583118569945.pth'
CHECKPOINT8 = '/kaggle/input/kha-deepfake-dataset/checkpoint_b0_LSTM16_2503_batchfirst1epochs_0.25805166250713446.pth'
CHECKPOINT9 = '/kaggle/input/kha-deepfake-dataset/cpt_nasnetnew_bbd_31031_epochs_0_moment.pth'
CHECKPOINT10 = '/kaggle/input/kha-deepfake-dataset/cpt_xception_sqrimg_25031_epochs_0_moment.pth'
CHECKPOINT11 = '/kaggle/input/kha-deepfake-dataset/cpt_resnet_bbd_31033_epochs_3_moment.pth'
CONVERT_RGB = True
MTCNN_BATCH = 15
GAP = 5
IMG_SIZE = 224
SCALE = 0.5
NUM_WORKERS = mp.cpu_count()<define_variables>
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 params = {
'task' : 'train',
'boosting_type' : 'gbdt',
'objective' : 'binary',
'metric' : {'binary_logloss'},
'num_leaves' : 51,
'learning_rate' : 0.01,
'max_bin': 397,
'feature_fraction' : 0.9,
'bagging_fraction' : 0.8,
'bagging_freq': 5,
'verbose' : 0
}
gbm = lgb.train(params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 y_pred = gbm.predict(X_valid, num_iteration=gbm.best_iteration)
 print(gbm.best_score['valid_1']['binary_logloss'])
 fourth = pd.DataFrame([np.round(y_pred,0), y_valid], index=['y_pred', 'y_valid'] ).T
accuracy = fourth[fourth['y_pred']==fourth['y_valid']].shape[0]/fourth.shape[0]
print(f'Model test accuracy: {accuracy*100:.3f}%' )
package_path = '.. /input/kha-efficientnet/EfficientNet-PyTorch/'
sys.path.append(package_path)
<install_modules>
print('SVM f1 score {}'.format(f1_score(y_valid, np.round(model.predict(X_valid), 0 ).astype(int))))
print('Logistic Regression f1 score {}'.format(f1_score(y_valid, np.round(second_model.predict(X_valid), 0 ).astype(int))))
print('Ridge Classifier f1 score {}'.format(f1_score(y_valid, np.round(third_model.predict(X_valid), 0 ).astype(int))))
print('Light GBM f1 score {}'.format(f1_score(y_valid, np.round(gbm.predict(X_valid, num_iteration=gbm.best_iteration), 0 ).astype(int))))
%%capture
!pip install /kaggle/input/khafacenet/facenet_pytorch-2.2.7-py3-none-any.whl
!pip install /kaggle/input/imutils/imutils-0.5.3<install_modules>
X_train = pd.DataFrame(vectors)
 y_train = tweets['target']
X_test = pd.DataFrame(vectors_pred)
 y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=100892)
 params = {
'task' : 'train',
'boosting_type' : 'gbdt',
'objective' : 'binary',
'metric' : {'binary_logloss'},
'num_leaves' : 51,
'max_bin' : 397,
'learning_rate' : 0.01,
'feature_fraction' : 0.9,
'bagging_fraction' : 0.8,
'bagging_freq': 5,
'verbose' : 0
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.iloc[train_index]
X_val = X_train.iloc[valid_index]
y_tr = y_train.iloc[train_index]
y_val = y_train.iloc[valid_index]
lgb_train = lgb.Dataset(X_tr, y_tr)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 model = lgb.train(params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val, num_iteration=model.best_iteration)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model)
 scores = [
m.best_score['valid_1']['binary_logloss'] for m in models
]
score = sum(scores)/ len(scores )
<load_from_zip>
print('===CV scores===')
 print(scores)
 print(score )
<set_options>
pred = pd.DataFrame(np.round(y_preds, 0)).astype('int8' ).T
%matplotlib inline
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
<categorify>
np.round(pred.mean(axis=1),0 ).astype(int )
def conv_bn(inp, oup, stride, conv_layer=nn.Conv2d, norm_layer=nn.BatchNorm2d, nlin_layer=nn.ReLU):
return nn.Sequential(
 conv_layer(inp, oup, 3, stride, 1, bias=False),
norm_layer(oup),
nlin_layer(inplace=True)
)
 def conv_1x1_bn(inp, oup, conv_layer=nn.Conv2d, norm_layer=nn.BatchNorm2d, nlin_layer=nn.ReLU):
return nn.Sequential(
 conv_layer(inp, oup, 1, 1, 0, bias=False),
norm_layer(oup),
nlin_layer(inplace=True)
)
 class Hswish(nn.Module):
def __init__(self, inplace=True):
super(Hswish, self ).__init__()
self.inplace = inplace
def forward(self, x):
return x * F.relu6(x + 3., inplace=self.inplace)/ 6.
 class Hsigmoid(nn.Module):
def __init__(self, inplace=True):
super(Hsigmoid, self ).__init__()
self.inplace = inplace
def forward(self, x):
return F.relu6(x + 3., inplace=self.inplace)/ 6.
 class SEModule(nn.Module):
def __init__(self, channel, reduction=4):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Sequential(
 nn.Linear(channel, channel // reduction, bias=False),
nn.ReLU(inplace=True),
nn.Linear(channel // reduction, channel, bias=False),
Hsigmoid() 
)
 def forward(self, x):
b, c, _, _ = x.size()
y = self.avg_pool(x ).view(b, c)
 y = self.fc(y ).view(b, c, 1, 1)
 return x * y.expand_as(x)
 class Identity(nn.Module):
def __init__(self, channel):
super(Identity, self ).__init__()
def forward(self, x):
return x
def make_divisible(x, divisible_by=8):
return int(np.ceil(x * 1./ divisible_by)* divisible_by)
 class MobileBottleneck(nn.Module):
def __init__(self, inp, oup, kernel, stride, exp, se=False, nl='RE'):
super(MobileBottleneck, self ).__init__()
assert stride in [1, 2]
assert kernel in [3, 5]
padding =(kernel - 1)// 2
self.use_res_connect = stride == 1 and inp == oup
conv_layer = nn.Conv2d
norm_layer = nn.BatchNorm2d
if nl == 'RE':
nlin_layer = nn.ReLU
elif nl == 'HS':
nlin_layer = Hswish
else:
raise NotImplementedError
if se:
SELayer = SEModule
else:
SELayer = Identity
self.conv = nn.Sequential(
 conv_layer(inp, exp, 1, 1, 0, bias=False),
norm_layer(exp),
nlin_layer(inplace=True),
conv_layer(exp, exp, kernel, stride, padding, groups=exp, bias=False),
norm_layer(exp),
SELayer(exp),
nlin_layer(inplace=True),
conv_layer(exp, oup, 1, 1, 0, bias=False),
norm_layer(oup), 
)
 def forward(self, x):
if self.use_res_connect:
return x + self.conv(x)
 else:
return self.conv(x)
 class MobileNetV3(nn.Module):
def __init__(self, n_class=1000, input_size=224, dropout=0.8, mode='small', width_mult=1.0):
super(MobileNetV3, self ).__init__()
input_channel = 16
last_channel = 1280
if mode == 'large':
mobile_setting = [
[3, 16, 16, False, 'RE', 1],
[3, 64, 24, False, 'RE', 2],
[3, 72, 24, False, 'RE', 1],
[5, 72, 40, True, 'RE', 2],
[5, 120, 40, True, 'RE', 1],
[5, 120, 40, True, 'RE', 1],
[3, 240, 80, False, 'HS', 2],
[3, 200, 80, False, 'HS', 1],
[3, 184, 80, False, 'HS', 1],
[3, 184, 80, False, 'HS', 1],
[3, 480, 112, True, 'HS', 1],
[3, 672, 112, True, 'HS', 1],
[5, 672, 160, True, 'HS', 2],
[5, 960, 160, True, 'HS', 1],
[5, 960, 160, True, 'HS', 1],
]
elif mode == 'small':
mobile_setting = [
[3, 16, 16, True, 'RE', 2],
[3, 72, 24, False, 'RE', 2],
[3, 88, 24, False, 'RE', 1],
[5, 96, 40, True, 'HS', 2],
[5, 240, 40, True, 'HS', 1],
[5, 240, 40, True, 'HS', 1],
[5, 120, 48, True, 'HS', 1],
[5, 144, 48, True, 'HS', 1],
[5, 288, 96, True, 'HS', 2],
[5, 576, 96, True, 'HS', 1],
[5, 576, 96, True, 'HS', 1],
]
else:
raise NotImplementedError
assert input_size % 32 == 0
last_channel = make_divisible(last_channel * width_mult)if width_mult > 1.0 else last_channel
self.features = [conv_bn(3, input_channel, 2, nlin_layer=Hswish)]
self.classifier = []
for k, exp, c, se, nl, s in mobile_setting:
output_channel = make_divisible(c * width_mult)
 exp_channel = make_divisible(exp * width_mult)
 self.features.append(MobileBottleneck(input_channel, output_channel, k, s, exp_channel, se, nl))
input_channel = output_channel
if mode == 'large':
last_conv = make_divisible(960 * width_mult)
 self.features.append(conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish))
self.features.append(nn.AdaptiveAvgPool2d(1))
self.features.append(nn.Conv2d(last_conv, last_channel, 1, 1, 0))
self.features.append(Hswish(inplace=True))
elif mode == 'small':
last_conv = make_divisible(576 * width_mult)
 self.features.append(conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish))
self.features.append(nn.AdaptiveAvgPool2d(1))
self.features.append(nn.Conv2d(last_conv, last_channel, 1, 1, 0))
self.features.append(Hswish(inplace=True))
else:
raise NotImplementedError
self.features = nn.Sequential(*self.features)
 self.classifier = nn.Sequential(
 nn.Dropout(p=dropout),
nn.Linear(last_channel, n_class), 
)
 self._initialize_weights()
def forward(self, x):
x = self.features(x)
 x = x.mean(3 ).mean(2)
 x = self.classifier(x)
 return x
def _initialize_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out')
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.ones_(m.weight)
 nn.init.zeros_(m.bias)
 elif isinstance(m, nn.Linear):
nn.init.normal_(m.weight, 0, 0.01)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 def mobilenetv3(pretrained=False, **kwargs):
model = MobileNetV3(**kwargs)
 if pretrained:
state_dict = torch.load(CFG.pretrained)
 model.load_state_dict(state_dict, strict=True)
 return model 
<set_options>
net = mobilenetv3(mode='small', pretrained=False)
 net.classifier[1] = torch.nn.Linear(in_features=1280, out_features=1)
 net = net.to(device)
 state_dict = torch.load(CHECKPOINT)
 net.load_state_dict(state_dict)
 net.cuda()
net.eval()<find_best_params>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission
net2 = mobilenetv3(mode='small', pretrained=False)
 net2.classifier[1] = torch.nn.Linear(in_features=1280, out_features=1)
 net2 = net2.to(device)
 state_dict = torch.load(CHECKPOINT2)
 net2.load_state_dict(state_dict)
 net2.cuda()
net2.eval()<set_options>
sample_submission["target"] = np.round(pred.mean(axis=1),0 ).astype(int)
 sample_submission["target"] = sample_submission["target"].astype('int8' )
<set_options><EOS> 
sample_submission.to_csv("submission.csv", index=False )