Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<predict_on_test>
start_time = time.time()
%matplotlib inline
preds = model.predict(test_sequences )<define_variables>
!pip install tensorflow-text==2.0.0 --user
preds =(preds>0.5 )<define_variables>
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text as textb
preds = [int(p)for p in preds]<save_to_csv>
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 100 )
submission2 = pd.DataFrame({'id': test.id.values, 'sentiment': preds})
 submission2.to_csv('submission2.csv', index=False )<define_variables>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 length_train = len(train.index)
 length_train
FileLink('./submission2.csv' )<define_variables>
df_mislabeled = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
index_misl = df_mislabeled.index.tolist()
lenght = len(index_misl)
 print(f"There are {lenght} equivalence classes with mislabelling" )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master' 
<import_modules>
train_nu_target = train[train['text'].isin(index_misl)].sort_values(by = 'text')
 train_nu_target.head(60 )
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import math
from torch.nn import Parameter
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import torch.nn.functional as F
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
num_records = train_nu_target.shape[0]
length = len(index_misl)
 print(f"There are {num_records} records in train set which generate {lenght} equivalence classes with mislabelling(raw text, no cleaning)" )
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'tf_efficientnet_b4_ns',
'img_size': 512,
'use_benchmark': False,
'use_deterministic': False,
'epochs': 10,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [6,8],
'weights': [1,1],
'classifier_type': 'linear',
'cosine_scale': 15,
'cosine_margin': 0.4
}<load_from_csv>
copy = train_nu_target.copy()
classes = copy.groupby('text' ).agg({'keyword':np.size, 'target':np.mean} ).rename(columns={'keyword':'Number of records in train set', 'target':'Target mean'})
 classes.sort_values('Number of records in train set', ascending=False ).head(20 )
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
majority_df = train_nu_target.groupby(['text'])['target'].mean()
train.label.value_counts()<load_from_csv>
def relabel(r, majority_index):
ind = ''
if r['text'] in majority_index:
ind = r['text']
if majority_df[ind] <= 0.5:
return 0
else:
return 1
else:
return r['target']
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<set_options>
train['target'] = train.apply(lambda row: relabel(row, majority_df.index), axis = 1 )
if CFG['use_benchmark']:
torch.backends.cudnn.benchmark = True
if CFG['use_deterministic']:
torch.backends.cudnn.deterministic = True<categorify>
new_df = train[train['text'].isin(majority_df.index)].sort_values(['target', 'text'], ascending = [False, True])
 new_df.head(15 )
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<import_modules>
df_mislabeled = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
index_misl = df_mislabeled.index.tolist()
len(index_misl )
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<normalization>
use = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3" )
class ContrastiveLoss(nn.Module):
def __init__(self, margin=0.3, **kwargs):
super(ContrastiveLoss, self ).__init__()
self.margin = margin
def forward(self, inputs, targets):
n = inputs.size(0)
 sim_mat = torch.matmul(inputs, inputs.t())
 targets = targets
loss = list()
c = 0
for i in range(n):
pos_pair_ = torch.masked_select(sim_mat[i], targets == targets[i])
 pos_pair_ = torch.masked_select(pos_pair_, pos_pair_ < 1)
 neg_pair_ = torch.masked_select(sim_mat[i], targets != targets[i])
 pos_pair_ = torch.sort(pos_pair_)[0]
neg_pair_ = torch.sort(neg_pair_)[0]
neg_pair = torch.masked_select(neg_pair_, neg_pair_ > self.margin)
 neg_loss = 0
pos_loss = torch.sum(-pos_pair_ + 1)
 if len(neg_pair)> 0:
neg_loss = torch.sum(neg_pair)
 loss.append(pos_loss + neg_loss)
 loss = sum(loss)/ n
return loss
class CircleLoss(nn.Module):
def __init__(self, in_features, out_features, s=256, m=0.25):
super(CircleLoss, self ).__init__()
self.weight = Parameter(torch.Tensor(out_features, in_features))
self._s = s
self._m = m
self.reset_parameters()
def reset_parameters(self):
nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
def __call__(self, bn_feat, targets):
sim_mat = F.linear(F.normalize(bn_feat), F.normalize(self.weight))
alpha_p = F.relu(-sim_mat.detach() + 1 + self._m)
 alpha_n = F.relu(sim_mat.detach() + self._m)
 delta_p = 1 - self._m
delta_n = self._m
s_p = self._s * alpha_p *(sim_mat - delta_p)
 s_n = self._s * alpha_n *(sim_mat - delta_n)
 one_hot = torch.zeros(sim_mat.size() , device=targets.device)
 one_hot.scatter_(1, targets.view(-1, 1 ).long() , 1)
 pred_class_logits = one_hot * s_p +(1.0 - one_hot)* s_n
return pred_class_logits
class Arcface(nn.Module):
r
def __init__(self, in_features, out_features, s=30.0, m=0.30, easy_margin=False, ls_eps=0.0):
super(Arcface, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.s = s
self.m = m
self.ls_eps = ls_eps
self.weight = Parameter(torch.FloatTensor(out_features, in_features))
nn.init.xavier_uniform_(self.weight)
 self.easy_margin = easy_margin
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.th = math.cos(math.pi - m)
 self.mm = math.sin(math.pi - m)* m
def forward(self, input, label):
cosine = F.linear(F.normalize(input), F.normalize(self.weight))
if not self.training:
return cosine
sine = torch.sqrt(1.0 - torch.pow(cosine, 2))
phi = cosine * self.cos_m - sine * self.sin_m
phi = phi.type_as(cosine)
 if self.easy_margin:
phi = torch.where(cosine > 0, phi, cosine)
 else:
phi = torch.where(cosine > self.th, phi, cosine - self.mm)
 one_hot = torch.zeros(cosine.size() , device='cuda')
 one_hot.scatter_(1, label.view(-1, 1 ).long() , 1)
 if self.ls_eps > 0:
one_hot =(1 - self.ls_eps)* one_hot + self.ls_eps / self.out_features
output =(one_hot * phi)+(( 1.0 - one_hot)* cosine)
 output *= self.s
return output
class Cosface(nn.Module):
r
def __init__(self, in_features, out_features, s=30.0, m=0.30):
super(Cosface, self ).__init__()
self.in_features = in_features
self.out_features = out_features
self.s = s
self.m = m
self.weight = Parameter(torch.FloatTensor(out_features, in_features))
nn.init.xavier_uniform_(self.weight)
 def forward(self, input, label):
cosine = F.linear(F.normalize(input), F.normalize(self.weight))
phi = cosine - self.m
one_hot = torch.zeros(cosine.size() , device='cuda')
 one_hot.scatter_(1, label.view(-1, 1 ).long() , 1)
 output =(one_hot * phi)+(( 1.0 - one_hot)* cosine)
 output *= self.s
return output
def __repr__(self):
return self.__class__.__name__ + '(' \
+ 'in_features=' + str(self.in_features)\
+ ', out_features=' + str(self.out_features)\
+ ', s=' + str(self.s)\
+ ', m=' + str(self.m)+ ')'
class AMSoftmax(nn.Module):
def __init__(self, in_features, out_features, s=30.0, m=0.30):
super(AMSoftmax, self ).__init__()
self.m = m
self.s = s
self.in_feats = in_features
self.W = torch.nn.Parameter(torch.randn(in_features, out_features), requires_grad=True)
 self.ce = nn.CrossEntropyLoss()
nn.init.xavier_normal_(self.W, gain=1)
 def forward(self, x, lb):
assert x.size() [0] == lb.size() [0]
assert x.size() [1] == self.in_feats
x_norm = torch.norm(x, p=2, dim=1, keepdim=True ).clamp(min=1e-12)
 x_norm = torch.div(x, x_norm)
 w_norm = torch.norm(self.W, p=2, dim=0, keepdim=True ).clamp(min=1e-12)
 w_norm = torch.div(self.W, w_norm)
 costh = torch.mm(x_norm, w_norm)
 lb_view = lb.view(-1, 1)
 delt_costh = torch.zeros(costh.size() , device='cuda' ).scatter_(1, lb_view, self.m)
 costh_m = costh - delt_costh
costh_m_s = self.s * costh_m
return costh_m_s<choose_model_class>
X_train = []
for r in tqdm(train.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_train.append(review_emb)
 X_train = np.array(X_train)
 y_train = train.target.values
X_test = []
for r in tqdm(test.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_test.append(review_emb)
 X_test = np.array(X_test )
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
backbone = timm.create_model(model_arch, pretrained=pretrained)
 n_features = backbone.classifier.in_features
self.backbone = nn.Sequential(*backbone.children())[:-1]

 if CFG['classifier_type'] == "arcface":
print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))
self.classifier = Arcface(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])
 elif CFG['classifier_type'] == "cosface":
print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))
self.classifier = Cosface(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])
 elif CFG['classifier_type'] == "amsoftmax":
print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))
self.classifier = AMSoftmax(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])
 elif CFG['classifier_type'] == "circle":
print('using {} with s:{}, m: {}'.format(CFG['classifier_type'],CFG['cosine_scale'],CFG['cosine_margin']))
self.classifier = CircleLoss(n_features, n_class,s=CFG['cosine_scale'], m=CFG['cosine_margin'])
 else:
self.classifier = nn.Linear(n_features, n_class)

def forward_features(self, x):
x = self.backbone(x)
 return x
def forward(self, x,label=None):
x = self.forward_features(x)
 if CFG['classifier_type'] in('arcface','cosface','amsoftmax','circle'):
x = self.classifier(x,label)
 else:
x = self.classifier(x)
 return x 
<split>
train_arrays, test_arrays, train_labels, test_labels = train_test_split(X_train,
y_train,
random_state =42,
test_size=0.20 )
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/linear-weight/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.mean(tst_preds, axis=0)
<feature_engineering>
def svc_param_selection(X, y, nfolds):
Cs = [1.07]
gammas = [2.075]
param_grid = {'C': Cs, 'gamma' : gammas}
grid_search = GridSearchCV(SVC(kernel='rbf'), param_grid, cv=nfolds, n_jobs=8)
 grid_search.fit(X, y)
 grid_search.best_params_
return grid_search
model = svc_param_selection(train_arrays,train_labels, 5 )
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
model.best_params_
test.to_csv('submission.csv', index=False )<import_modules>
pred = model.predict(test_arrays )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf<load_from_csv>
cm = confusion_matrix(test_labels,pred)
 cm
mnist = pd.read_csv("/kaggle/input/digit-recognizer/train.csv" )<prepare_x_and_y>
accuracy = accuracy_score(test_labels,pred)
 accuracy
y = mnist.iloc[:, 0].values
X = mnist.iloc[:, 1:].values<split>
test_pred = model.predict(X_test)
 submission['target'] = test_pred.round().astype(int)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1
, random_state = 0 )<data_type_conversions>
count = 2
prob_disaster = 0.9
keyword_list_disaster = list(ag[(ag['Count']>count)&(ag['Disaster Probability']>=prob_disaster)].index)
 keyword_list_disaster
X_train = X_train.reshape(-1, 28, 28, 1 ).astype('float32')/ 255.0
X_test = X_test.reshape(-1, 28, 28, 1 ).astype('float32')/ 255.0<categorify>
ids_disaster = test['id'][test.keyword.isin(keyword_list_disaster)].values
submission['target'][submission['id'].isin(ids_disaster)] = 1
y_train = tf.keras.utils.to_categorical(y_train)
 y_test = tf.keras.utils.to_categorical(y_test )<choose_model_class>
submission.to_csv("submission.csv", index=False)
 submission.head(10 )