Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<split>
import numpy as np
import pandas as pd
import torch
import re
import string
from transformers import BertTokenizer
from transformers import BertModel,BertConfig
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import train_test_split
import time
from matplotlib import pyplot as plt
y = train["label"]
x = train.drop(["label"], axis = 1)
 x_train, x_val, y_train, y_val = sklearn.model_selection.train_test_split(x.values, y.values, test_size = 0.10)
 print(y_train.shape)
 print(x_train.shape)
 print(y_val.shape)
 print(x_val.shape )<data_type_conversions>
training_csv =pd.read_csv(".. /input/nlp-getting-started/train.csv")
 training_csv.head()
print(x_train.shape[0])
 x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
 x_train = x_train.astype('float32')
 x_train /= 255
print(x_val.shape[0])
 x_val = x_val.reshape(x_val.shape[0], img_rows, img_cols, 1)
 x_val = x_val.astype('float32')
 x_val /= 255<normalization>
test_csv = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 test_csv.head()
y_train = tf.keras.utils.to_categorical(y_train, num_class)
 y_val = tf.keras.utils.to_categorical(y_val, num_class)
 k_init = tf.initializers.TruncatedNormal(mean = 0.1, stddev = 0.05)
 b_init = tf.initializers.constant(value = 1e-4)
<create_dataframe>
training_csv['target'].value_counts()
datagen = tf.keras.preprocessing.image.ImageDataGenerator(
 rotation_range=15,
zoom_range = 0.20,
width_shift_range=0.20,
height_shift_range=0.20
)<choose_model_class>
training_csv['keyword'].value_counts()
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(32, kernel_size =(3, 3), activation='relu', input_shape = input_shape),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(32, kernel_size =(3, 3), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(32, kernel_size =(3, 3), activation='relu', strides=2, padding='same'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(64, kernel_size =(3, 3), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(64, kernel_size =(3, 3), activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(64, kernel_size =(3, 3), activation='relu', strides=2, padding='same'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(num_class, activation='softmax')
 ] )<choose_model_class>
training_csv['keyword'].isnull().sum()
optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.002,
rho=0.9,
momentum=0.1,
epsilon=1e-07,
centered=True,
name='RMSprop' )<choose_model_class>
training_csv['location'].value_counts()
lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.25,
patience=2,
verbose=1,
mode="auto",
min_delta=0.0001,
cooldown=0,
min_lr=0.00001)
 es = tf.keras.callbacks.EarlyStopping(monitor='loss',
mode='min', verbose=1,
patience=300,
restore_best_weights=False )<choose_model_class>
training_csv['location'].isnull().sum()
model.compile(loss= "categorical_crossentropy", optimizer = optimizer, metrics=['accuracy'])
<train_model>
def clean(title):

 title = re.sub(r"\-"," ",title)
 title = re.sub(r"\+"," ",title)
 title = re.sub(r"&","and",title)
 title = re.sub(r"\|"," ",title)
 title = re.sub(r"\"," ",title)
 title = re.sub(r"\W"," ",title)
 title = title.lower()
for p in string.punctuation :
title = re.sub(r"f{p}"," ",title)
 title = re.sub(r"\s+"," ",title)
 return title
history = model.fit_generator(datagen.flow(x_train, y_train, batch_size = batch_size),
epochs = epochs,
steps_per_epoch = 100,
validation_data =(x_val, y_val),
validation_steps=50,
callbacks=[lr, es],
verbose=2 )<compute_test_metric>
training_csv["cleaned_text"] = training_csv["text"].map(clean)
 training_csv.head()
scores = model.evaluate(x_val, y_val )<load_from_csv>
tokenizer = BertTokenizer.from_pretrained('.. /input/bert-base-uncased')
test=pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_id=test.id
test=test.drop('id',axis=1)
 test=test/255
test=test.values.reshape(-1,28,28,1 )<predict_on_test>
X_train,X_test,y_train,y_test = train_test_split(training_csv["cleaned_text"].values,training_csv["target"].values, random_state=0,test_size=0.1,shuffle=True )
y_pre=model.predict(test)
 y_pre=np.argmax(y_pre,axis=1 )<save_to_csv>
class CreateDataset(Dataset):
def __init__(self, X, y, tokenizer, max_len):
self.X = X
self.y = y
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.y)
 def __getitem__(self, index):
text = self.X[index]
inputs = self.tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=self.max_len,
truncation=True,
pad_to_max_length=True 
)
 ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.LongTensor(ids),
'mask': torch.LongTensor(mask),
'labels': torch.Tensor(self.y[index])
 }
sample_sub=pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 sample_sub['label']=y_pre
sample_sub.to_csv('submission.csv',index=False)
 sample_sub.head()<set_options>
max_len = 45
dataset_train = CreateDataset(X_train, y_train, tokenizer, max_len)
 dataset_valid = CreateDataset(X_test, y_test, tokenizer, max_len)
%reload_ext autoreload
%autoreload 2
%matplotlib inline
path = '/kaggle/input/Kannada-MNIST/'
print(os.listdir(path))
def random_seed(seed_value, use_cuda):
np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 random.seed(seed_value)
 if use_cuda:
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
random_seed(13,True )<load_from_csv>
class BERTClass(torch.nn.Module):
def __init__(self, drop_rate, otuput_size):
super().__init__()
model_config = BertConfig.from_pretrained('.. /input/bert-base-uncased', output_hidden_states=True)
 self.bert = BertModel.from_pretrained('.. /input/bert-base-uncased', config=model_config)
 self.drop = torch.nn.Dropout(drop_rate)
 self.fc = torch.nn.Linear(768, otuput_size)
 def forward(self, ids, mask):
out = self.bert(ids, attention_mask=mask)[1]
out = self.fc(self.drop(out))
return out
class CustomImageList(ImageList):
def open(self, fn):
if(fn.size == 785):
fn = fn[1:]
img = fn.reshape(28,28)
 img = np.stack(( img,)*3, axis=-1)
 return Image(pil2tensor(img, dtype=np.float32))
@classmethod
def from_csv_custom(cls, path:PathOrStr, csv_name:str, imgIdx:int=1, header:str='infer', **kwargs)->'ItemList':
df = pd.read_csv(Path(path)/csv_name, header=header)
 res = super().from_df(df, path=path, cols=0, **kwargs)
 res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 255.0, axis=1 ).values
return res
@classmethod
def from_csv_custom_test(cls, path:PathOrStr, csv_name:str, imgIdx:int=1, header:str='infer', **kwargs)->'ItemList':
df = pd.read_csv(Path(path)/csv_name, header=header)
 res = super().from_df(df, path=path, cols=0, **kwargs)
 res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 255.0, axis=1 ).values
print(res)
 return res
@classmethod
def from_df_custom(cls, path:PathOrStr, df:DataFrame, imgIdx:int=1, header:str='', **kwargs)->'ItemList':
res = super().from_df(df, path=path, cols=0, **kwargs)
 res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 255.0, axis=1 ).values
return res<load_from_csv>
def calculate_loss_and_accuracy(model, criterion, loader, device):

 model.eval()
loss = 0.0
total = 0
correct = 0
with torch.no_grad() :
for data in loader:
ids = data['ids'].to(device)
 mask = data['mask'].to(device)
 labels = data['labels'].to(device)
 outputs = model(ids, mask)
 loss += criterion(outputs, labels ).item()
pred = torch.round(torch.sigmoid(outputs)).cpu().numpy()
labels = labels.cpu().numpy()
total += len(labels)
 correct +=(pred == labels ).sum().item()
return loss / len(loader), correct / total
def train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, device=None):

 model.to(device)
 dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True)
 dataloader_valid = DataLoader(dataset_valid, batch_size=len(dataset_valid), shuffle=False)
 log_train = []
log_valid = []
for epoch in range(num_epochs):
s_time = time.time()
model.train()
for data in dataloader_train:
ids = data['ids'].to(device)
 mask = data['mask'].to(device)
 labels = data['labels'].to(device)
 optimizer.zero_grad()
outputs = model(ids,mask)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
loss_train, acc_train = calculate_loss_and_accuracy(model, criterion, dataloader_train, device)
 loss_valid, acc_valid = calculate_loss_and_accuracy(model, criterion, dataloader_valid, device)
 log_train.append([loss_train, acc_train])
 log_valid.append([loss_valid, acc_valid])
 e_time = time.time()
print(f'epoch: {epoch + 1}, loss_train: {loss_train:.4f}, accuracy_train: {acc_train:.4f}, loss_valid: {loss_valid:.4f}, accuracy_valid: {acc_valid:.4f}, {(e_time - s_time):.4f}sec')
 return {'train': log_train, 'valid': log_valid}
test = CustomImageList.from_csv_custom_test(path=path, csv_name='test.csv', imgIdx=0 )<load_from_csv>
DROP_RATE = 0.4
OUTPUT_SIZE = 1
BATCH_SIZE = 32
NUM_EPOCHS = 2
LEARNING_RATE = 2e-5
model = BERTClass(DROP_RATE, OUTPUT_SIZE)
 criterion = torch.nn.BCEWithLogitsLoss()
optimizer = torch.optim.AdamW(params=model.parameters() , lr=LEARNING_RATE)
 device = 'cuda' if cuda.is_available() else 'cpu'
log = train_model(dataset_train, dataset_valid, BATCH_SIZE, model, criterion, optimizer, NUM_EPOCHS, device=device )
data =(CustomImageList.from_csv_custom(path=path, csv_name='train.csv', imgIdx=1)
.split_by_rand_pct (.02)
.label_from_df(cols='label')
.add_test(test, label=0)
.transform(get_transforms(do_flip=False,max_rotate=15,max_warp=0.4)) 
.databunch(bs=128, num_workers=0)
.normalize(imagenet_stats)) 
<concatenate>
test_csv["cleaned_text"] = test_csv["text"].map(clean)
 test_csv.head()
def conv2(ni,nf,stride=2,ks=5): return conv_layer(ni,nf,stride=stride,ks=ks)
<compute_test_metric>
class TestDataset(Dataset):
def __init__(self, X, tokenizer, max_len):
self.X = X
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self):
return len(self.X)
 def __getitem__(self, index):
text = self.X[index]
inputs = self.tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=self.max_len,
truncation=True,
pad_to_max_length=True 
)
 ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.LongTensor(ids),
'mask': torch.LongTensor(mask)
 }
def mish(input):
return input * torch.tanh(F.softplus(input))<define_search_model>
max_len = 45
dataset_test = TestDataset(test_csv["cleaned_text"].values, tokenizer, max_len )
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, in_planes, planes, stride=1):
super(BasicBlock, self ).__init__()
self.conv1 = nn.Conv2d(
 in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(
 planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.shortcut = nn.Sequential()
if stride != 1 or in_planes != self.expansion * planes:
self.shortcut = nn.Sequential(
 nn.Conv2d(
 in_planes,
self.expansion * planes,
kernel_size=1,
stride=stride,
bias=False), nn.BatchNorm2d(self.expansion * planes))
def forward(self, x):
out = self.bn1(self.conv1(x))
out = mish(out)
 out = self.bn2(self.conv2(out))
out = mish(out)
 shrt = self.shortcut(x)
 out += mish(shrt)
 return out
class ResNet(nn.Module):
def __init__(self, block, num_blocks, num_classes=10):
super(ResNet, self ).__init__()
self.in_planes = 64
self.conv1 = nn.Conv2d(
 3, 64, kernel_size=3, stride=1, padding=1, bias=False)
 self.bn1 = nn.BatchNorm2d(64)
 self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
 self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
 self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
 self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
 self.linear = nn.Linear(512 * block.expansion, 200)
 self.linear2 = nn.Linear(200, num_classes)
 self.bn_1d_1 = nn.BatchNorm1d(num_features=200)
 self.dropout = nn.Dropout(p=0.2)
 def _make_layer(self, block, planes, num_blocks, stride):
strides = [stride] + [1] *(num_blocks - 1)
 layers = []
for stride in strides:
layers.append(block(self.in_planes, planes, stride))
self.in_planes = planes * block.expansion
return nn.Sequential(*layers)
 def forward(self, x):
out = self.bn1(self.conv1(x))
out = mish(out)
 out = self.layer1(out)
 out = self.layer2(out)
 out = self.layer3(out)
 out = self.layer4(out)
 out = F.avg_pool2d(out, 4)
 out = self.dropout(torch.flatten(out, 1))
out = self.linear(out)
 out = self.dropout(mish(self.bn_1d_1(out)))
 out = self.linear2(out)
 return F.log_softmax(out, dim=1)
 def ResNetCustom() :
return ResNet(BasicBlock, [4, 4, 4, 4] )<choose_model_class>
loader = DataLoader(dataset_test, batch_size=len(dataset_test), shuffle=False)
 model.eval()
with torch.no_grad() :
for data in loader:
ids = data['ids'].to(device)
 mask = data['mask'].to(device)
 outputs = model.forward(ids, mask)
 pred = torch.round(torch.sigmoid(outputs)).cpu().numpy()
model = ResNetCustom()<choose_model_class>
submit_csv =pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submit_csv.head()
learn = Learner(data, model, loss_func = nn.CrossEntropyLoss() , metrics=[accuracy] )<train_model>
submit_csv['target'] = pred.astype('int64')
 submit_csv.head(10 )
<train_model><EOS> 
submit_csv.to_csv("submission2.csv",index = False )