Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
!pip install transformers==3.5.1
!pip install pyspellchecker
!pip install -U joblib textblob
!python -m textblob.download_corpora|
predictions, *_ = learn.get_preds(DatasetType.Test)
 labels = np.argmax(predictions, 1)
 submission_df = pd.DataFrame({'id': list(range(0,len(labels))), 'label': labels})
 submission_df.to_csv(f'submission.csv', index=False )<choose_model_class>
import pandas as pd
import torchtext
from transformers import BertTokenizer, BertForMaskedLM, BertConfig
import transformers
import torch
from torch.utils.data import Dataset, DataLoader
from torch import optim
from torch import cuda
from sklearn.model_selection import train_test_split
import re
import string
from joblib import Parallel, delayed
from textblob import TextBlob
from textblob.translate import NotTranslated
from time import sleep
interp = ClassificationInterpretation.from_learner(learn )<import_modules>
train_val_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
train_val_df = train_val_df.loc[:,["text","target"]]
test_df = test_df.loc[:,["text"]]
test_df["target"] = [0]*len(test_df["text"] )
raw_train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 raw_test = pd.read_csv('.. /input/Kannada-MNIST/test.csv' )<prepare_x_and_y>
print(train_val_df)
 print(test_df.head())
 original_df = train_val_df.copy()
x = raw_train.iloc[:, 1:].values.astype('float32')/ 255
y = raw_train.iloc[:, 0]<split>
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not"}
def replace_typical_misspell(text):
text = text.lower()
mispellings_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
def replace(match):
return mispell_dict[match.group(0)]
return mispellings_re.sub(replace, text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : replace_typical_misspell(x))
test_df['text'] = test_df['text'].apply(lambda x : replace_typical_misspell(x))
def remove_space(string):
string = BeautifulSoup(string ).text.strip().lower()
string = re.sub(r'(( http)\S+)', 'http', string)
 string = re.sub(r'\s+', ' ', string)
 return string
train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_space(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_space(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x: remove_emoji(x))
test_df['text'] = test_df['text'].apply(lambda x: remove_emoji(x))
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.2, random_state=42 )<categorify>
print(train_val_df.loc[31])
 print(original_df.loc[31] )
x_train = x_train.reshape(-1, 28, 28,1)
 x_val = x_val.reshape(-1, 28, 28,1)
 y_train = to_categorical(y_train)
 y_val = to_categorical(y_val )<choose_model_class>
test_df.to_csv("test.tsv", sep='\t', index=False, header=None)
 print(test_df.shape)
 train_val_df.to_csv("train_eval.tsv", sep='\t', index=False, header=None)
 print(train_val_df.shape )
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)) ,
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation='softmax')
 ] )<choose_model_class>
max_length = 50
def tokenizer_50(input_text):

 return tokenizer.encode(input_text, max_length=50, return_tensors='pt')[0]
TEXT = torchtext.data.Field(sequential=True, tokenize=tokenizer_50, use_vocab=False, lower=False,
include_lengths=True, batch_first=True, fix_length=max_length, pad_token=0)
 LABEL = torchtext.data.Field(sequential=False, use_vocab=False )
optimizer = RMSprop(learning_rate=0.002,
rho=0.9,
momentum=0.1,
epsilon=1e-07,
centered=True,
name='RMSprop')
 model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'] )<define_variables>
tokenizer = BertTokenizer.from_pretrained('bert-base-cased' )
batch_size = 1024
num_classes = 10
epochs = 40<train_model>
dataset_train_eval, dataset_test = torchtext.data.TabularDataset.splits(path='.', train='./train_eval.tsv', test='./test.tsv', format='tsv', fields=[('Text', TEXT),('Label', LABEL)] )
history = model.fit(x_train, y_train, batch_size=batch_size,
epochs=15,
validation_data=(x_val, y_val))<load_from_csv>
dataset_train, dataset_eval = dataset_train_eval.split(
 split_ratio=1.0 - 1800/7613, random_state=random.seed(1234))
print(dataset_train.__len__())
 print(dataset_eval.__len__())
 print(dataset_test.__len__() )
sample_sub=pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 raw_test = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 raw_test_id=raw_test.id
raw_test=raw_test.drop("id",axis="columns")
 raw_test=raw_test / 255
test=raw_test.values.reshape(-1,28,28,1)
 test.shape<save_to_csv>
print(tokenizer.convert_ids_to_tokens(item.Text.tolist()))
print(int(item.Label))
sub=model.predict(test)
 sub=np.argmax(sub,axis=1)
 sample_sub['label']=sub
sample_sub.to_csv('submission.csv',index=False )<load_from_csv>
batch_size = 32
dl_train = torchtext.data.Iterator(
 dataset_train, batch_size=batch_size, train=True)
 dl_eval = torchtext.data.Iterator(
 dataset_eval, batch_size=batch_size, train=False, sort=False)
 dl_test = torchtext.data.Iterator(
 dataset_test, batch_size=batch_size, train=False, sort=False)
 dataloaders_dict = {"train": dl_train, "val": dl_eval}
train_csv = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 x_train = np.array(train_csv.iloc[:, 1:])
 y_train = to_categorical(train_csv.iloc[:, 0])
 x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
 x_train = x_train.astype(np.float32)
 x_train /= 255.0
y_train = y_train.astype(np.float32)
 print(x_train.shape)
 print(x_train.dtype)
 print(y_train.shape)
 print(y_train.dtype)
<load_from_csv>
model = BertModel.from_pretrained('bert-base-cased' )
val_csv = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')
 x_val = np.array(val_csv.iloc[:, 1:])
 y_val = to_categorical(val_csv.iloc[:, 0])
 x_val = x_val.reshape(x_val.shape[0], 28, 28, 1)
 x_val = x_val.astype(np.float32)
 x_val /= 255.0
y_val = y_val.astype(np.float32)
 print(x_val.shape)
 print(x_val.dtype)
 print(y_val.shape)
 print(y_val.dtype )<choose_model_class>
class BertForTwitter(nn.Module):
def __init__(self):
super(BertForTwitter, self ).__init__()
self.bert = model
self.cls = nn.Linear(in_features=768, out_features=2)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids):

 result = self.bert(input_ids)
 vec_0 = result[0]
vec_0 = vec_0[:, 0, :]
vec_0 = vec_0.view(-1, 768)
 output = self.cls(vec_0)
 return output
<load_from_csv>
net = BertForTwitter()
net.train()
print('ネットワーク設定完了' )
train = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 test = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 submission = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv" )<drop_column>
for param in net.parameters() :
param.requires_grad = False
for param in net.bert.encoder.layer[-1].parameters() :
param.requires_grad = True
for param in net.cls.parameters() :
param.requires_grad = True
X = train.drop(['label'], axis = 1)
 X_valid = test.drop(['id'], axis = 1 )<compute_test_metric>
optimizer = optim.Adam([
{'params': net.bert.encoder.layer[-1].parameters() , 'lr': 5e-5},
{'params': net.cls.parameters() , 'lr': 1e-4}
])
 criterion = nn.CrossEntropyLoss()
def haar(block):
a = pywt.dwt2(block, 'db1')
 return a<prepare_x_and_y>
def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):
max_acc = 0
Stop_flag = False
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("使用デバイス：", device)
 print('-----start-------')
 net.to(device)
 torch.backends.cudnn.benchmark = True
batch_size = dataloaders_dict["train"].batch_size
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
iteration = 1
for batch in(dataloaders_dict[phase]):
inputs = batch.Text[0].to(device)
 labels = batch.Label.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 loss = criterion(outputs, labels)
 _, preds = torch.max(outputs, 1)
 if phase == 'train':
loss.backward()
optimizer.step()
if(iteration % 50 == 0):
acc =(torch.sum(preds == labels.data)
 ).double() /batch_size
print('イテレーション {} || Loss: {:.4f} || 10iter.|| 本イテレーションの正解率：{}'.format(
 iteration, loss.item() , acc))
iteration += 1
epoch_loss += loss.item() * batch_size
epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_acc = epoch_corrects.double(
)/ len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,
phase, epoch_loss, epoch_acc))
if phase == "val":
if epoch_acc < max_acc:
count += 1
if count >= 3:
Stop_flag = True
else:
count = 0
max_acc = epoch_acc
print(count)
 if Stop_flag:
break
return net
Y = train['label'].values
X_exp = []
for i in tnrange(train.shape[0]):
im = train.iloc[i][train.columns[1:]].values.reshape(( 28,28))
a,(b,c,d)= haar(im)
 newim = np.zeros(( 14,14,4))
newim[:,:,0] = a
newim[:,:,1] = b
newim[:,:,2] = c
newim[:,:,3] = d
X_exp.append(newim)
 X = np.array(X_exp)
 X_exp = []
for i in tnrange(test.shape[0]):
im = test.iloc[i][test.columns[1:]].values.reshape(( 28,28))
a,(b,c,d)= haar(im)
 newim = np.zeros(( 14,14,4))
newim[:,:,0] = a
newim[:,:,1] = b
newim[:,:,2] = c
newim[:,:,3] = d
X_exp.append(newim)
 X_valid = np.array(X_exp)
<split>
num_epochs = 50
net_trained = train_model(net, dataloaders_dict,
criterion, optimizer, num_epochs=num_epochs )
X_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size = 0.2 )<choose_model_class>
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = ans_list
sample_submission
<choose_model_class><EOS> 
sample_submission.to_csv("submission_plus.csv", index=False )