Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<train_model>
warnings.filterwarnings('ignore')
 stop = set(stopwords.words('english'))
%matplotlib inline
plt.style.use('ggplot')
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 train_sent, test_sent, train_label = train.text.values, test.text.values, train.target.values
num_epochs = 3
batch_size = 32
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=seed )<load_from_csv>
word_tokenizer = Tokenizer()
word_tokenizer.fit_on_texts(train_sent)
 vocab_length = len(word_tokenizer.word_index)+ 1
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
longest_train = max(train_sent, key=lambda sentence: len(word_tokenize(sentence)))
 length_long_sentence = len(word_tokenize(longest_train))
padded_sentences = pad_sequences(embed(train_sent), length_long_sentence, padding='post')
 test_sentences = pad_sequences(embed(test_sent), length_long_sentence, padding='post' )
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
%%time
embeddings_dictionary = dict()
embedding_dim = 100
glove_file = open('.. /input/glove-file/glove.6B.100d.txt')
 for line in glove_file:
records = line.split()
word = records[0]
vector_dimensions = np.asarray(records[1:], dtype='float32')
 embeddings_dictionary[word] = vector_dimensions
glove_file.close()
embedding_matrix = np.zeros(( vocab_length, embedding_dim))
for word, index in word_tokenizer.word_index.items() :
embedding_vector = embeddings_dictionary.get(word)
 if embedding_vector is not None:
embedding_matrix[index] = embedding_vector
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
X_train, X_test, y_train, y_test = train_test_split(padded_sentences,
train_label,
test_size=0.25,
random_state=42,
shuffle=True)
print('TF version',tf.__version__ )<load_from_csv>
def training(model, model_name):
checkpoint = ModelCheckpoint(model_name + '.h5', monitor = 'val_loss', verbose = 1, save_best_only = True)
 reduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, verbose = 1, patience = 5, min_lr = 0.001)
 early_stop = EarlyStopping(monitor='val_loss', patience=1)
 start_time = time.time()
history = model.fit(X_train, y_train,
epochs = 100,
batch_size = 32,
validation_data = [X_test, y_test],
verbose = 1,
callbacks = [reduce_lr, checkpoint, early_stop])
 end_time = time.time()
print("
=>Training time :", round(end_time - start_time, 1), 's')
 model.load_weights(model_name + '.h5')
 start_time = time.time()
preds = model.predict_classes(X_test)
 end_time = time.time()
print('
=>Average Inference Time :', round(( end_time - start_time)/ len(X_test)* 1000, 1), 'ms')
 print('
=>Model Size :', round(os.stat(model_name + '.h5' ).st_size /(1024 ** 2), 1), 'MB')
 metrics(y_test, preds)
 plot(history, [['loss', 'val_loss'],['accuracy', 'val_accuracy']] )
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<categorify>
def CNN() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=length_long_sentence))
model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
 model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 return model
model = CNN()
model.summary()
train_list = train_df.to_dict('records' )<define_variables>
training(model, 'model_cnn' )
MAX_SEQUENCE_LENGTH = 0
for image_data in train_list:
MAX_SEQUENCE_LENGTH = max(MAX_SEQUENCE_LENGTH, len(image_data['text'].split(' ')))
 print(MAX_SEQUENCE_LENGTH )<define_variables>
def RNN() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=length_long_sentence))
model.add(Bidirectional(SimpleRNN(length_long_sentence, return_sequences = True, recurrent_dropout=0.2)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 return model
model = RNN()
model.summary()
MAX_SEQUENCE_LENGTH = 0
for image_data in train_list:
MAX_SEQUENCE_LENGTH = max(MAX_SEQUENCE_LENGTH, len(image_data['selected_text'].split(' ')))
 print(MAX_SEQUENCE_LENGTH )<string_transform>
training(model, 'model_rnn' )
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
def BiGRU() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=length_long_sentence))
model.add(Bidirectional(GRU(length_long_sentence, return_sequences = True, recurrent_dropout=0.2)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 return model
model = BiGRU()
model.summary()
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
training(model, 'model_bigru' )
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<categorify>
def BiLSTM() :
model = Sequential()
model.add(Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=length_long_sentence))
model.add(Bidirectional(LSTM(length_long_sentence, return_sequences = True, recurrent_dropout=0.2)))
 model.add(GlobalMaxPool1D())
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(length_long_sentence, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
 return model
model = BiLSTM()
model.summary()
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<import_modules>
training(model, 'model_bilstm' )
<load_pretrained><EOS> 
submission.target = model.predict_classes(test_sentences)
 submission.to_csv("submission.csv", index=False )