Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
sns.set_style("whitegrid")
 notebookstart = time.time()
pd.options.display.max_colwidth = 500
print("Keras Version: ",keras.__version__)
 print("Tensorflow Version: ", tensorflow.__version__)
 EMBEDDING_FILES = [
'.. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim',
'.. /input/gensim-embeddings-dataset/glove.840B.300d.gensim'
]
seed = 25
N_ROWS = None
BATCH_SIZE = 64
EPOCHS = 100
N_CLASSES = 1
f1_strategy = 'macro'
MAX_LEN = 34
AUX_COLUMNS = ['target']
TEXT_COLUMN = 'text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
def build_matrix(word_index, path):
embedding_index = KeyedVectors.load(path, mmap='r')
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
for candidate in [word, word.lower() ]:
if candidate in embedding_index:
embedding_matrix[i] = embedding_index[candidate]
break
return embedding_matrix
def text_processing(df):
df['keyword'] = df['keyword'].str.replace("%20", " ")
 df['hashtags'] = df['text'].apply(lambda x: " ".join(re.findall(r"
df['hash_loc_key'] = df[['hashtags', 'location','keyword']].astype(str ).apply(lambda x: " ".join(x), axis=1)
 df['hash_loc_key'] = df["hash_loc_key"].astype(str ).str.lower().str.strip().fillna('nan')
 textfeats = ['hash_loc_key', 'text']
for cols in textfeats:
df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split()))
df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100
if cols == "text":
df[cols+"_vader_Compound"]= df[cols].apply(lambda x:SIA.polarity_scores(x)['compound'])
 return df
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
print("Read Data")
 train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv', nrows = N_ROWS)
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv', nrows = N_ROWS)
 X = train_df[TEXT_COLUMN].astype(str)
 y = train_df[TARGET_COLUMN].values
test = test_df[TEXT_COLUMN].astype(str)
 print("Train Shape: {} Rows".format(X.shape[0]))
print("Test Shape: {} Rows".format(test.shape[0]))
print('Dependent Variable Factor Ratio: ',train_df[TARGET_COLUMN].value_counts(normalize=True ).to_dict())
 tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE, lower=True)
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 test = tokenizer.texts_to_sequences(test)
 length_info = [len(x)for x in X]
print("Train Sequence Length - Mean {:.1f} +/- {:.1f}, Max {:.1f}, Min {:.1f}".format(
 np.mean(length_info), np.std(length_info), np.max(length_info), np.min(length_info)))
 X = sequence.pad_sequences(X, maxlen=MAX_LEN)
 test = sequence.pad_sequences(test, maxlen=MAX_LEN)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 print("Embeddings Matrix Shape:", embedding_matrix.shape)
 checkpoint_predictions = []
weights = []
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.3)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<feature_engineering>
SIA = SentimentIntensityAnalyzer()
train_df = text_processing(train_df)
 test_df = text_processing(test_df)
 count_vectorizer = TfidfVectorizer(
 analyzer="word",
tokenizer=nltk.word_tokenize,
preprocessor=None,
stop_words='english',
sublinear_tf=True,
ngram_range=(1, 1),
max_features=None)
 hash_loc_tfidf = count_vectorizer.fit(train_df['hash_loc_key'])
 tfvocab = hash_loc_tfidf.get_feature_names()
print("Number of TF-IDF Features: {}".format(len(tfvocab)))
 train_tfidf = count_vectorizer.transform(train_df['hash_loc_key'])
 test_tfidf = count_vectorizer.transform(test_df['hash_loc_key'] )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
dense_vars = [
'hash_loc_key_num_words',
'hash_loc_key_num_unique_words',
'hash_loc_key_words_vs_unique',
'text_num_words',
'text_num_unique_words',
'text_words_vs_unique',
'text_vader_Compound']
for d_i in dense_vars:
scaler = StandardScaler()
scaler.fit(train_df.loc[:,d_i].values.reshape(-1, 1))
train_df.loc[:,d_i] = scaler.transform(train_df.loc[:,d_i].values.reshape(-1, 1))
test_df.loc[:,d_i] = scaler.transform(test_df.loc[:,d_i].values.reshape(-1, 1))
train_num = hstack([csr_matrix(train_df.loc[:,dense_vars].values),train_tfidf] ).tocsr()
test_num = hstack([csr_matrix(test_df.loc[:,dense_vars].values),test_tfidf] ).tocsr()
num_cols = train_df[dense_vars].columns.tolist() + tfvocab
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
def build_model(embedding_matrix, n_classes, dropout=0.2):
words_inputs = Input(shape=(None,))
numeric_inputs = Input(shape=(len(num_cols),))
numeric_x = Dense(512, activation='relu' )(numeric_inputs)
 numeric_x = Dropout(dropout )(numeric_x)
 numeric_x = Dense(64, activation='relu' )(numeric_x)
 words_x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix],
trainable=False, input_length=MAX_LEN )(words_inputs)
 words_x = SpatialDropout1D(dropout )(words_x)
 words_x = Bidirectional(LSTM(128, return_sequences=True))(words_x)
 words_x = Bidirectional(LSTM(128, return_sequences=True))(words_x)
 emb_pool = concatenate([
GlobalMaxPooling1D()(words_x),
GlobalAveragePooling1D()(words_x)
 ])
 concat_x = concatenate([emb_pool, numeric_x])
 concat_x = Dropout(dropout )(concat_x)
 output = Dense(n_classes, activation='sigmoid' )(concat_x)
 model = Model(inputs=[words_inputs,numeric_inputs], outputs=output)
 opt = optimizers.Adam(learning_rate=0.00004, beta_1=0.9, beta_2=0.999, amsgrad=True)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['acc'])
 return model
model = build_model(embedding_matrix, N_CLASSES)
 model.summary()
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 25
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048<prepare_x_and_y>
oof_preds = np.zeros(X.shape[0])
 test_preds = np.zeros(test.shape[0])
 n_splits = 3
folds = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 plot_metrics = ['loss','acc']
fold_hist = {}
for i,(trn_idx, val_idx)in enumerate(folds.split(X)) :
modelstart = time.time()
model = build_model(embedding_matrix, N_CLASSES)
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=4, verbose=1,
mode='min', baseline=None, restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, min_lr=1e-7,
mode='min', verbose=1)
 history = model.fit(
 [X[trn_idx], train_num[trn_idx]],
y[trn_idx],
validation_data=([X[val_idx], train_num[val_idx]], y[val_idx]),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=0,
callbacks=[es, rlr] 
)
 best_index = np.argmin(history.history['val_loss'])
 fold_hist[i] = history
oof_preds[val_idx] = model.predict([X[val_idx], train_num[val_idx]] ).ravel()
test_preds += model.predict([test, test_num] ).ravel()
f1_sc = f1_score(y[val_idx],(oof_preds[val_idx] > 0.5 ).astype(int), average=f1_strategy)
 print("
FOLD {} COMPLETE in {:.1f} Minutes - Avg F1 {:.5f} - Best Epoch {}".format(i,(time.time() - modelstart)/60, f1_sc, best_index + 1))
best_metrics = {metric: scores[best_index] for metric, scores in history.history.items() }
pprint.pprint(best_metrics)
 f, ax = plt.subplots(1,len(plot_metrics),figsize = [12,4])
 for p_i,metric in enumerate(plot_metrics):
ax[p_i].plot(history.history[metric], label='Train ' + metric)
 ax[p_i].plot(history.history['val_' + metric], label='Val ' + metric)
 ax[p_i].set_title("{} Fold Loss Curve - {}
Best Epoch {}".format(i, metric, best_index))
ax[p_i].legend()
ax[p_i].axvline(x=best_index, c='black')
 plt.show()
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
print("OOF Classification Report for Optimised Threshold: {:.3f}".format(best_cutoff))
print(classification_report(y,(oof_preds > best_cutoff ).astype(int), digits = 4))
print(f1_score(y,(oof_preds > cutoff ).astype(int), average=f1_strategy))
print("
OOF Non-Optimised Cutoff (.5)")
 print(classification_report(y,(oof_preds >.5 ).astype(int), digits = 4))
print(f1_score(y,(oof_preds >.5 ).astype(int), average=f1_strategy))
cnf_matrix = confusion_matrix(y,(oof_preds >.5 ).astype(int))
print("OOF Confusion Matrix")
 print(cnf_matrix)
 print("OOF Normalised Confusion Matrix")
 print(( cnf_matrix.astype('float')/ cnf_matrix.sum(axis=1)[:, np.newaxis] ).round(3))
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<feature_engineering>
show_cols = [
'id',
'keyword',
'location',
'text',
'target',
'oof_preds',
'error']
print("Look at False Negative")
 display(train_df[show_cols].sort_values(by = 'error', ascending=False ).iloc[:20])
 print("Look at False Positives")
 display(train_df[show_cols].sort_values(by = 'error', ascending=True ).iloc[:20] )
SEED = [456]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<merge>
submission = pd.DataFrame.from_dict({
'id': test_df.id,
TARGET_COLUMN:(( test_preds / n_splits)> best_cutoff ).astype(int)
 })
 submission.to_csv('submission_optimised_cutoff.csv', index=False)
 print(submission[TARGET_COLUMN].value_counts(normalize = True ).to_dict())
 submission.head()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<merge>
submission = pd.DataFrame.from_dict({
'id': test_df.id,
TARGET_COLUMN:(( test_preds / n_splits)>.5 ).astype(int)
 })
 submission.to_csv('submission_fixed_cutoff.csv', index=False)
 print(submission[TARGET_COLUMN].value_counts(normalize = True ).to_dict())
 submission.head()
sub= sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )<import_modules>
oof_pd = pd.DataFrame(oof_preds, columns = ['dense_oof'])
 oof_pd.to_csv("oof_dense_nn.csv" )
<import_modules><EOS> 
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))