Kaggle competition: Natural Language Processing with Disaster Tweets
def seed_everything(seed=1903):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=1903 )<concatenate>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
n_comp = 28
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
!pip install inflect
var_thresh = VarianceThreshold(threshold=0.4)
 data = train_features.append(test_features)
 feature_cols = train_features.columns.values[4:]
data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed, columns=feature_cols[var_thresh.get_support() ])], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed, columns=feature_cols[var_thresh.get_support() ])], axis=1)
 train_features<merge>
b_add_keyword = True
cleaning = []
max_len = 160
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
<drop_column>
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print('n train/test: {}/{}'.format(train_df.shape[0], test_df.shape[0]))
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
print('n per class:
{}'.format(train_df['target'].value_counts()))
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )<feature_engineering>
train_df['location'].value_counts()
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
train_df['keyword'].isna().value_counts()
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
a = train_df['text']
gen_cols = [c for c in feature_cols if 'g-' == c[:2]]
cell_cols = [c for c in feature_cols if 'c-' == c[:2]]
pca_gen_cols = [c for c in feature_cols if 'pca_G' == c[:5]]
pca_cell_cols = [c for c in feature_cols if 'pca_C' == c[:5]]
ohe_cols = [c for c in feature_cols if c not in gen_cols+cell_cols+pca_gen_cols+pca_cell_cols]
print(len(gen_cols), len(cell_cols), len(pca_gen_cols), len(pca_cell_cols),len(ohe_cols))<choose_model_class>
all_words = []
for t in train_df['text'].values:
all_words.extend(wordpunct_tokenize(t))
counter = Counter(all_words)
 counter.most_common(10 )
class Model(nn.Module):
def __init__(self, num_gen_features, num_cell_features,num_targets, hidden_size):
super(Model, self ).__init__()
self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(0.2)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(0.3)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(0.2)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, int(hidden_size/2)))
 self.cell_batch_norm2 = nn.BatchNorm1d(int(hidden_size/2))
self.cell_dropout2 = nn.Dropout(0.3)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(int(hidden_size/2), int(hidden_size/2)))
 self.batch_norm3 = nn.BatchNorm1d(hidden_size + int(hidden_size/2))
self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size + int(hidden_size/2), num_targets))
def forward(self, gen_x, cell_x):
gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.cell_dense2(cell_x))
x = torch.cat(( gen_x,cell_x),dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<categorify>
def remove_url(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_numbers(text):
return re.sub(r'[^a-zA-Z']', ' ', text)
 def remove_unicode(text):
return re.sub(r'[^\x00-\x7F]+', '', text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 stop_words = set(stopwords.words('english'))
def remove_stopwords(text):
text = text.lower()
text = " ".join([w for w in wordpunct_tokenize(text)if w not in stop_words])
 return text
p = inflect.engine()
def convert_number(text):
temp_str = text.split()
new_string = []
for word in temp_str:
if word.isdigit() :
temp = p.number_to_words(word)
 new_string.append(temp)
 else:
new_string.append(word)
 temp_str = ' '.join(new_string)
 return temp_str
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
gen_x, cell_x = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(gen_x, cell_x)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<define_variables>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024<prepare_x_and_y>
def encode_text(texts, tokenizer, max_len):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 gen_x_train, cell_x_train, y_train = train_df[gen_cols + pca_gen_cols + ohe_cols].values, train_df[cell_cols + pca_cell_cols + ohe_cols].values, train_df[target_cols].values
gen_x_valid, cell_x_valid, y_valid = valid_df[gen_cols + pca_gen_cols + ohe_cols].values, valid_df[cell_cols + pca_cell_cols + ohe_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(gen_x_train, cell_x_train, y_train)
 valid_dataset = MoADataset(gen_x_valid, cell_x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_gen_features=len(gen_cols + pca_gen_cols + ohe_cols),
num_cell_features=len(cell_cols + pca_cell_cols + ohe_cols),
num_targets=len(target_cols),
hidden_size=hidden_size 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
start = time.time()
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 elapse = time.time() - start
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss} {elapse:.2f} seconds")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
gen_x_test, cell_x_test = test_[gen_cols + pca_gen_cols + ohe_cols].values, test_[cell_cols + pca_cell_cols + ohe_cols].values
testdataset = TestDataset(gen_x_test, cell_x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_gen_features=len(gen_cols + pca_gen_cols + ohe_cols),
num_cell_features=len(cell_cols + pca_cell_cols + ohe_cols),
num_targets=len(target_cols),
hidden_size=hidden_size 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
train_text = train_df['text']
test_text = test_df['text']
if b_add_keyword:
for ii in range(train_text.shape[0]):
if not pd.isna(train_df.loc[ii, 'keyword']):
train_text[ii] = train_text[ii] + ' ' + train_df.loc[ii, 'keyword']
for ii in range(test_text.shape[0]):
if not pd.isna(test_df.loc[ii, 'keyword']):
test_text[ii] = test_text[ii] + ' ' + test_df.loc[ii, 'keyword']
for ii in cleaning:
train_text = train_text.apply(eval(ii))
test_text = test_text.apply(eval(ii))
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<feature_engineering>
all_words_train = []
for t in train_text.values:
all_words_train.extend(wordpunct_tokenize(t))
all_words_test = []
for t in test_text.values:
all_words_test.extend(wordpunct_tokenize(t))
SEED = [1903, 1881]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions<merge>
counter = Counter(all_words_train)
 print('train
{}'.format(counter.most_common(10)))
 counter = Counter(all_words_test)
 print('test
{}'.format(counter.most_common(10)) )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
wordpunct_tokenize("this is great don't"), word_tokenize("this is great don't"), tokenizer.tokenize("this is great don't" )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<set_options>
train_text_enc = encode_text(train_text.values, tokenizer, max_len)
 test_text_enc = encode_text(test_text.values, tokenizer, max_len )
warnings.filterwarnings('ignore' )<import_modules>
def build_bert_model(bert_layer, max_len):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 x = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 return model
from sklearn.preprocessing import QuantileTransformer<load_from_csv>
es = EarlyStopping(monitor='val_accuracy', mode='max', patience=3, verbose=1)
 m = build_bert_model(bert_layer, max_len)
 m.summary()
train_perf = pd.DataFrame()
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
m.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'] )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<set_options>
fit_hist = m.fit(train_text_enc, train_df['target'].values, epochs=10, validation_split=0.2, batch_size=32)
 train_perf = pd.concat(( train_perf, pd.DataFrame(fit_hist.history)) , ignore_index=True )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<create_dataframe>
test_output = m.predict(test_text_enc)
 test_pred =(test_output >= 0.5 ).astype(int )
<concatenate><EOS> 
submission = pd.DataFrame(data={'id': test_df['id'].values, 'target': test_pred.reshape(-1,)})
 submission.head()
submission.to_csv(r'submission.csv', index=False)
 model_score = pd.DataFrame(data={'id': test_df['id'].values, 'target': test_pred.reshape(-1,), 'score': test_output.reshape(-1,)})
 model_score.to_csv(r'model_score_bert.csv', index=False)
 m.save('m_bert.h5' )