Kaggle competition: Natural Language Processing with Disaster Tweets
def preprocess_df(df):
df['cp_type'] =(df['cp_type'] == 'trt_cp' ).astype(int)
 df['cp_dose'] =(df['cp_dose'] == 'D2' ).astype(int)
 return df
x_train = preprocess_df(train_df.drop(columns="sig_id"))
y_train = tr_target_df.drop(columns="sig_id")
 x_test = preprocess_df(test_df.drop(columns="sig_id"))
N_FEATURES = x_train.shape[1]<compute_train_metric>
df_train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 df_test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 df_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
EPOCHS = 64
BATCH_SIZE = 128
FOLDS = 5
REPEATS = 5
LR = 0.0008
N_TARGETS = len(target_cols)
 def multi_log_loss(y_true, y_pred):
losses = []
for col in y_true.columns:
losses.append(log_loss(y_true.loc[:, col], y_pred.loc[:, col]))
return np.mean(losses )<choose_model_class>
if torch.cuda.is_available() :
device = torch.device("cuda")
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device("cpu" )
def my_model() :
model = tf.keras.Sequential([
tf.keras.layers.Input(N_FEATURES),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(3500, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(1750, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(875, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(412, activation="relu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(N_TARGETS, activation="sigmoid")
 ])
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = LR),
loss = 'binary_crossentropy',
metrics = ["accuracy"])
 return model<train_model>
import matplotlib.pyplot as plt
import nltk
import numpy as np
import re
import string
import seaborn as sns
from nltk.corpus import stopwords
from nltk.stem.lancaster import LancasterStemmer
from sklearn.metrics import accuracy_score
from sklearn.metrics import matthews_corrcoef
def train(resume_models = None, repeat_number = 0, folds = 5, skip_folds = 0):
models = []
oof_pred = y_train.copy()
kfold = KFold(folds, shuffle = True, random_state = 9)
 for fold,(i_tr, i_va)in enumerate(kfold.split(x_train)) :
print('-'*85)
 print(f'Repeat {repeat_number}, Fold {fold}')
 cb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
factor = 0.4,
patience = 1,
verbose = 2,
min_delta = 0.0001,
mode = 'auto')
 cb_early_stop = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss',
min_delta = 0,
patience = 4,
verbose = 1,
mode = 'min')
 checkpt_path = f'repeat:{repeat_number}_Fold:{fold}.hdf5'
cb_checkpt = tf.keras.callbacks.ModelCheckpoint(checkpt_path,
monitor = 'val_loss',
verbose = 2,
save_best_only = True,
save_weights_only = True,
mode = 'min')
 model = my_model()
model.fit(x_train.values[i_tr], y_train.values[i_tr],
validation_data =(x_train.values[i_va], y_train.values[i_va]),
callbacks = [cb_lr_schedule, cb_early_stop, cb_checkpt],
epochs = EPOCHS,
batch_size = BATCH_SIZE,
verbose = 2)
 model.load_weights(checkpt_path)
 oof_pred.loc[i_va, :] = model.predict(x_train.values[i_va])
 models.append(model)
 return models, oof_pred<train_model>
nltk.download('stopwords' )
models = []
oof_pred = []
for i in range(REPEATS):
m, oof = train(repeat_number = i, folds = FOLDS)
 print('-'*85)
 models = models + m
oof_pred.append(oof )<compute_test_metric>
df_train.target.value_counts(normalize=True )
mean_oof_pred = y_train.copy()
mean_oof_pred.loc[:, target_cols] = 0
for i, p in enumerate(oof_pred):
print(f"Repeat {i} OOF Log Loss: {multi_log_loss(y_train, p)}")
 mean_oof_pred.loc[:, target_cols] += p[target_cols]
mean_oof_pred.loc[:, target_cols] /= len(oof_pred)
 print(f"Mean OOF Log Loss: {multi_log_loss(y_train, mean_oof_pred)}")
 mean_oof_pred.loc[x_train['cp_type'] == 0, target_cols] = 0
print(f"Mean OOF Log Loss(ctl adjusted): {multi_log_loss(y_train, mean_oof_pred)}" )<save_to_csv>
alphanumeric = lambda x: re.sub('\w*\d\w*', ' ', x)
 punc_lower = lambda x: re.sub('[%s]' % re.escape(string.punctuation), ' ', x.lower())
 ' in the string and replace it with a space
remove_n = lambda x: re.sub("
", " ", x)
 remove_non_ascii = lambda x: re.sub(r'[^\x00-\x7f]',r' ', x)
 df_train['text'] = df_train['text'].map(alphanumeric ).map(punc_lower ).map(remove_n ).map(remove_non_ascii)
 df_test['text'] = df_test['text'].map(alphanumeric ).map(punc_lower ).map(remove_n ).map(remove_non_ascii)
 df_train['text'][3]
test_pred = sample_df.copy()
test_pred[target_cols] = 0
for model in models:
test_pred.loc[:,target_cols] += model.predict(x_test)
 test_pred.loc[:,target_cols] /= len(models)
 test_pred.loc[x_test['cp_type'] == 0, target_cols] = 0
test_pred.to_csv('submission.csv', index=False )<set_options>
!pip install transformers
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 100, 'display.max_columns', 900)
<load_from_csv>
sentences = df_train.text.values
sentences_test = df_test.text.values
labels = df_train.target.values
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug= pd.read_csv('.. /input/lish-moa/train_drug.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
print('Loading BERT tokenizer...')
 tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True )
genes=[col for col in train_features.columns if col.startswith('g-')]
cells=[col for col in train_features.columns if col.startswith('c-')]
print(genes)
 print(' <----------------------------------------------------------------------------------------------------------------->')
 print(cells)
 print('
')
 print(len(genes))
print('
')
 print(len(cells))<count_missing_values>
print(' Original: ', sentences[100])
 print('Tokenized: ', tokenizer.tokenize(sentences[100]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[100])) )
print(train_features.isna().sum().sum())
 print(test_features.isna().sum().sum())
 print(train_drug.isna().sum().sum())
 print(train_targets_scored.isna().sum().sum())
 print(train_targets_nonscored.isna().sum().sum() )<create_dataframe>
def tocknize(text_input):
input_ids = []
for sent in text_input:
encoded_sent = tokenizer.encode(
 sent,
add_special_tokens = True, 
)
 input_ids.append(encoded_sent)
 return input_ids
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
input_ids = tocknize(sentences )
g_features = [feature for feature in train_features.columns if feature.startswith('g-')]
c_features = [feature for feature in train_features.columns if feature.startswith('c-')]
other_features = [feature for feature in train_features.columns if feature not in g_features and
feature not in c_features and
feature not in train_targets_scored and
feature not in train_targets_nonscored]
print(f'Number of g- Features: {len(g_features)}')
 print(f'Number of c- Features: {len(c_features)}')
 print(f'Number of Other Features: {len(other_features)}({other_features})' )<create_dataframe>
MAX_LEN = 64
print('
Padding/truncating all sentences to %d values...' % MAX_LEN)
 print('
Padding token: "{:}", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long",
value=0, truncating="post", padding="post")
 print('
Done.' )
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[g_features+c_features]), pd.DataFrame(test_features[g_features+c_features])])
 data2 = qt.fit_transform(data[g_features+c_features])
 train_features[g_features+c_features] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[g_features+c_features] = pd.DataFrame(data2[-test_features.shape[0]:] )<categorify>
attention_masks = []
for sent in input_ids:
att_mask = [int(token_id > 0)for token_id in sent]
attention_masks.append(att_mask )
for col in(genes + cells):
transformer = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution='normal')
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=2018, test_size=0.1)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=2018, test_size=0.1 )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<create_dataframe>
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks )
n_comp = 650
data = pd.concat([pd.DataFrame(train_features[genes]), pd.DataFrame(test_features[genes])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[genes]))
train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )<concatenate>
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[cells]), pd.DataFrame(test_features[cells])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[cells]))
train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )<create_dataframe>
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
var_thresh = VarianceThreshold(0.845)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape 
<concatenate>
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
def feat_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = g_features
features_c = c_features
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=feat_cluster(train_features2,test_features2 )<concatenate>
optimizer = AdamW(model.parameters() ,
lr = 2e-5,
eps = 1e-8 
)
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
def feat_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = feat_cluster_pca(train_pca,test_pca )<split>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy,eval_mcc_accuracy = 0, 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
pred_flat = np.argmax(logits, axis=1 ).flatten()
labels_flat = label_ids.flatten()
tmp_eval_accuracy = accuracy_score(pred_flat, labels_flat)
 tmp_eval_mcc_accuracy = matthews_corrcoef(labels_flat, pred_flat)
 eval_accuracy += tmp_eval_accuracy
eval_mcc_accuracy += tmp_eval_mcc_accuracy
nb_eval_steps += 1
print(F'
\tValidation Accuracy: {eval_accuracy/nb_eval_steps}')
 print(F'
\tValidation MCC Accuracy: {eval_mcc_accuracy/nb_eval_steps}')
 print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
train_cluster_pca = train_cluster_pca.iloc[:,700:]
test_cluster_pca = test_cluster_pca.iloc[:,700:]<split>
given_labels = np.array(df_submission['target'])
 tensor_labels = torch.tensor(given_labels)
 input_ids_test = tocknize(sentences_test)
 def padding_seq(input_token):
out_seq = pad_sequences(input_token,
maxlen=MAX_LEN,
dtype="long",
truncating="post",
padding="post")
 return out_seq
def mask_atten(input_seq):
attention_masks = []
for seq in input_seq:
out_mask = [float(i>0)for i in seq]
attention_masks.append(out_mask)
 return attention_masks
test_pad = padding_seq(input_ids_test)
 test_mask = mask_atten(test_pad)
 tensor_input = torch.tensor(test_pad)
 tensor_mask = torch.tensor(test_mask)
 BATCH_SIZE = 32
prediction_data = TensorDataset(tensor_input, tensor_mask, tensor_labels)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler = prediction_sampler, batch_size = BATCH_SIZE)
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
print('Predicting labels for {:,} test sentences...'.format(len(tensor_input)))
 model.eval()
predictions , true_labels = [], []
for batch in prediction_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
predictions.append(logits)
 true_labels.append(label_ids)
 print('DONE.')
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167',\
'g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486',\
'g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298',\
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
<split><EOS> 
df_submission['target'] = flat_predictions
df_submission.to_csv('submission.csv', index = False)