Kaggle competition: Natural Language Processing with Disaster Tweets
def macro(y_pred, y_true):
y_pred = np.argmax(inv_y(y_pred), axis=1)
 f1 = sklearn.metrics.f1_score(y_true, y_pred, average='macro')
 return f1<choose_model_class>
train_data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 print(train_data.shape)
 train_data.head()
def getPreModel(md):
return md.get_learner(emb_szs, len(df.columns)-len(cat_vars),
0.05, 5, [100], [0.5] )<choose_model_class>
train_data.isnull().sum()
def getModel(md):
return md.get_learner(emb_szs, len(df.columns)-len(cat_vars),
0.05, 5, [100, 100], [0.5, 0.5] )<prepare_x_and_y>
train_data.fillna(value = 'None', inplace = True )
def preTrainModel(trn_idx, val_idx):
trn_Housholds = train['idhogar'].iloc[trn_idx]
val_Housholds = train['idhogar'].iloc[val_idx]
ptrn_idx = pretrain[pretrain['idhogar'].isin(trn_Housholds)].index
pval_idx = pretrain[pretrain['idhogar'].isin(val_Housholds)].index
tmpDF = df_pre.copy()
tmpY = yp.copy()
maxN = np.sum(np.equal(4, yp[ptrn_idx]))
for c in [1,2,3]:
rows = pd.Series(np.equal(c, yp[ptrn_idx]), name='bools')
 n = np.sum(rows)
 nCopyMult = maxN//n
for j in range(nCopyMult):
tmpDF = tmpDF.append(df_pre.iloc[ptrn_idx][rows.values].copy() , ignore_index=True)
 tmpY = np.append(tmpY, yp[ptrn_idx][rows].copy())
 pmd = ColumnarModelData.from_data_frame(PATH, pval_idx, tmpDF, tmpY.astype(int), cat_flds=cat_vars, bs=16,
is_reg=False, is_multi=False)
 print('Pretraining model')
 m = getPreModel(pmd)
 m.fit(2e-3, 1, wds=1e-3, metrics=[macro], cycle_len=15,use_clr=(5,8), best_save_name='pre' )<prepare_x_and_y>
sample_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 print(sample_submission.shape)
 sample_submission.head()
def trainModel(trn_idx, val_idx):
tmpDF = df.copy()
tmpY = y.copy()
maxN = np.sum(np.equal(4, y[trn_idx]))
for c in [1,2,3]:
rows = pd.Series(np.equal(c, y[trn_idx]), name='bools')
 n = np.sum(rows)
 nCopyMult = maxN//n
for j in range(nCopyMult):
tmpDF = tmpDF.append(df.iloc[trn_idx][rows.values].copy() , ignore_index=True)
 tmpY = np.append(tmpY, y[trn_idx][rows].copy())
 md = ColumnarModelData.from_data_frame(PATH, val_idx, tmpDF, tmpY.astype(int), cat_flds=cat_vars, bs=16,
test_df=df_test, is_reg=False, is_multi=False)
 m = getModel(md)
 m.model.load_state_dict(torch.load(m.get_model_path('pre')) , strict=False)
 m.freeze_to(2)
 print('Training last layer')
 m.fit(8e-2,1,wds=1e-3,cycle_len=15,use_clr=(5,8), metrics=[macro], best_save_name='tmpbest')
 m.load('tmpbest')
 m.unfreeze()
m.bn_freeze(True)
 lr = 8e-3
print('Final training')
 m.fit(np.array([lr/9,lr/3,lr])/5, 1, wds=1e-3, metrics=[macro], cycle_len=15,use_clr=(5,8), best_save_name='best')
 m.load('best')
 return m<find_best_model_class>
if torch.cuda.is_available() :
device = torch.device('cuda')
 else:
device = torch.device('cpu')
 print(device )
%%time
nSplits = 10
skf = StratifiedKFold(nSplits, True, 1234)
 folds = skf.split(df, y)
 pred_test = []
valScore = 0
for i,(trn_idx, val_idx)in enumerate(folds):
print('________________________')
 print('Running fold', i)
 preTrainModel(trn_idx, val_idx)
 m = trainModel(trn_idx, val_idx)
 score = macro(*m.predict_with_targs())
 valScore += score
print('Fold', i, 'score:', score)
 pred_test.append(m.predict(True))
print('________________________
' )<compute_test_metric>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case = True )
print("
CV finished, mean validation score:", valScore/nSplits )<prepare_output>
train_values = []
for i in trange(len(train_data)) :
keyword = train_data['keyword'][i]
location = train_data['location'][i]
text = train_data['text'][i]
sentence = keyword+' '+location+' '+text
train_values.append(sentence )
testClassPred = np.argmax(inv_y(np.mean(pred_test, axis=0)) , axis=1)
 testClassPred<feature_engineering>
train_values = ['[CLS] '+sent+' [SEP]' for sent in train_values]
tokenized_statements = [tokenizer.tokenize(sent)for sent in train_values]
print(tokenized_statements[0] )
test['Target']=testClassPred<define_variables>
input_ids = [tokenizer.convert_tokens_to_ids(x)for x in tokenized_statements]
input_ids = pad_sequences(input_ids, maxlen = 512, dtype = 'long', truncating = 'post', padding = 'post' )
csv_fn=f'{PATH}sub.csv'<save_to_csv>
attention_masks = []
for sent in input_ids:
att_mask = [int(token_id>0)for token_id in sent]
attention_masks.append(att_mask )
test[['Target']].to_csv(csv_fn, index=True )<install_modules>
labels = train_data['target'].values
!rm -rf fastai<import_modules>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, random_state = 42, test_size = 0.15)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels, random_state = 42, test_size = 0.15 )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from skimage.io import imshow, imsave
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import TruncatedSVD
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
import lightgbm as lgb
import matplotlib.pyplot as plt
import seaborn as sns
import os, sys, time
from bayes_opt import BayesianOptimization 
<load_from_csv>
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks )
Costatrain = pd.read_csv(".. /input/train.csv")
 Costatest = pd.read_csv(".. /input/test.csv")
 print("Train Dataset: Rows, Columns: ", Costatrain.shape)
 print("Test Dataset: Rows, Columns: ", Costatest.shape )<count_values>
batch_size = 8
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler = train_sampler, batch_size = batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = RandomSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler = validation_sampler, batch_size = batch_size )
Train_target = Costatrain['Target']
Train_target.value_counts(normalize=True )<create_dataframe>
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels = 2, output_attentions = False,
output_hidden_states = False)
 model.cuda()
naData = Costatrain.isnull().sum().values / Costatrain.shape[0] *100
df_na = pd.DataFrame(naData, index=Costatrain.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_count} rows which have missing value in train set ')
 df_na.head(10)
<create_dataframe>
optimizer = AdamW(model.parameters() ,
lr = 2e-5,
eps = 1e-8
)
naData = Costatest.isnull().sum().values / Costatest.shape[0] *100
df_na = pd.DataFrame(naData, index=Costatest.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_count} rows which have missing value in test set ')
 df_na.head(10)
<data_type_conversions>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis = 1 ).flatten()
labels_flat = labels.flatten()
return(np.sum(pred_flat == labels_flat)/len(labels_flat))
Costatrain['v2a1'] = Costatrain['v2a1'].fillna(value=Costatrain['tipovivi3'])
 Costatest['v2a1'] = Costatest['v2a1'].fillna(value=Costatest['tipovivi3'])
 Costatrain['v18q1'] = Costatrain['v18q1'].fillna(value=Costatrain['v18q'])
 Costatest['v18q1'] = Costatest['v18q1'].fillna(value=Costatest['v18q'])
 Costatrain.info()
Costatest.info()
Costatrain.head(5)
 Costatest.head(5 )<data_type_conversions>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
cols = ['edjefe', 'edjefa']
Costatrain[cols] = Costatrain[cols].replace({'no': 0, 'yes':1} ).astype(float)
 Costatest[cols] = Costatest[cols].replace({'no': 0, 'yes':1} ).astype(float)
<categorify>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step%40 == 0 and not step == 0:
elapsed = format_time(time.time())
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids, token_type_ids = None, attention_mask = b_input_mask, labels = b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss/len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
cols=['techozinc','techoentrepiso','techocane','techootro']
Costatrain['roof_waste_material'] = np.nan
Costatest['roof_waste_material'] = np.nan
Costatrain['electricity_other'] = np.nan
Costatest['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
Costatrain['roof_waste_material'] = Costatrain.apply(lambda x : fill_roof_exception(x),axis=1)
 Costatest['roof_waste_material'] = Costatest.apply(lambda x : fill_roof_exception(x),axis=1)
 Costatrain['electricity_other'] = Costatrain.apply(lambda x : fill_no_electricity(x),axis=1)
 Costatest['electricity_other'] = Costatest.apply(lambda x : fill_no_electricity(x),axis=1)
<feature_engineering>
real_test_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print(real_test_data.shape)
 real_test_data.head()
Costatrain['adult'] = Costatrain['hogar_adul'] - Costatrain['hogar_mayor']
Costatrain['dependency_count'] = Costatrain['hogar_nin'] + Costatrain['hogar_mayor']
Costatrain['dependency'] = Costatrain['dependency_count'] / Costatrain['adult']
Costatrain['child_percent'] = Costatrain['hogar_nin']/Costatrain['hogar_total']
Costatrain['elder_percent'] = Costatrain['hogar_mayor']/Costatrain['hogar_total']
Costatrain['adult_percent'] = Costatrain['hogar_adul']/Costatrain['hogar_total'] 
<feature_engineering>
real_test_data.isnull().sum()
Costatest['adult'] = Costatest['hogar_adul'] - Costatest['hogar_mayor']
Costatest['dependency_count'] = Costatest['hogar_nin'] + Costatest['hogar_mayor']
Costatest['dependency'] = Costatest['dependency_count'] / Costatest['adult']
Costatest['child_percent'] = Costatest['hogar_nin']/Costatest['hogar_total']
Costatest['elder_percent'] = Costatest['hogar_mayor']/Costatest['hogar_total']
Costatest['adult_percent'] = Costatest['hogar_adul']/Costatest['hogar_total'] 
<drop_column>
real_test_data.fillna(value = 'None', inplace = True )
submission = Costatest[['Id']]
Costatrain.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 Costatest.drop(columns=['idhogar','Id', 'tamhog', 'agesq', 'hogar_adul', 'SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace=True)
 correlation = Costatrain.corr()
correlation = correlation['Target'].sort_values(ascending=False)
 print(f'The most 20 positive feature:
{correlation.head(20)}')
 print('*'*50)
 print(f'The most 20 negative feature:
{correlation.tail(20)}' )<split>
test_values = []
for i in trange(len(real_test_data)) :
keyword = real_test_data['keyword'][i]
location = real_test_data['location'][i]
text = real_test_data['text'][i]
sentence = keyword+' '+location+' '+text
test_values.append(sentence )
y = Costatrain['Target']
Costatrain.drop(['Target'], inplace=True, axis=1)
<train_model>
test_values = ['[CLS] '+sent+' [SEP]' for sent in test_values]
test_tokenized_statements = [tokenizer.tokenize(sent)for sent in test_values]
print(test_tokenized_statements[0] )
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.89, min_child_samples = 90, num_leaves = 14, subsample = 0.96)
 kfold = 5
skf = StratifiedKFold(n_splits=kfold, shuffle=True)
 X=Costatrain
folds = skf.split(X,y)
 type(folds)
 predicts = []
for train_index, test_index in folds:
print("train_test_index")
 X_train, X_val = Costatrain.iloc[train_index], Costatrain.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)],
early_stopping_rounds=400, verbose=100)
 predicts.append(clf.predict(Costatest)) 
<feature_engineering>
test_input_ids = [tokenizer.convert_tokens_to_ids(x)for x in test_tokenized_statements]
test_input_ids = pad_sequences(test_input_ids, maxlen = 512, dtype = 'long', truncating = 'post', padding = 'post' )
y_pred =np.array(predicts ).mean(axis=0 ).round().astype(int)
 sub = pd.DataFrame()
sub['Id'] = submission['Id']
sub['Target'] = y_pred
sub.to_csv('submission.csv', index=False)
 sub.head() 
<import_modules>
test_attention_masks = []
for sent in test_input_ids:
att_mask = [int(token_id>0)for token_id in sent]
test_attention_masks.append(att_mask )
import numpy as np
import pandas as pd
from umap import UMAP
from sklearn.decomposition import PCA, FastICA
from sklearn.manifold import TSNE
import gc
import featuretools as ft
import featuretools.variable_types as vtypes
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import Imputer
import lightgbm as lgb
from sklearn.model_selection import KFold, train_test_split, StratifiedKFold
from sklearn.metrics import roc_auc_score, f1_score<load_from_csv>
test_inputs = torch.tensor(test_input_ids)
 test_masks = torch.tensor(test_attention_masks )
feature_matrix = pd.read_csv('.. /input/costa-rican-poverty-derived-data/ft_2000.csv')
 feature_matrix.shape<data_type_conversions>
batch_size = 8
test_data = TensorDataset(test_inputs, test_masks)
 test_sampler = SequentialSampler(test_data)
 test_dataloader = DataLoader(test_data, sampler = test_sampler, batch_size = batch_size )
feature_matrix['SUM(ind.rez_esc / escolari)'] = feature_matrix['SUM(ind.rez_esc / escolari)'].astype(np.float32)
 feature_matrix['SUM(ind.age / escolari)'] = feature_matrix['SUM(ind.age / escolari)'].astype(np.float32 )<data_type_conversions>
model.eval()
predictions , true_labels = [], []
for batch in test_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print('DONE.' )
for col in feature_matrix:
if feature_matrix[col].dtype == 'object':
if col != 'idhogar':
feature_matrix[col] = feature_matrix[col].astype(np.float32 )<filter>
preds = []
for i in range(len(predictions)) :
for j in range(len(predictions[i])) :
preds.append(np.argmax(predictions[i][j]))
feature_matrix.columns[np.where(feature_matrix.dtypes == 'object')]<feature_engineering>
submission = pd.DataFrame(columns = ['id', 'target'])
 submission['id'] = real_test_data['id']
submission['target'] = preds
<create_dataframe><EOS> 
submission.to_csv('submission.csv', index = False )