Kaggle competition: Natural Language Processing with Disaster Tweets
print('The prediction was based on LGBoost model with early stopping, trained on ', train_df.shape[1],' features.F1 score for the the training dataset was ',f1,'.' )<load_from_csv>
!pip install transformers
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<data_type_conversions>
from tqdm import tqdm
mapping = {'yes':1,'no':0}
needToChange = train.select_dtypes('object' ).columns[-1:1:-1]
for df in [train,test]:
for col in needToChange:
df[col] = df[col].replace(mapping ).astype(np.float64)
 train.loc[:,needToChange].describe()<concatenate>
df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 df.head()
test['Target'] = np.nan
data = train.append(test, ignore_index = True )<groupby>
sentences = df.text.values
labels = df.target.values
all_equal = train.groupby('idhogar')['Target'].apply(lambda x:x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<filter>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case = True )
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]<groupby>
input_ids = []
for sent in tqdm(sentences):
encoded_sent = tokenizer.encode(
 sent,
add_special_tokens = True, 
)
 input_ids.append(encoded_sent)
 print('Original: ', sentences[0])
 print('Token IDs:', input_ids[0] )
household_leader = train.groupby('idhogar')['parentesco1'].sum()
household_nohead = train.loc[train['idhogar'].isin(household_leader[household_leader==0].index),:]
print('There are {} households without a head.'.format(household_nohead['idhogar'].nunique()))<count_unique_values>
MAX_LEN = 64
print('
Padding/truncating all sentences to %d values...' % MAX_LEN)
 print('
Padding token: "{:}", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))
input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype="long",
value=0, truncating="post", padding="post")
 print('
Done.' )
household_nohead_equal = household_nohead.groupby('idhogar')['Target'].apply(lambda x:x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(household_nohead_equal == False)) )<feature_engineering>
attention_masks = []
for sent in tqdm(input_ids):
att_mask = [int(token_id > 0)for token_id in sent]
attention_masks.append(att_mask )
for household in not_equal.index:
true_label = int(train.loc[(train['idhogar'] == household)&(train['parentesco1'] == 1)]['Target'])
 train.loc[train['idhogar'] == household,'Target'] = true_label
all_equal = train.groupby('idhogar')['Target'].apply(lambda x:x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )<sort_values>
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=2018, test_size=0.1)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=2018, test_size=0.1 )
missing = pd.DataFrame(data.isnull().sum() ).rename(columns={0:'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values(by='percent',ascending=False ).head(10 ).drop('Target' )<groupby>
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels)
 validation_labels = torch.tensor(validation_labels)
 train_masks = torch.tensor(train_masks)
 validation_masks = torch.tensor(validation_masks )
heads.groupby('v18q')['v18q1'].apply(lambda x:x.isnull().sum() )<data_type_conversions>
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )
data['v18q1'] = data['v18q1'].fillna(0 )<count_values>
model = BertForSequenceClassification.from_pretrained(
 "bert-base-uncased",
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.cuda()
data.loc[data['tipovivi1'] == 1,'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()<feature_engineering>
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()<feature_engineering>
optimizer = AdamW(model.parameters() ,
lr = 1e-5,
eps = 1e-8
)
data.loc[data['rez_esc']>5,'rez_esc'] = 5<define_variables>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
id_ = ['Id', 'idhogar', 'Target']<define_variables>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']<define_variables>
df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 print('Number of test sentences: {:,}
'.format(df.shape[0]))
sentences = df.text.values
input_ids = []
for sent in tqdm(sentences):
encoded_sent = tokenizer.encode(
 sent,
add_special_tokens = True, 
)
 input_ids.append(encoded_sent)
 input_ids = pad_sequences(input_ids, maxlen=MAX_LEN,
dtype="long", truncating="post", padding="post")
 attention_masks = []
for seq in tqdm(input_ids):
seq_mask = [float(i>0)for i in seq]
attention_masks.append(seq_mask)
 prediction_inputs = torch.tensor(input_ids)
 prediction_masks = torch.tensor(attention_masks)
 batch_size = 32
prediction_data = TensorDataset(prediction_inputs, prediction_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))
 model.eval()
predictions=[]
for batch in tqdm(prediction_dataloader):
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print(' DONE.' )
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
answers = []
for i in tqdm(range(len(predictions))):
for j in range(len(predictions[i])) :
answers.append(softmax(predictions[i][j]))
final_answers = []
for i in range(len(answers)) :
final_answers.append(np.argmax(answers[i]))
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )<count_values>
sample = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )
np.all(np.array(list(Counter(x ).values())) == 1 )<define_variables>
sample['target'] = final_answers
<data_type_conversions><EOS> 
sample.to_csv('samplesubmission.csv',index = False)
 sample.head()