Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<compute_test_metric>
!pip -q install transformers > /dev/null
nltk.download('stopwords', quiet=True)
 stop_words = stopwords.words('english' )
lgb_clf = LGBMClassifier(random_state=17)
 lgb_clf.fit(x_train, y_train)
 y_pred = lgb_clf.predict(x_test)
 mse = mean_squared_error(y_test, y_pred)
 r = r2_score(y_test, y_pred)
 mae = mean_absolute_error(y_test,y_pred)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )<train_on_grid>
def clean_text(text):

 text = str(text ).lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text
param_grid = {'num_leaves': [7, 15, 31, 63],
'max_depth': [3, 4, 5, 6, -1]}
grid_searcher = GridSearchCV(estimator=lgb_clf, param_grid=param_grid,
cv=5, verbose=1, n_jobs=4)
 grid_searcher.fit(x_train, y_train)
 mse = mean_squared_error(y_test, y_pred)
 r = r2_score(y_test, y_pred)
 mae = mean_absolute_error(y_test,y_pred)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )<choose_model_class>
df_train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 df_train['clean_text'] = df_train['text'].apply(lambda x:clean_text(x))
df_train['Length'] = df_train.text.str.len()
df_train['Word_count'] = df_train['text'].str.split().map(len )
rf = RandomForestClassifier()
rf.fit(x,y);
Prediction = rf.predict(testdata )<save_to_csv>
dis = " ".join(df_train.loc[df_train['target'] == 1]['clean_text'] ).lower().split()
non_dis = " ".join(df_train.loc[df_train['target'] == 0]['clean_text'] ).lower().split()
top_dis = Counter([i for i in dis if i not in stop_words])
 top_non_dis = Counter([i for i in non_dis if i not in stop_words])
 temp_dis = pd.DataFrame(top_dis.most_common(40))
temp_non_dis = pd.DataFrame(top_non_dis.most_common(40))
temp_dis.columns = ['Common_words','count']
temp_non_dis.columns = ['Common_words','count']
predictionlist=Prediction.tolist()
Passengerid=testdata2['Id'].tolist()
output=pd.DataFrame(list(zip(Passengerid, predictionlist)) ,
columns=['Id','Cover_type'])
 output.head()
output.to_csv('my_submission(ForestCoverTypePrediction ).csv', index=False )<set_options>
model_class, tokenizer_class, pretrained_weights =(ppb.BertModel, ppb.BertTokenizer, 'bert-base-uncased')

tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
 model = model_class.from_pretrained(pretrained_weights)
 tokenized = df_train['text'].apply(( lambda x: tokenizer.encode(x, add_special_tokens=True)))
 max_len = 0
for i in tokenized.values:
if len(i)> max_len:
max_len = len(i)
 padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])
 attention_mask = np.where(padded != 0, 1, 0 )
pd.set_option('display.max_columns', 100)
 pd.options.mode.chained_assignment = None<define_variables>
class Dataset(torch.utils.data.Dataset):
def __init__(self, input_ids, attention_masks):
self.input_ids = input_ids
self.attention_mask = attention_masks
def __len__(self):
return len(self.input_ids)
 def __getitem__(self, index):
input_ids = self.input_ids[index]
attention_mask = self.attention_mask[index]
return input_ids, attention_mask
train_path = '.. /input/forest-cover-type-prediction/train.csv'
test_path = '.. /input/forest-cover-type-prediction/test.csv'
submit_path = '.. /input/forest-cover-type-prediction/sampleSubmission.csv'<load_from_csv>
input_ids = torch.tensor(padded)
 attention_mask = torch.tensor(attention_mask)
 dataset = Dataset(input_ids=input_ids, attention_masks=attention_mask)
 dataloader = torch.utils.data.DataLoader(dataset, batch_size=8, drop_last=False )
dtrain = pd.read_csv(train_path, index_col=0)
 dtest = pd.read_csv(test_path, index_col=0 )<count_values>
features = []
for batch_input_ids, batch_attention_mask in tqdm(dataloader):
with torch.no_grad() :
tmp = model(batch_input_ids, attention_mask=batch_attention_mask)
 features.extend(tmp[0][:,0,:].numpy())
 clear_output(True )
dtrain['Cover_Type'].value_counts()<create_dataframe>
labels = df_train['target'][:len(features)]
train_features, test_features, train_labels, test_labels = train_test_split(features, labels )
grouped_dataframe = dtrain.groupby(['Cover_Type'])
 label_dict = {1 : 'Spruce/Fir', 2 : 'Lodgepole Pine', 3 : 'Ponderosa Pine', 4 : 'Cottonwood/Willow', 5 :
'Aspen', 6 : 'Douglas-fir', 7 : 'Krummholz'}<drop_column>
clf = GradientBoostingClassifier(random_state=0)
 clf.fit(train_features, train_labels)
 clf.score(test_features, test_labels )
dtrain.drop(['Soil_Type7', 'Soil_Type15'], axis = 1, inplace = True)
 soil_columns = dtrain.columns[dtrain.columns.str.contains('Soil')].to_list()<drop_column>
pred_val = clf.predict_proba(pd.DataFrame(test_features)) [:,1]
fpr, tpr, _ = roc_curve(test_labels, pred_val)
 roc_auc = auc(fpr, tpr )
label = dtrain['Cover_Type']
dtrain.drop(['Cover_Type'], axis = 1, inplace=True )<import_modules>
test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test_tokenized = test_df['text'].apply(( lambda x: tokenizer.encode(x, add_special_tokens=True)))
 test_max_len = 0
for i in test_tokenized.values:
if len(i)> test_max_len:
test_max_len = len(i)
 test_padded = np.array([i + [0]*(test_max_len-len(i)) for i in test_tokenized.values])
 test_attention_mask = np.where(test_padded != 0, 1, 0)
 test_input_ids = torch.tensor(test_padded)
 test_attention_mask = torch.tensor(test_attention_mask)
 test_dataset = Dataset(input_ids=test_input_ids, attention_masks=test_attention_mask)
 test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=8, drop_last=False )
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.metrics import accuracy_score, f1_score, classification_report,confusion_matrix
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier<train_model>
test_features = []
for batch_input_ids, batch_attention_mask in tqdm(test_dataloader):
with torch.no_grad() :
tmp = model(batch_input_ids, attention_mask=batch_attention_mask)
 test_features.extend(tmp[0][:,0,:].numpy())
 clear_output(True )
x_train, x_test, y_train, y_test = train_test_split(dtrain, label, test_size =.3)
 dirty_clf = RandomForestClassifier()
dirty_clf.fit(x_train, y_train)
 print(dirty_clf.score(x_test, y_test))
imp_feat = pd.DataFrame(index= dtrain.columns.to_list() , data= dirty_clf.feature_importances_)
 imp_feat.rename(columns={0 : 'Importance'}, inplace=True)
 imp_feat.sort_values(by='Importance', axis =0, ascending=False)[:15]<split>
predict = clf.predict(pd.DataFrame(test_features))
sub = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 sub['target'] = predict
sub.to_csv('sample_submission.csv', index=False )
baseline_features = ['Elevation', 'Horizontal_Distance_To_Roadways']
features = ['Elevation', 'Horizontal_Distance_To_Roadways', 'Horizontal_Distance_To_Hydrology',
'Horizontal_Distance_To_Fire_Points', 'Aspect','Wilderness_Area1', 'Wilderness_Area4', 'Soil_Type3',
'Soil_Type4','Soil_Type10', 'Soil_Type29',
'Soil_Type38']
x_train, x_test, y_train, y_test = train_test_split(dtrain[features], label, test_size =.3 )<choose_model_class>
model_class, tokenizer_class, pretrained_weights =(ppb.BertForSequenceClassification, ppb.BertTokenizer, 'bert-base-cased')
 tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
 model = model_class.from_pretrained(pretrained_weights)
 tokenized = df_train['text'].apply(( lambda x: tokenizer.encode(x, add_special_tokens=True)))
 max_len = 0
for i in tokenized.values:
if len(i)> max_len:
max_len = len(i)
 padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])
 attention_mask = np.where(padded != 0, 1, 0 )
clf = DecisionTreeClassifier(criterion='gini', max_depth=8, min_samples_split=2, class_weight= None, max_features=None,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
min_impurity_split=None,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
presort='deprecated',
random_state=None, )<train_on_grid>
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )
grid_params = {'criterion' : ["gini", "entropy"]}
grid = GridSearchCV(estimator=clf, param_grid=grid_params, n_jobs=-1, cv = 5)
 grid.fit(x_train, y_train)
 grid.score(x_test, y_test )<predict_on_test>
class Dataset(torch.utils.data.Dataset):
def __init__(self, input_ids, attention_masks, labels):
self.input_ids = input_ids
self.attention_mask = attention_masks
self.labels = labels
def __len__(self):
return len(self.input_ids)
 def __getitem__(self, index):
input_ids = self.input_ids[index]
attention_mask = self.attention_mask[index]
labels = self.labels[index]
return input_ids, attention_mask, labels
grid.best_estimator_
y_pred = grid.predict(x_test )<train_model>
train_input_ids = torch.tensor(padded[:7000])
 train_attention_mask = torch.tensor(attention_mask[:7000])
 train_labels = list(df_train['target'][:7000])
 validation_input_ids = torch.tensor(padded[7000:7613])
 validation_attention_mask = torch.tensor(attention_mask[7000:7613])
 validation_labels = list(df_train['target'][7000:7613])
 dataset = Dataset(input_ids=train_input_ids, attention_masks=train_attention_mask, labels=train_labels)
 dataloader = torch.utils.data.DataLoader(dataset, batch_size=8, drop_last=False)
 validation_dataset = Dataset(input_ids=validation_input_ids, attention_masks=validation_attention_mask, labels=validation_labels)
 validation_dataloader = torch.utils.data.DataLoader(validation_dataset, batch_size=8, drop_last=False )
clf.fit(x_train, y_train)
 print(f'No of Leaves : {clf.get_n_leaves() }')
 clf.feature_importances_<compute_test_metric>
optimizer = AdamW(model.parameters() , lr = 2e-5, eps = 1e-8)
 epochs = 2
total_steps = len(dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
print(classification_report(y_test, y_pred, labels= list(label_dict.keys()), target_names=list(label_dict.values())) )<train_on_grid>
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )
rnd_clf = RandomForestClassifier()
grid_params_1 = {'max_depth' : [18], 'n_estimators' : [127], 'criterion':['entropy']}
grid = GridSearchCV(estimator=rnd_clf, param_grid=grid_params_1, n_jobs=-1, cv = 5)
 grid.fit(x_train, y_train )<compute_test_metric>
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(dataloader):
if step % 40 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )
print(grid.best_score_)
 print(grid.score(x_test, y_test)) 
<predict_on_test>
class Dataset(torch.utils.data.Dataset):
def __init__(self, input_ids, attention_masks):
self.input_ids = input_ids
self.attention_mask = attention_masks
def __len__(self):
return len(self.input_ids)
 def __getitem__(self, index):
input_ids = self.input_ids[index]
attention_mask = self.attention_mask[index]
return input_ids, attention_mask
final_clf = RandomForestClassifier(max_depth=18, n_estimators=127, criterion='entropy')
 final_clf.fit(x_train, y_train)
 print(final_clf.score(x_train, y_train))
print(final_clf.score(x_test, y_test))
y_hat = final_clf.predict(x_test )<compute_test_metric>
test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test_tokenized = test_df['text'].apply(( lambda x: tokenizer.encode(x, add_special_tokens=True)))
 test_max_len = 0
for i in test_tokenized.values:
if len(i)> test_max_len:
test_max_len = len(i)
 test_padded = np.array([i + [0]*(test_max_len-len(i)) for i in test_tokenized.values])
 test_attention_mask = np.where(test_padded != 0, 1, 0)
 test_input_ids = torch.tensor(test_padded)
 test_attention_mask = torch.tensor(test_attention_mask)
 test_dataset = Dataset(input_ids=test_input_ids, attention_masks=test_attention_mask)
 test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=8, drop_last=False )
print(classification_report(y_test, y_hat, target_names=label_dict.values()))<create_dataframe>
model.eval()
test_features = []
preds = []
for batch_input_ids, batch_attention_mask in tqdm(test_dataloader):
with torch.no_grad() :
logits = model(batch_input_ids, attention_mask=batch_attention_mask)
 a = logits[0]
a =(np.argmax(a.numpy() ,axis=1))
logits = outputs[0]
logits = logits.detach().cpu().numpy()
pred_flat = np.argmax(logits, axis=1 ).flatten()
preds.extend(a)
 clear_output(True )
imp_feat = pd.DataFrame(index= features , data= final_clf.feature_importances_)
 imp_feat.rename(columns={0 : 'Importance'}, inplace=True)
 imp_feat.sort_values(by='Importance', axis =0, ascending=False )<choose_model_class>
sub = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('sample_submission2.csv', index=False )