Kaggle competition: House Prices - Advanced Regression Techniques
if remove_rows_based_on_cv:
train_df = train_df[(( train_df['target']==0)&(train_df['outfold_pred']<0.3)) |(( train_df['target']==1)&(train_df['outfold_pred']>0.7)) ]<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
train_df.index = range(train_df.shape[0] )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
sentences = train_df.text.values
labels = train_df.target.values<load_pretrained>
train_ids = train.Id
test_ids = test.Id
train.drop('Id', inplace=True, axis=1)
 test.drop('Id', inplace=True, axis=1)
 target = train.SalePrice
train.drop('SalePrice', axis=1, inplace=True )
if model_name == 'roberta':
print('Loading RoBERTa tokenizer...')
 tokenizer = RobertaTokenizer.from_pretrained(model_shortcut_name, do_lower_case=do_lower_case)
 elif model_name == 'bert':
print('Loading BERT tokenizer...')
 tokenizer = BertTokenizer.from_pretrained(model_shortcut_name, do_lower_case=do_lower_case)
 elif model_name == 'albert':
print('Loading AlBERT tokenizer...')
 tokenizer = AlbertTokenizer.from_pretrained(model_shortcut_name, do_lower_case=do_lower_case )<categorify>
for col in train.columns:
if(train[col].dtype=='object' and train[col].isnull().sum() > 0):
filling_value = train[col].value_counts().index[0]
train[col].fillna(filling_value, axis=0, inplace=True)
 else:
if train[col].isnull().sum() >0:
filling_value = train[col].median()
train[col].fillna(filling_value, axis=0, inplace=True )
print(' Original: ', sentences[0])
 print('Tokenized: ', tokenizer.tokenize(sentences[0]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])) )<categorify>
for col in test.columns:
if(test[col].dtype=='object' and test[col].isnull().sum() > 0):
filling_value = test[col].value_counts().index[0]
test[col].fillna(filling_value, axis=0, inplace=True)
 else:
if test[col].isnull().sum() >0:
filling_value = test[col].median()
test[col].fillna(filling_value, axis=0, inplace=True )
max_len = 0
for sent in sentences:
input_ids = tokenizer.encode(sent, add_special_tokens=True)
 max_len = max(max_len, len(input_ids))
print('Max sentence length: ', max_len )<categorify>
all_data = pd.concat([train, test], sort=False)
 all_data = pd.get_dummies(all_data)
 train_ = all_data[:train.shape[0]]
test_ = all_data[train.shape[0]:]
train_['SalePrice'] = target.values
print(train_.shape)
 print(test_.shape )
input_ids = []
attention_masks = []
for sent in sentences:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = True,
max_length = 100,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 labels = torch.tensor(labels)
 print('Original: ', sentences[1])
 print('Token IDs:', input_ids[1] )<define_variables>
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.ensemble import IsolationForest
from sklearn.pipeline import Pipeline
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(0.9 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 label_temp_list = []
for a,b,c in train_dataset:
label_temp_list.append(c)
 print('{:>5,} training samples'.format(train_size))
print('{:>5,} training samples with real disater tweets'.format(sum(label_temp_list)))
 label_temp_list = []
for a,b,c in val_dataset:
label_temp_list.append(c)
 print('{:>5,} validation samples'.format(val_size))
print('{:>5,} validation samples with real disater tweets'.format(sum(label_temp_list)) )<load_pretrained>
isof = IsolationForest(n_estimators=1000, random_state=4747, n_jobs=-1, behaviour='auto')
 isof.fit(train_)
 isof_predict = isof.predict(train_)
 isof_predict = pd.DataFrame(isof_predict, columns=['Value'])
 train_ready = train_.iloc[isof_predict[ isof_predict.Value==1 ].index.values]
train_ids = train_ids.iloc[isof_predict[isof_predict.Value==1].index.values]
print('Outlier: ', isof_predict[isof_predict.Value!=1].shape[0])
 print(train_.shape)
 print(train_ready.shape )
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = batch_size 
)
 full_train_dataloader = DataLoader(
 dataset,
sampler = SequentialSampler(dataset),
batch_size = batch_size 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = batch_size
)<load_pretrained>
train_model = train_ready.drop('SalePrice', axis=1, inplace=False)
 target_model = train_ready['SalePrice']
test_model = test_
def start_model(model_name,model_shortcut_name):
if model_name == 'bert':
model = BertForSequenceClassification.from_pretrained(
 model_shortcut_name,
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 elif model_name == 'roberta':
model = RobertaForSequenceClassification.from_pretrained(
 model_shortcut_name,
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 elif model_name == 'albert':
model = AlbertForSequenceClassification.from_pretrained(
 model_shortcut_name,
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 try:
model.cuda()
except:
pass
return model<train_model>
xgb_model = XGBRegressor(n_jobs=-1, max_depth=3, min_child_weight=3,n_estimators=999, subsample=0.9)
 train_x, test_x, train_y, test_y = train_test_split(train_model,np.log(target_model), test_size=19,
random_state=47)
 xgb_model.fit(X=train_x, y= train_y)
 preds = xgb_model.predict(test_x)
 print("r2: {:.4f}".format(r2_score(test_y, preds)))
 print("mse: {:.4f}".format(mean_squared_error(test_y, preds)) )
model = start_model(model_name,model_shortcut_name )<train_model>
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))<choose_model_class>
lgb_model = LGBMRegressor(n_jobs=-1)
 train_x, test_x, train_y, test_y = train_test_split(train_model,np.log(target_model), test_size=19,
random_state=47)
 lgb_model.fit(train_x, train_y)
 preds_lgbm = lgb_model.predict(test_x)
 print("r2: {:.4f}".format(r2_score(test_y, preds_lgbm)))
 print("mse: {:.4f}".format(mean_squared_error(test_y, preds_lgbm)) )
def get_optimizer_and_scheduler(learning_rate,train_dataloader,epochs):
optimizer = AdamW(model.parameters() ,
lr = learning_rate,
eps = 1e-8 
)
 total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps)
 return optimizer,scheduler<choose_model_class>
ab = AdaBoostRegressor(random_state=4747)
 bg = BaggingRegressor(max_features=0.33, n_jobs=-1, random_state=4747)
 gbr = GradientBoostingRegressor(random_state=4747 )
optimizer,scheduler = get_optimizer_and_scheduler(learning_rate,train_dataloader,epochs )<compute_test_metric>
train_x, test_x, train_y, test_y = train_test_split(train_model,np.log(target_model), test_size=19,
random_state=47)
 ab.fit(train_x, train_y)
 bg.fit(train_x, train_y)
 gbr.fit(train_x, train_y)
 pred_ab = ab.predict(test_x)
 pred_bg = bg.predict(test_x)
 pred_gbr=gbr.predict(test_x)
 print('AdaBoost Regression')
 print("r2: {:.4f}".format(r2_score(test_y, pred_ab)))
 print("mse: {:.4f}".format(mean_squared_error(test_y, pred_ab)))
 print('Bagging Regression')
 print("r2: {:.4f}".format(r2_score(test_y, pred_bg)))
 print("mse: {:.4f}".format(mean_squared_error(test_y, pred_bg)))
 print('GradientBoosting Regression')
 print("r2: {:.4f}".format(r2_score(test_y, pred_gbr)))
 print("mse: {:.4f}".format(mean_squared_error(test_y, pred_gbr)))
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<compute_test_metric>
[x for x in(range(3, 10, 1)) ]
def flat_f1_score(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return f1_score(labels_flat,pred_flat )<set_options>
params_grid = {
'loss':['ls','lad'],
'learning_rate':[0.1, 0.5, 0.01, 0.05],
'n_estimators': [x for x in range(100, 1100, 100)],
'max_depth':[x for x in(range(3, 10, 1)) ]
}
estimator = GradientBoostingRegressor(random_state=4747)
 grid_src = GridSearchCV(estimator=estimator, param_grid=params_grid, cv=5, n_jobs=-1, verbose=2)
 grid_src.fit(train_x, train_y)
if run_test_to_get_best_epoch:
seed_val = 66
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 training_stats = []
total_t0 = time.time()
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
logits_list = []
labels_list = []
for step, batch in enumerate(train_dataloader):
if step % 40 == 0 :
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 logits_list.append(logits.detach().cpu().numpy())
 labels_list.append(b_labels.to('cpu' ).numpy())
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_accuracy = flat_accuracy(np.concatenate(logits_list),np.concatenate(labels_list),)
 training_f1_score = flat_f1_score(np.concatenate(logits_list),np.concatenate(labels_list),)
 training_time = format_time(time.time() - t0)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epcoh took: {:}".format(training_time))
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
logits_list = []
labels_list = []
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() : 
(loss, logits)= model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 logits_list.append(logits.detach().cpu().numpy())
 labels_list.append(b_labels.to('cpu' ).numpy())
 total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(" Accuracy: {0:.2f}".format(avg_val_accuracy))
testing_accuracy = flat_accuracy(np.concatenate(logits_list),np.concatenate(labels_list),)
 testing_f1_score = flat_f1_score(np.concatenate(logits_list),np.concatenate(labels_list),)
 avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'Training Time': training_time,
'Validation Time': validation_time,
'train_accuracy':training_accuracy,
'train_f1_score':training_f1_score,
'val_accuracy':testing_accuracy,
'val_f1_score':testing_f1_score,
} 
)
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )<save_to_csv>
submission_preds = lgb_model.predict(test_model)
 submission_preds_transformed = np.exp(submission_preds)
 df_lgb_submission = pd.DataFrame({'Id':test_ids, 'SalePrice':np.ceil(submission_preds_transformed)})
 df_lgb_submission.head()
if run_test_to_get_best_epoch:
df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('epoch')
 print(df_stats.to_csv() )<train_model>
submission_preds = xgb_model.predict(test_model)
 submission_preds_transformed = np.exp(submission_preds)
 df_xgb_submission = pd.DataFrame({'Id':test_ids, 'SalePrice':np.ceil(submission_preds_transformed)})
 df_xgb_submission.head()
if run_test_to_get_best_epoch:
pd.set_option('precision', 3)
 df_stats<import_modules>
merge_df = pd.merge(df_lgb_submission,df_xgb_submission, how='inner', left_on='Id', right_on='Id')
 merge_df.head()
<import_modules>
merge_df['SalePrice'] =(merge_df.SalePrice_x + merge_df.SalePrice_y)/ 2
merge_df.drop(['SalePrice_x','SalePrice_y'], axis=1, inplace=True)
 merge_df.head()
<find_best_params><EOS> 
merge_df.to_csv('submission.csv', index=False )