Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_values>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
df_train["target"].value_counts()<define_variables>
data_train = pd.read_csv(".. /input/train.csv")
 data_test = pd.read_csv(".. /input/test.csv" )
texts = df_train.text.values
labels = df_train.target.values<load_pretrained>
ntrain = data_train.shape[0]
ntest = data_test.shape[0]
train_label = data_train["SalePrice"]
data_train = data_train.drop(["SalePrice"], axis=1)
 all_data = pd.concat(( data_train, data_test)).reset_index(drop=True )
tokenizer = ElectraTokenizer.from_pretrained('google/electra-base-discriminator')
 model = ElectraForSequenceClassification.from_pretrained('google/electra-base-discriminator',num_labels=2)
 model.cuda()<categorify>
total = all_data.isnull().sum().sort_values(ascending=False)
 percent =(all_data.isnull().sum() /all_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(35 )
indices=tokenizer.batch_encode_plus(texts,max_length=64,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids=indices["input_ids"]
attention_masks=indices["attention_mask"]<split>
for col in("PoolQC", "MiscFeature", "Alley", "Fence", "FireplaceQu"):
all_data["has_" + col] = all_data[col].apply(lambda x: 0 if pd.isnull(x)else 1)
 all_data[col] = all_data[col].fillna("None" )
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,
random_state=42, test_size=0.2)
 train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,
random_state=42, test_size=0.2 )<data_type_conversions>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.mean()))
train_inputs = torch.tensor(train_inputs)
 validation_inputs = torch.tensor(validation_inputs)
 train_labels = torch.tensor(train_labels, dtype=torch.long)
 validation_labels = torch.tensor(validation_labels, dtype=torch.long)
 train_masks = torch.tensor(train_masks, dtype=torch.long)
 validation_masks = torch.tensor(validation_masks, dtype=torch.long )<load_pretrained>
all_data["has_garage"] = all_data["GarageCond"].apply(lambda x: 0 if pd.isnull(x)else 1)
 for col in("GarageType", "GarageFinish", "GarageQual","GarageCond"):
all_data[col] = all_data[col].fillna('None')
 for col in("GarageYrBlt", "GarageCars","GarageArea"):
all_data[col] = all_data[col].fillna(0 )
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
 train_sampler = RandomSampler(train_data)
 train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
 validation_sampler = SequentialSampler(validation_data)
 validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size )<compute_test_metric>
for col in("BsmtCond", "BsmtExposure", "BsmtQual", "BsmtFinType2", "BsmtFinType1"):
all_data[col] = all_data[col].fillna('None')
 for col in("BsmtHalfBath", "BsmtFullBath", "BsmtFinSF2", "BsmtUnfSF", "BsmtFinSF1","TotalBsmtSF"):
all_data[col] = all_data[col].fillna(0 )
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<categorify>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<define_variables>
for col in("MSZoning","Functional", "Electrical","KitchenQual", "Exterior1st","Exterior2nd","SaleType"):
all_data[col] = all_data[col].fillna(all_data[col].mode() [0] )
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 loss_values = []
for epoch_i in range(0, epochs):
print("")
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 model.zero_grad()
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 loss = outputs[0]
total_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_loss / len(train_dataloader)
 loss_values.append(avg_train_loss)
 print("")
 print(" Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(format_time(time.time() - t0)))
 print("")
 print("Training complete!" )<train_model>
all_data.Utilities.value_counts()
print("")
 print("Running Validation...")
 t0 = time.time()
model.eval()
preds=[]
true=[]
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in validation_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, b_labels = batch
with torch.no_grad() :
outputs = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
preds.append(logits)
 true.append(label_ids)
 tmp_eval_accuracy = flat_accuracy(logits, label_ids)
 eval_accuracy += tmp_eval_accuracy
nb_eval_steps += 1
print(" Accuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
print(" Validation took: {:}".format(format_time(time.time() - t0)) )<define_variables>
all_data = all_data.drop(["Utilities"], axis=1 )
flat_predictions = [item for sublist in preds for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
flat_true_labels = [item for sublist in true for item in sublist]<compute_test_metric>
all_data["Age_of_house"] = all_data.YrSold - all_data.YearBuilt
print(classification_report(flat_predictions,flat_true_labels))<define_variables>
for col in all_data.columns[all_data.dtypes == "object"]:
all_data[col] = all_data[col].factorize() [0]
all_data = all_data.drop(["Id"], axis=1 )
comments1 = df_test.text.values
indices1=tokenizer.batch_encode_plus(comments1,max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
 prediction_masks1 = torch.tensor(attention_masks1)
 batch_size = 32
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
 prediction_sampler1 = SequentialSampler(prediction_data1)
 prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
<predict_on_test>
data_train = all_data.iloc[:ntrain]
data_test = all_data.iloc[ntrain:]
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
 model.eval()
predictions = []
for batch in prediction_dataloader1:
batch = tuple(t.to(device)for t in batch)
 b_input_ids1, b_input_mask1 = batch
with torch.no_grad() :
outputs1 = model(b_input_ids1, token_type_ids=None,
attention_mask=b_input_mask1)
 logits1 = outputs1[0]
logits1 = logits1.detach().cpu().numpy()
predictions.append(logits1)
 flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()<save_to_csv>
train_x, val_x, train_y, val_y = train_test_split(data_train, train_label, test_size=0.3, random_state=42 )
sample_sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submit=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':flat_predictions})
<data_type_conversions>
def model_tuning(n_jobs=4,colsample_bylevel = 1, colsample_bytree=1, gamma=0.0,
learning_rate=0.1, max_depth= 3, min_child_weight=1, n_estimators=300, reg_alpha=0, reg_lambda=1,subsample=1):
model = XGBRegressor(n_jobs=n_jobs, objective='reg:linear', seed=42,silent=True,colsample_bylevel = colsample_bylevel, colsample_bytree=colsample_bytree,
gamma=gamma,learning_rate=learning_rate, max_depth= max_depth,
min_child_weight=min_child_weight, n_estimators=n_estimators, reg_alpha=reg_alpha, reg_lambda=reg_lambda,subsample=subsample)
 model_cv = cross_val_score(model, train_x, train_y, cv=5, scoring="neg_mean_squared_log_error")
 return np.sqrt(-1*model_cv.mean() )
df_leak = pd.read_csv('/kaggle/input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv', encoding ='ISO-8859-1')[['choose_one', 'text']]
df_leak['target'] =(df_leak['choose_one'] == 'Relevant' ).astype(np.int8)
 df_leak['id'] = df_leak.index.astype(np.int16)
 df_leak.drop(columns=['choose_one', 'text'], inplace=True)
 df_test = df_test.merge(df_leak, on=['id'], how='left')
 print('Leaked Data Set Shape = {}'.format(df_leak.shape))
print('Leaked Data Set Memory Usage = {:.2f} MB'.format(df_leak.memory_usage().sum() / 1024**2))
perfect_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 perfect_submission['target'] = df_test['target'].values
perfect_submission.to_csv('submission.csv', index=False)
 perfect_submission.describe()<set_options>
model = XGBRegressor(max_depth=3, subsample=0.8, colsample_bytree=0.2, colsample_bylevel=1.0, n_jobs=4, n_estimators=600, learning_rate=0.03, seed=42)
 model.fit(train_x, train_y )
%matplotlib inline 
<load_from_csv>
print("val acc", np.sqrt(mean_squared_log_error(model.predict(val_x), val_y)))
 print("train acc", np.sqrt(mean_squared_log_error(model.predict(train_x), train_y)) )
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<feature_engineering>
def model_tuner(max_depth=4, subsample=0.68,
colsample_bytree=0.2, colsample_bylevel=1.0,
n_estimators=600, learning_rate=0.03,
min_child_weight=4.5, reg_lambda=5.9):
model = XGBRegressor(max_depth=max_depth, subsample=subsample,
colsample_bytree=colsample_bytree, colsample_bylevel=colsample_bylevel,
n_jobs=4, n_estimators=n_estimators, learning_rate=learning_rate, seed=42,
min_child_weight=min_child_weight, reg_lambda=reg_lambda)
 model.fit(train_x, train_y)
 train_accuracy = np.sqrt(mean_squared_log_error(model.predict(train_x), train_y))
test_accuracy = np.sqrt(mean_squared_log_error(model.predict(val_x), val_y))
return train_accuracy, test_accuracy
train_df['char'] = train_df['text'].str.len()
train_df['words'] = train_df['text'].str.split().map(lambda x: len(x))
train_df.head(3 )<count_missing_values>
model = XGBRegressor(max_depth=4, subsample=0.68, colsample_bytree=0.2,
colsample_bylevel=1.0, n_estimators=5000,
learning_rate=0.01, min_child_weight=4.5,
reg_lambda=5.9, n_jobs=4, seed=42)
 history = model.fit(train_x, train_y,
eval_set=[(val_x, val_y)], verbose=False )
print('There are {} tweets in total and {} tweets with keyword'.format(train_df['keyword'].shape[0], train_df['keyword'].notna().sum()))<feature_engineering>
model = XGBRegressor(max_depth=4, subsample=0.68, colsample_bytree=0.2,
colsample_bylevel=1.0, n_estimators=3000,
learning_rate=0.01, min_child_weight=4.5,
reg_lambda=5.9, n_jobs=4, seed=42)
 accuracy = cross_val_score(model, data_train, train_label, cv=5, scoring="neg_mean_squared_log_error")
 print("Accuracy", np.sqrt(-accuracy.mean()))
train_df['text'] = train_df['text'].str.replace("http\S+", " ")
 test_df['text'] = test_df['text'].str.replace("http\S+", " " )<groupby>
model.fit(data_train, train_label )
<filter><EOS> 
y_pred = model.predict(data_test)
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission["SalePrice"] = y_pred
submission.to_csv('submission.csv', index=False )