Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_pretrained>
import numpy as np
import pandas as pd
import missingno as msno
import seaborn as sns
import matplotlib.pyplot as plt
nlp = spacy.load('en_core_web_lg' )<train_model>
train_df=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 train_df.head()
X_train, X_valid, y_train, y_valid = train_test_split(vectors, tweets.target, test_size=0.2,
random_state=52, stratify = tweets.target)
 model = LinearSVC(random_state=1, dual=False)
 model.fit(X_train, y_train)
 print(f'Model test accuracy: {model.score(X_valid, y_valid)*100:.3f}%' )<train_model>
test_df=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 test_df.head()
second_model = LogisticRegression(solver='saga')
 second_model.fit(X_train, y_train)
 print(f'Model test accuracy: {second_model.score(X_valid, y_valid)*100:.3f}%' )<train_model>
isna_train = train_df.isnull().sum().sort_values(ascending=False)
 isna_test = test_df.isnull().sum().sort_values(ascending=False )
third_model = linear_model.RidgeClassifier()
third_model.fit(X_train, y_train)
 print(f'Model test accuracy: {third_model.score(X_valid, y_valid)*100:.3f}%' )<train_model>
( train_df.isnull().sum() /len(train_df)).sort_values(ascending=False)[:20]
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 params = {
'task' : 'train',
'boosting_type' : 'gbdt',
'objective' : 'binary',
'metric' : {'binary_logloss'},
'num_leaves' : 51,
'learning_rate' : 0.01,
'max_bin': 397,
'feature_fraction' : 0.9,
'bagging_fraction' : 0.8,
'bagging_freq': 5,
'verbose' : 0
}
gbm = lgb.train(params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 y_pred = gbm.predict(X_valid, num_iteration=gbm.best_iteration)
 print(gbm.best_score['valid_1']['binary_logloss'])
 fourth = pd.DataFrame([np.round(y_pred,0), y_valid], index=['y_pred', 'y_valid'] ).T
accuracy = fourth[fourth['y_pred']==fourth['y_valid']].shape[0]/fourth.shape[0]
print(f'Model test accuracy: {accuracy*100:.3f}%' )<compute_test_metric>
missing_percentage=(train_df.isnull().sum() /len(train_df)).sort_values(ascending=False)[:20]
print(missing_percentage.index[:5] )
print('SVM f1 score {}'.format(f1_score(y_valid, np.round(model.predict(X_valid), 0 ).astype(int))))
print('Logistic Regression f1 score {}'.format(f1_score(y_valid, np.round(second_model.predict(X_valid), 0 ).astype(int))))
print('Ridge Classifier f1 score {}'.format(f1_score(y_valid, np.round(third_model.predict(X_valid), 0 ).astype(int))))
print('Light GBM f1 score {}'.format(f1_score(y_valid, np.round(gbm.predict(X_valid, num_iteration=gbm.best_iteration), 0 ).astype(int))))<prepare_x_and_y>
train_df=train_df.drop(missing_percentage.index[:5],1)
 test_df=test_df.drop(missing_percentage.index[:5],1 )
X_train = pd.DataFrame(vectors)
 y_train = tweets['target']
X_test = pd.DataFrame(vectors_pred)
 y_preds = []
models = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=100892)
 params = {
'task' : 'train',
'boosting_type' : 'gbdt',
'objective' : 'binary',
'metric' : {'binary_logloss'},
'num_leaves' : 51,
'max_bin' : 397,
'learning_rate' : 0.01,
'feature_fraction' : 0.9,
'bagging_fraction' : 0.8,
'bagging_freq': 5,
'verbose' : 0
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.iloc[train_index]
X_val = X_train.iloc[valid_index]
y_tr = y_train.iloc[train_index]
y_val = y_train.iloc[valid_index]
lgb_train = lgb.Dataset(X_tr, y_tr)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 model = lgb.train(params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train[valid_index] = model.predict(X_val, num_iteration=model.best_iteration)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 y_preds.append(y_pred)
 models.append(model)
 scores = [
m.best_score['valid_1']['binary_logloss'] for m in models
]
score = sum(scores)/ len(scores )<compute_test_metric>
train_df['LotFrontage']=train_df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
train_df['GarageYrBlt']=train_df.groupby('Neighborhood')['GarageYrBlt'].transform(lambda x: x.fillna(x.median()))
train_df['MasVnrArea']=train_df.groupby('Neighborhood')['MasVnrArea'].transform(lambda x: x.fillna(x.median()))
print('===CV scores===')
 print(scores)
 print(score )<data_type_conversions>
test_df['LotFrontage']=test_df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
test_df['GarageYrBlt']=test_df.groupby('Neighborhood')['GarageYrBlt'].transform(lambda x: x.fillna(x.median()))
test_df['MasVnrArea']=test_df.groupby('Neighborhood')['MasVnrArea'].transform(lambda x: x.fillna(x.median()))
pred = pd.DataFrame(np.round(y_preds, 0)).astype('int8' ).T<data_type_conversions>
for column in cat_cols:
train_df[column]=train_df.groupby('Neighborhood')[column].transform(lambda x: x.fillna(x.mode()))
test_df[column]=test_df.groupby('Neighborhood')[column].transform(lambda x: x.fillna(x.mode()))
np.round(pred.mean(axis=1),0 ).astype(int )<train_model>
num_cols = train_df._get_numeric_data().columns
print("Numerical Columns",num_cols)
 cat_cols=list(set(cols)- set(num_cols))
print("Categorical Columns:",cat_cols )
<load_from_csv>
Neighbour=train_df.groupby(['Neighborhood','YearBuilt'])['SalePrice']
Neighbour=Neighbour.describe() ['mean'].to_frame()
Neighbour=Neighbour.reset_index(level=[0,1])
 Neighbour=Neighbour.groupby('Neighborhood' )
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission<data_type_conversions>
for i in cat_cols:
train_df[i]=LabelEncoder().fit_transform(train_df[i].astype(str))
test_df[i]=LabelEncoder().fit_transform(test_df[i].astype(str))
sample_submission["target"] = np.round(pred.mean(axis=1),0 ).astype(int)
 sample_submission["target"] = sample_submission["target"].astype('int8' )<save_to_csv>
index=[]
Train=pd.DataFrame()
Y=train_df['SalePrice']
for i in range(0,len(corr)) :
if corr[i] > 0.15 and corr.index[i]!='SalePrice':
index.append(corr.index[i])
 X=train_df[index]
sample_submission.to_csv("submission.csv", index=False )<install_modules>
X['cond*qual'] =(train_df['OverallCond'] * train_df['OverallQual'])/ 100.0
X['home_age_when_sold'] = train_df['YrSold'] - train_df['YearBuilt']
X['garage_age_when_sold'] = train_df['YrSold'] - train_df['GarageYrBlt']
X['TotalSF'] = train_df['TotalBsmtSF'] + train_df['1stFlrSF'] + train_df['2ndFlrSF']
X['total_porch_area'] = train_df['WoodDeckSF'] + train_df['OpenPorchSF'] + train_df['EnclosedPorch'] + train_df['3SsnPorch'] + train_df['ScreenPorch']
X['Totalsqrfootage'] =(train_df['BsmtFinSF1'] + train_df['BsmtFinSF2'] +train_df['1stFlrSF'] + train_df['2ndFlrSF'])
 X['Total_Bathrooms'] =(train_df['FullBath'] +(0.5 * train_df['HalfBath'])+train_df['BsmtFullBath'] +(0.5 * train_df['BsmtHalfBath']))
!pip install -q keras-bert keras-rectified-adam
!wget -q https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
!unzip -o uncased_L-12_H-768_A-12.zip<define_variables>
test_df['cond*qual'] =(test_df['OverallCond'] * test_df['OverallQual'])/ 100.0
test_df['home_age_when_sold'] = test_df['YrSold'] - test_df['YearBuilt']
test_df['garage_age_when_sold'] =test_df['YrSold'] - test_df['GarageYrBlt']
test_df['TotalSF'] = test_df['TotalBsmtSF'] + test_df['1stFlrSF'] + test_df['2ndFlrSF']
test_df['total_porch_area'] = test_df['WoodDeckSF'] +test_df['OpenPorchSF'] + test_df['EnclosedPorch'] + test_df['3SsnPorch'] + test_df['ScreenPorch']
test_df['Totalsqrfootage'] =(test_df['BsmtFinSF1'] + test_df['BsmtFinSF2'] +test_df['1stFlrSF'] + test_df['2ndFlrSF'])
 test_df['Total_Bathrooms'] =(test_df['FullBath'] +(0.5 * test_df['HalfBath'])+test_df['BsmtFullBath'] +(0.5 * test_df['BsmtHalfBath']))
SEQ_LEN = 128
BATCH_SIZE = 1024
EPOCHS = 15
LR = 1e-4<define_variables>
Old_Cols=['OverallCond','OverallQual','YrSold','YearBuilt','YrSold','GarageYrBlt','TotalBsmtSF','1stFlrSF','2ndFlrSF','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','BsmtFinSF1','BsmtFinSF2','1stFlrSF','2ndFlrSF','FullBath','HalfBath','BsmtFullBath','BsmtHalfBath']
pretrained_path = 'uncased_L-12_H-768_A-12'
config_path = os.path.join(pretrained_path, 'bert_config.json')
 checkpoint_path = os.path.join(pretrained_path, 'bert_model.ckpt')
 vocab_path = os.path.join(pretrained_path, 'vocab.txt')
 os.environ['TF_KERAS'] = '1'<set_options>
temp=pd.DataFrame()
temp=X
temp['SalePrice']=Y
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )<load_pretrained>
X=temp.loc[:, temp.columns != 'SalePrice']
Y=temp['SalePrice']
token_dict = {}
with codecs.open(vocab_path, 'r', 'utf8')as reader:
for line in reader:
token = line.strip()
token_dict[token] = len(token_dict)
 with tpu_strategy.scope() :
model = load_trained_model_from_checkpoint(
 config_path,
checkpoint_path,
training=True,
trainable=True,
seq_len=SEQ_LEN,
)<load_from_csv>
test_df=test_df[Final_cols]
train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv', index_col='id')
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv', index_col='id' )<string_transform>
X.isnull().sum()
def strip_html(text):
soup = BeautifulSoup(text, "html.parser")
 return soup.get_text()
def remove_between_square_brackets(text):
return re.sub('\[[^]]*\]', '', text)
 def remove_url(text):
return re.sub(r'http\S+', '', text)
 def add_space(text):
return re.sub('%20', ' ', text)
 def remove_hashtags(text):
return re.sub('
def remove_at_signs(text):
return re.sub('@', '', text)
 def remove_punctuation(text):
return text.translate(str.maketrans('', '', string.punctuation))
def denoise_text(text):
text = strip_html(text)
 text = remove_between_square_brackets(text)
 text = add_space(text)
 text = remove_url(text)
 text = remove_hashtags(text)
 text = remove_at_signs(text)
 text = remove_punctuation(text)
 return text<data_type_conversions>
test=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
def preprocess_df(df):
df = df.fillna("")
 df['text'] = df['keyword'] + " " + df['text']
del df['keyword']
df['location'] = df['location'].astype('category')
 df['location'] = df['location'].cat.codes
df['text'] = df['text'].apply(denoise_text)
 return df
train_df = preprocess_df(train_df)
 test_df = preprocess_df(test_df )<randomize_order>
test_df.fillna(test_df.mean() , inplace=True )
np.random.seed(1)
 msk = np.random.rand(len(train_df)) < 0.8
train_df, dev_df = train_df[msk], train_df[~msk]<string_transform>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 model_xgb.fit(X,Y )
tokenizer = Tokenizer(token_dict)
 def tokenize(df):
X, y = [], []
for i, index in enumerate(tqdm(df.index.values)) :
ids, segments = tokenizer.encode(df.text.values[i], max_len=SEQ_LEN)
 X.append(ids)
 try:
label = df.target.values[i]
y.append(label)
 except:
y.append(0)
 items = list(zip(X, y))
np.random.shuffle(items)
 indices, sentiments = zip(*items)
 indices = np.array(indices)
 mod = indices.shape[0] % BATCH_SIZE
if mod > 0:
indices, sentiments = indices[:-mod], sentiments[:-mod]
return [indices, np.zeros_like(indices)], np.array(sentiments)
 X_train, y_train = tokenize(train_df)
 X_dev, y_dev = tokenize(dev_df )<string_transform>
prediction = model_xgb.predict(test_df)
 pred_xgb = pd.DataFrame()
pred_xgb['Id']=test['Id']
pred_xgb['SalePrice'] = prediction
pred_xgb.to_csv(".. /working/submission_xgb.csv", index = False )
X_test = []
for i, index in enumerate(tqdm(test_df.index.values)) :
ids, segments = tokenizer.encode(test_df.text.values[i], max_len=SEQ_LEN)
 X_test.append(ids)
 X_test = [np.array(X_test), np.zeros_like(X_test)]<choose_model_class>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 GBoost.fit(X,Y )
with tpu_strategy.scope() :
inputs = model.inputs[:2]
dense = model.get_layer('NSP-Dense' ).output
outputs = keras.layers.Dense(units=2, activation='softmax' )(dense)
 model = keras.models.Model(inputs, outputs)
 model.compile(
 RAdam(lr=LR),
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'],
)<choose_model_class>
prediction = GBoost.predict(test_df)
 pred_GB = pd.DataFrame()
pred_GB['Id']=test['Id']
pred_GB['SalePrice'] = prediction
pred_GB.to_csv(".. /working/submission_GB.csv", index = False )
learning_rate_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy', patience=2, verbose=1,factor=0.5, min_lr=1e-5 )<train_model>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 model_lgb.fit(X,Y )
hist = model.fit(X_train, y_train, validation_data=(X_dev, y_dev), epochs=EPOCHS, batch_size=BATCH_SIZE, callbacks=[learning_rate_reduction] )<compute_test_metric>
prediction = model_lgb.predict(test_df)
 pred_LGB = pd.DataFrame()
pred_LGB['Id']=test['Id']
pred_LGB['SalePrice'] = prediction
pred_LGB.to_csv(".. /working/submission_LGB.csv", index = False )
print("Accuracy of the model on Training Data is - {} %".format(model.evaluate(X_train,y_train)[1]*100))
print("Accuracy of the model on Dev Data is - {} %".format(model.evaluate(X_dev,y_dev)[1]*100))<predict_on_test>
cb_model = CatBoostRegressor(iterations=700,
learning_rate=0.02,
depth=12,
eval_metric='RMSE',
random_seed = 23,
bagging_temperature = 0.2,
od_type='Iter',
metric_period = 75,
od_wait=100)
 cb_model.fit(X, Y )
classes = model.predict(X_test)[:, 0]<create_dataframe>
prediction = cb_model.predict(test_df)
 pred_CB = pd.DataFrame()
pred_CB['Id']=test['Id']
pred_CB['SalePrice'] = prediction
pred_CB.to_csv(".. /working/submission_CB.csv", index = False )
<save_to_csv><EOS> 
pred_ensemble = pd.DataFrame()
pred_ensemble['Id']=test['Id']
pred_ensemble['SalePrice'] =(0.6* pred_xgb['SalePrice'] +0.1* pred_CB['SalePrice']+0.2*pred_GB['SalePrice'] +0.1*pred_LGB['SalePrice'])
 pred_ensemble.to_csv(".. /working/submission_ensemble.csv", index = False )