Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<prepare_x_and_y>
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', None)
 sns.set()
X_train = fin_data.values[:,1:]
Y_train = fin_data.values[:,0]
X_test = fin_data_test.values[:,:]
Y_test = gender.values[:,1:]<train_on_grid>
home = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
params = {'penalty':['l1','l2'], 'C':[0.01,0.1,1,10,100]}
lr = LogisticRegression(solver = 'liblinear')
 grid = GridSearchCV(lr, param_grid=params, scoring ='f1', cv = 10, n_jobs=-1)
 grid.fit(X_train, Y_train)
 grid.best_params_<train_model>
test_Id = test['Id']
home.drop('Id', axis=1, inplace=True)
 test.drop('Id', axis=1, inplace=True)
lr = LogisticRegression(C = 1, penalty='l2', solver='liblinear')
 lr.fit(X_train, Y_train)
 predict_lr = lr.predict(X_test )<compute_test_metric>
home['SalePrice'] = np.log1p(home['SalePrice'] )
print("Accuracy = {0}%".format(round(accuracy_score(Y_test, predict_lr)*100, 2)))
 print(classification_report(Y_test, predict_lr))
print("Score = {0}".format(f1_score(Y_test, predict_lr)))
 print(confusion_matrix(Y_test, predict_lr))<train_on_grid>
numerical_features = home.select_dtypes(exclude='object' ).drop(['SalePrice'], axis=1 )
rf = RandomForestClassifier()
params_rf = {'n_estimators':list(range(1,20)) , 'max_depth':list(range(1,10)) , 'criterion':['gini', 'entropy']}
grid_rf = GridSearchCV(rf, param_grid=params_rf, cv = 5, scoring='accuracy', n_jobs = -1)
 grid_rf.fit(X_train,Y_train)
 print('HyperParameter optimization')
 grid_rf.best_params_<train_model>
pos = [1298,523, 297]
home.drop(home.index[pos], inplace=True )
rf = RandomForestClassifier(criterion = 'entropy', max_depth = 7, n_estimators = 16, bootstrap=False)
 rf.fit(X_train, Y_train)
 predict_rf = rf.predict(X_test)
 print("Accuracy Score = {} %".format(rf.score(X_test, Y_test)*100))
print(classification_report(Y_test, predict_rf))
print('F1 Score = {}'.format(f1_score(Y_test, predict_rf)) )<choose_model_class>
y = home.SalePrice.values
home.drop(['SalePrice'], axis=1, inplace=True)
 dataset = pd.concat(( home, test)).reset_index(drop=True )
def model() :
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(120, activation = tf.nn.relu, kernel_initializer = 'normal'))
model.add(tf.keras.layers.Dense(120, activation = tf.nn.relu, kernel_initializer = 'normal'))
model.add(tf.keras.layers.Dense(120, activation = tf.nn.relu, kernel_initializer = 'normal'))
model.add(tf.keras.layers.Dense(1, activation = tf.nn.sigmoid))
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 return model<train_model>
na_percent =(dataset.isnull().sum() /len(dataset)) [(dataset.isnull().sum() /len(dataset)) >0].sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Percentage':na_percent*100})
 missing_data
model = model()
model.fit(X_train, Y_train, epochs = 500, verbose = 0 )<compute_test_metric>
for col in('FireplaceQu', 'Fence', 'Alley', 'MiscFeature', 'PoolQC', 'MSSubClass'):
dataset[col] = dataset[col].fillna('None' )
val_loss, val_acc = model.evaluate(X_train, Y_train )<compute_test_metric>
for col in('Electrical', 'KitchenQual', 'Exterior1st', 'Exterior2nd', 'Functional', 'MSZoning', 'SaleType', 'Utilities'):
dataset[col] = dataset[col].fillna(dataset[col].mode() [0] )
val_loss, val_acc = model.evaluate(X_test, Y_test )<predict_on_test>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
dataset[col] = dataset[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
dataset[col] = dataset[col].fillna('None' )
predict_nn = model.predict([X_test])
 predict_nn_fin = []
for i in range(len(predict_nn)) :
if predict_nn[i]>=0.7:
predict_nn_fin.append(1)
 else:
predict_nn_fin.append(0 )<compute_test_metric>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
dataset[col] = dataset[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
dataset[col] = dataset[col].fillna('None' )
print("Accuracy = {0}%".format(round(accuracy_score(Y_test, predict_nn_fin)*100, 2)))
 print(classification_report(Y_test, predict_nn_fin))
print("Score = {0}".format(f1_score(Y_test, predict_nn_fin)))
 print(confusion_matrix(Y_test, predict_nn_fin))<save_to_csv>
dataset.isnull().sum() [dataset.isnull().sum() > 0].sort_values(ascending=False )
<load_from_url>
dataset['MasVnrType'] = dataset['MasVnrType'].fillna('None')
 dataset['MasVnrArea'] = dataset['MasVnrArea'].fillna(0 )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<define_variables>
dataset["LotFrontage"] = dataset.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
BASE_PATH = "/kaggle/input/nlp-getting-started/"<load_from_csv>
na_percent =(dataset.isnull().sum() /len(dataset)) [(dataset.isnull().sum() /len(dataset)) >0].sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Percentage':na_percent})
 missing_data
train =pd.read_csv(BASE_PATH + "train.csv")
 train.head()<load_from_csv>
dataset['MSSubClass'] = dataset['MSSubClass'].apply(str)
 dataset['YrSold'] = dataset['YrSold'].apply(str)
 dataset['MoSold'] = dataset['MoSold'].apply(str)
 dataset['OverallCond'] = dataset['OverallCond'].astype(str)
 dataset['LotArea'] = dataset['LotArea'].astype(np.int64)
 dataset['MasVnrArea'] = dataset['MasVnrArea'].astype(np.int64 )
test =pd.read_csv(BASE_PATH + "test.csv")
 test.head()<choose_model_class>
dataset['TotalSF']=dataset['TotalBsmtSF'] + dataset['1stFlrSF'] + dataset['2ndFlrSF']
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
dataset['TotalBath']=dataset['BsmtFullBath'] + dataset['FullBath'] +(0.5*dataset['BsmtHalfBath'])+(0.5*dataset['HalfBath'] )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<string_transform>
dataset['YrBltAndRemod']=dataset['YearBuilt']+dataset['YearRemodAdd']
text = "This is a Goat, and I am riding a Boat.... "
tokenize_ = tokenizer.tokenize(text)
 print("Text after tokenization: ")
 print(tokenize_)
 max_len = 25
text = tokenize_[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 print("After adding [CLS] and [SEP]: ")
 print(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 print("After converting Tokens to Id: ")
 print(tokens)
 tokens += [0] * pad_len
print("tokens: ")
 print(tokens)
 pad_masks = [1] * len(input_sequence)+ [0] * pad_len
print("Pad Masking: ")
 print(pad_masks)
 segment_ids = [0] * max_len
print("Segment Ids: ")
 print(segment_ids )<choose_model_class>
dataset['Porch_SF'] =(dataset['OpenPorchSF'] + dataset['3SsnPorch'] + dataset['EnclosedPorch'] + dataset['ScreenPorch'] + dataset['WoodDeckSF'] )
input_word_id = Input(shape=(max_len,),dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_id = Input(shape=(max_len,), dtype=tf.int32, name = "segment_id")
 _, sequence_output = bert_layer([input_word_id, input_mask, segment_id])
 clf_output = sequence_output[:, 0, :]
model = Model(inputs=[input_word_id, input_mask, segment_id],outputs=clf_output)
 model.compile(Adam(lr=2e-5), loss='binary_crossentropy', metrics=['accuracy'])
 model.summary()
print("shape of _ layer of BERT: "+str(_.shape))
print("shape of last layer of BERT: "+str(sequence_output.shape))<choose_model_class>
dataset['Has2ndfloor'] = dataset['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasBsmt'] = dataset['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasFirePlace'] =dataset['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 dataset['Has2ndFlr']=dataset['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 dataset['HasPool']=dataset['PoolArea'].apply(lambda x: 1 if x > 0 else 0 )
def build_model(bert_layer, max_len=512):
input_word_id = Input(shape=(max_len,),dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_id = Input(shape=(max_len,), dtype=tf.int32, name = "segment_id")
 _, sequence_output = bert_layer([input_word_id, input_mask, segment_id])
 clf_output = sequence_output[:, 0, :]
dense_layer1 = Dense(units=256,activation='relu' )(clf_output)
 dense_layer1 = Dropout(0.4 )(dense_layer1)
 dense_layer2 = Dense(units=128, activation='relu' )(dense_layer1)
 dense_layer2 = Dropout(0.4 )(dense_layer2)
 out = Dense(1, activation='sigmoid' )(dense_layer2)
 model = Model(inputs=[input_word_id, input_mask, segment_id],outputs=out)
 model.compile(Adam(lr=2e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<define_variables>
categorical_col =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for col in categorical_col:
label = LabelEncoder()
label.fit(list(dataset[col].values))
dataset[col] = label.transform(list(dataset[col].values))
print('Shape all_data: {}'.format(dataset.shape))
train_input = pre_Process_data(train.text.values, tokenizer, max_len=260)
 test_input = pre_Process_data(test.text.values, tokenizer, max_len=260)
 train_labels = train.target.values<train_model>
skewness = skewed_features[abs(skewed_features)> 0.75]
skewed_features = skewness.index
lam = 0.15
for i in skewed_features:
dataset[i] = boxcox1p(dataset[i], lam )
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=10,
callbacks=[checkpoint],
batch_size=32
)<load_from_csv>
from datetime import datetime
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error , make_scorer
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression
from sklearn.kernel_ridge import KernelRidge
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from mlxtend.regressor import StackingCVRegressor
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
submission = pd.read_csv(BASE_PATH + "sample_submission.csv")
 submission.head()<save_to_csv>
dataset = pd.get_dummies(dataset)
 X = dataset[:len(home)]
test = dataset[len(home):]
X.shape
model.load_weights('model.h5')
 test_pred = model.predict(test_input)
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False)
<load_from_url>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.20, random_state=0 )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
kfolds = KFold(n_splits=5, shuffle=True, random_state=42)
 e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [0.00005, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier
import tensorflow_hub as hub
import tokenization
from tensorflow.keras.layers import Dense, Input, Bidirectional, SpatialDropout1D, Embedding, add, concatenate
from tensorflow.keras.layers import GRU, GlobalAveragePooling1D, LSTM, GlobalMaxPooling1D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from tensorflow.keras.models import Model
import tensorflow as tf
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS
from sklearn.feature_extraction.text import TfidfVectorizer<set_options>
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas = alphas2, random_state = 42, cv=kfolds))
lasso_score = cross_val_score(lasso, X, y, cv=kfolds, scoring='neg_mean_squared_error')
 lasso_rmse = np.sqrt(-lasso_score.mean())
 print("LASSO RMSE: ", lasso_rmse)
 print("LASSO STD: ", lasso_score.std() )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_from_csv>
lasso.fit(X_train, y_train )
df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )<count_missing_values>
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas = alphas_alt, cv=kfolds))
ridge_score = cross_val_score(ridge, X, y, cv=kfolds, scoring='neg_mean_squared_error')
 ridge_rmse = np.sqrt(-ridge_score.mean())
 print("RIDGE RMSE: ", ridge_rmse)
 print("RIDGE STD: ", ridge_score.std() )
df.isnull().any()<feature_engineering>
ridge.fit(X_train, y_train )
comment_words = ' '
stopwords = set(STOPWORDS)
 for tweet in df.loc[df.target == 1].text:
tokens = tweet.split()
for i in range(len(tokens)) :
tokens[i] = tokens[i].lower()
for words in tokens:
comment_words = comment_words + words + ' '
wordcloud = WordCloud(width = 1000, height = 1000,
stopwords = stopwords,
min_font_size = 10 ).generate(comment_words)
 plt.figure(figsize =(15, 15))
plt.imshow(wordcloud)
 plt.title('Most used word in Disaster Tweet(Target = 1)')
 plt.axis("off")
 plt.show()<feature_engineering>
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
elastic_score = cross_val_score(elasticnet, X, y, cv=kfolds, scoring='neg_mean_squared_error')
 elastic_rmse = np.sqrt(-elastic_score.mean())
 print("ELASTICNET RMSE: ", elastic_rmse)
 print("ELASTICNET STD: ", elastic_score.std() )
comment_words = ' '
stopwords = set(STOPWORDS)
 for tweet in df.loc[df.target == 0].text:
tokens = tweet.split()
for i in range(len(tokens)) :
tokens[i] = tokens[i].lower()
for words in tokens:
comment_words = comment_words + words + ' '
wordcloud = WordCloud(width = 1000, height = 1000,
stopwords = stopwords,
min_font_size = 10 ).generate(comment_words)
 plt.figure(figsize =(15, 15))
plt.imshow(wordcloud)
 plt.title('Most used word in Non Disaster Tweet(Target = 0)')
 plt.axis("off")
 plt.show()<split>
elasticnet.fit(X_train, y_train )
X_train, X_test, y_train, y_test = train_test_split(df.iloc[:, 3], df.iloc[:, 4], test_size=0.2 )<feature_engineering>
lightgbm = make_pipeline(RobustScaler() ,
LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6,
min_sum_hessian_in_leaf = 11))
lightgbm_score = cross_val_score(lightgbm, X, y, cv=kfolds, scoring='neg_mean_squared_error')
 lightgbm_rmse = np.sqrt(-lightgbm_score.mean())
 print("LIGHTGBM RMSE: ", lightgbm_rmse)
 print("LIGHTGBM STD: ", lightgbm_score.std() )
def tfidf(data):
tfidf_vectorizer = TfidfVectorizer()
train = tfidf_vectorizer.fit_transform(data)
 return train, tfidf_vectorizer
X_train_tfidf, tfidf_vectorizer = tfidf(X_train)
 X_test_tfidf = tfidf_vectorizer.transform(X_test)
 submission_tfidf = tfidf_vectorizer.transform(test.iloc[:, 3] )<train_model>
lightgbm.fit(X_train, y_train )
model = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0,
learning_rate=0.1, max_delta_step=0, max_depth=7,
min_child_weight=1, missing=None, n_estimators=100, n_jobs=-1,
nthread=None, objective='binary:logistic', random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 model.fit(X_train_tfidf, y_train )<compute_test_metric>
xgboost = make_pipeline(RobustScaler() ,
XGBRegressor(learning_rate =0.01, n_estimators=3460,
max_depth=3,min_child_weight=0 ,
gamma=0, subsample=0.7,
colsample_bytree=0.7,nthread=4,
scale_pos_weight=1,seed=27,
reg_alpha=0.00006))
xgboost_score = cross_val_score(xgboost, X, y, cv=kfolds, scoring='neg_mean_squared_error')
 xgboost_rmse = np.sqrt(-xgboost_score.mean())
 print("XGBOOST RMSE: ", xgboost_rmse)
 print("XGBOOST STD: ", xgboost_score.std() )
model.score(X_test_tfidf, y_test )<predict_on_test>
xgboost.fit(X_train, y_train )
predictions = model.predict(submission_tfidf )<prepare_output>
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 stack_score = cross_val_score(stack_gen, X, y, cv=kfolds, scoring='neg_mean_squared_error')
 stack_rmse = np.sqrt(-stack_score.mean())
 print("STACK RMSE: ", stack_rmse)
 print("STACK STD: ", stack_score.std() )
submission['target'] = predictions
submission<save_to_csv>
stack_gen.fit(X_train, y_train )
submission.to_csv('tfidf_submission.csv', index=False )<choose_model_class>
results = pd.DataFrame({
'Model':['Lasso',
'Ridge',
'ElasticNet',
'LightGBM',
'XGBOOST',
'STACK'],
'Score':[lasso_rmse,
ridge_rmse,
elastic_rmse,
lightgbm_rmse,
xgboost_rmse,
stack_rmse
]})
 sorted_result = results.sort_values(by='Score', ascending=True ).reset_index(drop=True)
 sorted_result
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<categorify>
lasso_pred = lasso.predict(test)
 ridge_pred = ridge.predict(test)
 elasticnet_pred = elasticnet.predict(test)
 lightgbm_pred = lightgbm.predict(test)
 xgboost_pred = xgboost.predict(test)
 stacked_pred = stack_gen.predict(test )
def bert_encoder(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segments_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segments_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
final_predictions = np.expm1(( 0.2*elasticnet_pred)+(0.2*lasso_pred)+(0.2*ridge_pred)+ 
(0.1*xgboost_pred)+(0.1*lightgbm_pred)+(0.2*stacked_pred))
<load_pretrained><EOS> 
submission = pd.DataFrame()
submission['Id'] = test_Id
submission['SalePrice'] = final_predictions
submission.to_csv('submission.csv',index=False )