Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_values>
%matplotlib inline
train['target'].value_counts()<import_modules>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
4313/3245<randomize_order>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
train = train.reindex(np.random.permutation(train.index))
train= train.reset_index(drop=True)
 train.head()<import_modules>
y_train = train['SalePrice']
test_id = test['Id']
ntrain = train.shape[0]
ntest = test.shape[0]
from sklearn.model_selection import KFold, StratifiedKFold
from scipy.special import softmax<find_best_params>
all_data = pd.concat(( train, test), sort=True ).reset_index(drop=True)
 all_data['Dataset'] = np.repeat(['Train', 'Test'], [ntrain, ntest], axis=0)
 all_data.drop('Id', axis=1,inplace=True )
f1=sklearn.metrics.f1_score<init_hyperparams>
cols_with_missing_values = all_data.isnull().sum().sort_values(ascending=False)
 display(pd.DataFrame(cols_with_missing_values[cols_with_missing_values[cols_with_missing_values > 0].index],
columns=["Number of Missing Values"]))
model_args = {
"save_eval_checkpoints": False,
"save_model_every_epoch": False,
'reprocess_input_data': True,
'overwrite_output_dir': True,
'manual_seed': 79,
"silent": True,
'num_train_epochs': 2,
'learning_rate': 2e-5,
'fp16': False,
'max_seq_length': 64,
} 
<train_model>
all_data['PoolQC'].replace(np.nan, 'None', regex=True, inplace=True )
%%time
torch.cuda.empty_cache()
kf = StratifiedKFold(n_splits=15, shuffle=True, random_state=79)
 err=[]
y_pred_tot=[]
for train_index, test_index in kf.split(train, train['target']):
train1_trn, train1_val = train.iloc[train_index], train.iloc[test_index]
model_rb = ClassificationModel('roberta', 'roberta-base', weight=[1, 1.329], args=model_args)
 model_rb.train_model(train1_trn, eval_df=train1_val)
 result, model_outputs, _ = model_rb.eval_model(train1_val, f1=sklearn.metrics.f1_score, acc=sklearn.metrics.accuracy_score)
 print(f"f1 score:{result['f1']}", f"Accuracy:{result['acc']}")
 err.append(( result['f1']))
predictions, _ = model_rb.predict(test['final_text'])
 y_pred_tot.append(predictions)
 print("Mean f1 score: ",np.mean(err))<feature_engineering>
all_data['MiscFeature'].replace(np.nan, 'None', regex=True, inplace=True )
to_submit =np.mean(y_pred_tot,0 )<predict_on_test>
all_data['Alley'].replace(np.nan, 'None', regex=True, inplace=True )
final['target']=to_submit
final['target'] = final['target'].apply(lambda x: 1 if x>0.5 else 0)
 final.head()<save_to_csv>
all_data['Fence'].replace(np.nan, 'None', regex=True, inplace=True )
final.to_csv('model_robert_base_lr2e5_ep2_skf15_.csv',index=False )<load_from_url>
all_data['FireplaceQu'].replace(np.nan, 'None', regex=True, inplace=True )
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
garage_categorical = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']
for variable in garage_categorical:
all_data[variable].replace(np.nan, 'None', regex=True, inplace=True )
import os
import sys
import logging
import itertools
import re
import pandas as pd
import numpy as np
import sklearn.metrics
import sklearn.preprocessing
import nltk
import tensorflow as tf
import tensorflow_hub as hub
import tokenization
import matplotlib.pyplot as plt
import plotly
import plotly.graph_objects as go
import plotly.offline
import wordcloud<set_options>
garage_numeric = ['GarageYrBlt', 'GarageCars', 'GarageArea']
for variable in garage_numeric:
all_data[variable].replace(np.nan, 0, regex=True, inplace=True )
plotly.offline.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_rows = 500
pd.options.display.max_columns = None
pd.options.display.max_colwidth = 160<set_options>
basement_categorical = ['BsmtQual','BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']
for variable in basement_categorical:
all_data[variable].replace(np.nan, 'None', regex=True, inplace=True )
log = logging.getLogger(name=__name__)
 log.setLevel(logging.INFO)
 logging.captureWarnings(True)
 formatter = logging.Formatter(
 '%(asctime)s - %(name)s - %(levelname)s - %(message)s' 
)
 stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
 stream_handler.setFormatter(formatter)
 log.addHandler(stream_handler)
 log.info(f"Python version: {sys.version}")
 log.info(f"Numpy version: {np.__version__}")
 log.info(f"Pandas version: {pd.__version__}")
 log.info(f"Scikit-learn version: {sklearn.__version__}")
 log.info(f"TensorFlow version: {tf.__version__}")
 log.info(f"Plotly version: {plotly.__version__}")
 log.info(f"WordCloud version: {wordcloud.__version__}" )<define_variables>
basement_numeric = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']
for variable in basement_numeric:
all_data[variable].replace(np.nan, 0, regex=True, inplace=True )
SEED = 1
tf.random.set_seed(SEED)
 log.info(f"tensorflow.random seed: {SEED}" )<define_variables>
all_data['MasVnrType'].replace(np.nan, 'None', regex=True, inplace=True)
 all_data['MasVnrArea'].replace(np.nan, 0, regex=True, inplace=True )
SUCCESS = 0
UNK = "UNK"
NUM = "number"
AT = "recipient"
http = "http"
html = "html"
target = "target"
keyword = "keyword"
old_text = "text"
location = "location"
text = "t"
hashtag = "hashtag"
at = "at"
href = "href"
y_cols = [f"{target}_0", f"{target}_1"]<load_pretrained>
all_data['Functional'].replace(np.nan, 'Typ', regex=True, inplace=True )
try:
nltk.download('stopwords')
 except:
log.error('...')
 try:
stopwords =(nltk.corpus.stopwords.words("english")
 + ["u", "im", "st", "nd", "rd", "th"] 
)
 except:
log.error('...' )<define_variables>
mode_impute_and_plot('Utilities')
 all_data = all_data.drop('Utilities', axis=1 )
data_dir = ".. /input/nlp-getting-started"
log.info(f"Data directory: {data_dir}")
 train_bn = "train.csv"
test_bn = "test.csv"
train_fn = os.path.join(data_dir, train_bn)
 test_fn = os.path.join(data_dir, test_bn)
<load_from_csv>
cols_with_missing_values = all_data.isnull().sum().sort_values(ascending=False)
 display(pd.DataFrame(cols_with_missing_values[cols_with_missing_values[cols_with_missing_values > 0].index], columns=["Number of Missing Values"]))
df_train = pd.read_csv(train_fn)
 df_test = pd.read_csv(test_fn)
 log.info(f"Training data shape: {df_train.shape}")
 log.info(f"Test data shape: {df_test.shape}")
 train_pts = df_train.shape[0]<categorify>
all_data['MixedConditions'] = all_data['Condition1'] + ' - ' + all_data['Condition2']
all_data.drop(labels=['Condition1', 'Condition2'], axis=1, inplace=True )
def to_lower(df, col=text):

 df[col] = df[col].apply(lambda x: x.casefold())
 return SUCCESS<categorify>
all_data['MixedExterior'] = all_data['Exterior1st'] + ' - ' + all_data['Exterior2nd']
all_data.drop(labels=['Exterior1st', 'Exterior2nd'], axis=1, inplace=True )
def hash_handling(df, col=text):

 reg_hash_full = re.compile("(
 reg_hash = re.compile("(
 f = lambda x: [y.group() for y in reg_hash_full.finditer(x)]
g = lambda x: ' '.join(x)
 df[hashtag] = df[col].apply(f ).apply(g)
 df[col] = df[col].apply(lambda x: reg_hash.sub(' ', x))
return SUCCESS<categorify>
SF_df = all_data[['TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'SalePrice']]
SF_df = SF_df[~SF_df['SalePrice'].isnull() ]
SF_vars = list(SF_df)
 del SF_vars[-1]
SF_summary = []
for SF_type in SF_vars:
corr_val = np.corrcoef(SF_df[SF_type], SF_df['SalePrice'])[1][0]
SF_summary.append(corr_val)
 pd.DataFrame([SF_summary], columns=SF_vars, index=['SalePrice Correlation'] )
def at_handling(df, col=text, at_col=at):

 reg_at = re.compile("(@)")
 reg_at_full = re.compile("(@)\w+")
 f = lambda x: [y.group() for y in reg_at_full.finditer(x)]
g = lambda x: ' '.join(x)
 df[at_col] = df[col].apply(f ).apply(g)
 df[col] = df[col].apply(lambda x: reg_at_full.sub(f" {AT} ", x))
return SUCCESS<feature_engineering>
bath_df = all_data[['BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'SalePrice']]
bath_df = bath_df[~bath_df['SalePrice'].isnull() ]
bath_vars = list(bath_df)
 del bath_vars[-1]
bath_summary = []
for bath_type in bath_vars:
corr_val = np.corrcoef(bath_df[bath_type], bath_df['SalePrice'])[1][0]
bath_summary.append(corr_val)
 pd.DataFrame([bath_summary], columns=bath_vars, index=['SalePrice Correlation'] )
def href_handling(df, col=text, new_col=href):

 reg_href_full = re.compile("(htt)\S+")
 f = lambda x: len(list(reg_href_full.finditer(x)))
 df[new_col] = df[col].apply(f)
 df[col] = df[col].apply(lambda x: reg_href_full.sub(f' {http} ', x))
return SUCCESS<feature_engineering>
neighborhood_dict = dict(zip(neighborhood_prices.index, neighborhood_prices.Cluster))
all_data['Neighborhood_Class'] = all_data['Neighborhood']
all_data['Neighborhood_Class'].replace(neighborhood_dict, inplace = True )
def html_special_handling(df, col=text):

 reg_html = re.compile("(&)\w+(;)")
 df[col] = df[col].apply(lambda x: reg_html.sub(f' {html} ', x))
return SUCCESS<feature_engineering>
def outlier_detection(data):
Q1, Q3 = np.percentile(data, [25,75])
 IQR = Q3-Q1
lower_cutoff = Q1 -(IQR * 1.5)
 upper_cutoff = Q3 +(IQR * 1.5)
 outliers =(data > Q3+1.5*IQR)|(data < Q1-1.5*IQR)
 return outliers
def xc2x89_byte_handling(df, col=text):

 reg_x89 = re.compile(b"\xc2\x89".decode('utf-8')+"\S+")
 df[col] = df[col].apply(lambda x: reg_x89.sub(' ', x))
return SUCCESS<feature_engineering>
temp_train = train_LotFrontage_no_outliers.copy()
temp_train.rename(columns={'1stFlrSF':'X1stFlrSF', '2ndFlrSF': 'X2ndFlrSF', '3SsnPorch': 'X3SsnPorch'}, inplace=True)
 mod = ols('LotFrontage ~ X1stFlrSF + X2ndFlrSF + X3SsnPorch + Alley + BedroomAbvGr + BldgType + BsmtCond + BsmtExposure + BsmtFinSF1 + BsmtFinSF2 + BsmtFinType1 + BsmtFinType2 + BsmtFullBath + BsmtHalfBath + BsmtQual + BsmtUnfSF + CentralAir + Electrical + EnclosedPorch + ExterCond + ExterQual + Fence + FireplaceQu + Fireplaces + Foundation + FullBath + Functional + GarageArea + GarageCars + GarageCond + GarageFinish + GarageQual + GarageType + GarageYrBlt + GrLivArea + HalfBath + Heating + HeatingQC + HouseStyle + KitchenAbvGr + KitchenQual + LandContour + LandSlope + LotArea + LotConfig + LotShape + LowQualFinSF + MSSubClass + MSZoning + MasVnrArea + MasVnrType + MiscFeature + MiscVal + MoSold + Neighborhood + OpenPorchSF + OverallCond + OverallQual + PavedDrive + PoolArea + PoolQC + RoofMatl + RoofStyle + SaleCondition + SaleType + ScreenPorch + Street + TotRmsAbvGrd + TotalBsmtSF + WoodDeckSF + YearBuilt + YearRemodAdd + YrSold + MixedConditions + MixedExterior + TotalSF + TotalBath + Neighborhood_Class', data=temp_train ).fit()
aov_table = sm.stats.anova_lm(mod, typ=3)
 display(aov_table[aov_table['PR(>F)'] < 0.01] )
def special_char_handling(df, col=text):

 reg_special = re.compile("[^\w\s]")
 df[col] = df[col].apply(lambda x: reg_special.sub(' ', x))
df[col] = df[col].apply(lambda x: re.sub('_', ' ', x))
return SUCCESS<categorify>
X = train_LotFrontage_no_outliers[aov_table[aov_table['PR(>F)'] < 0.01].index]
y = train_LotFrontage_no_outliers['LotFrontage']
X = pd.get_dummies(X)
 transformer = StandardScaler().fit(X)
 X = transformer.transform(X )
def contraction_handling(df, col=text):

 reg_contract = re.compile("\s(s|m|t|(nt)|(ve)|w)\s")
 df[col] = df[col].apply(lambda x: reg_contract.sub(' ', x))
return SUCCESS<categorify>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1234)
 clf = svm.SVR(kernel='rbf', C=100, gamma=0.001)
 clf.fit(X_train, y_train)
 preds = clf.predict(X_test)
 print('Mean Absolute Error: %.3f' % mean_absolute_error(y_test, preds))
def encode_numerals(df, col=text):

 reg_numerals = re.compile("\d+[\s\d]*")
 df[col] = df[col].apply(lambda x: reg_numerals.sub(f' {NUM} ', x))
return SUCCESS<feature_engineering>
model_data = train_LotFrontage_no_outliers.copy()
model_data= model_data.append(test_LotFrontage)
 y = model_data['LotFrontage']
model_data = model_data[['MSZoning', 'Alley', 'LotArea', 'LotShape', 'LotConfig', 'Neighborhood', 'MixedConditions', 'GarageType', 'GarageCars', 'GarageArea']]
model_data = pd.get_dummies(model_data)
 model_X_train = model_data[~y.isnull() ]
model_X_test = model_data[y.isnull() ]
model_y_train = y[~y.isnull() ]
transformer = StandardScaler().fit(model_X_train)
 model_X_train = transformer.transform(model_X_train)
 model_X_test = transformer.transform(model_X_test)
 clf = svm.SVR(kernel='rbf', C=100, gamma=0.001)
 clf.fit(model_X_train, model_y_train)
 LotFrontage_preds = clf.predict(model_X_test )
def remove_stopwords(df, col=text, to_remove=stopwords):

 f =(lambda x:
' '.join([y for y in x.strip().split() if y not in to_remove])
)
 df[col] = df[col].apply(f)
 return SUCCESS<categorify>
all_data.LotFrontage[all_data.LotFrontage.isnull() ] = LotFrontage_preds
def preprocess(df, col=text, old_col=old_text):

 df[col] = df[old_col]
to_lower(df)
 hash_handling(df)
 at_handling(df)
 href_handling(df)
 html_special_handling(df)
 xc2x89_byte_handling(df)
 special_char_handling(df)
 contraction_handling(df)
 remove_stopwords(df)
 encode_numerals(df)
 return SUCCESS<train_model>
all_data.drop(all_data['TotalSF'].iloc[:ntrain].nlargest().index[:2], axis=0, inplace=True)
 ntrain = train.shape[0]-2
log.info(f"Training set preprocessing status: {preprocess(df_train)}.")
 log.info(f"Test set preprocessing status: {preprocess(df_test)}." )<count_values>
num_to_str_columns = ['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']
for col in num_to_str_columns:
all_data[col] = all_data[col].astype('str' )
ave_words_positive =(
 df_train.loc[df_train[target]==1, text].apply(lambda x: len(x.split())) 
.sum()
/ df_train.loc[df_train[target]==1, text].count() 
)
 ave_words_negative =(
 df_train.loc[df_train[target]==0, text].apply(lambda x: len(x.split())) 
.sum()
/ df_train.loc[df_train[target]==0, text].count() 
)
 log.info(f"Average words in a positive example {ave_words_positive}.
 "
f"Average words in a negative example {ave_words_negative}."
)<count_unique_values>
cat_cols = ['OverallQual', 'OverallCond', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'BsmtFinSF2', 'HeatingQC', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr',
'KitchenAbvGr', 'KitchenQual', 'TotRmsAbvGrd', 'Fireplaces', 'FireplaceQu', 'GarageFinish', 'GarageCars',
'GarageQual', 'GarageCond', 'PoolQC', 'Fence', 'YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'MoSold', 'YrSold']
for col in cat_cols:
label = LabelEncoder()
label.fit(list(all_data[col].values))
all_data[col] = label.transform(list(all_data[col].values))
print('Shape all_data: {}'.format(all_data.shape))
all_data.head()
tokenize_flatten = lambda series:(
 list(itertools.chain(*[x.split() for x in series])) 
)
 wc_size =(14, 14)
 tdf = df_train[df_train[target]==1]
unique_words, word_counts =(
 np.unique(tokenize_flatten(tdf[text]), return_counts=True)
)
 sm = np.sum(word_counts)
 frequency_dict = {
x: word_counts[i]/sm
for i, x in np.ndenumerate(unique_words)
 }
try:
frequency_dict.pop(NUM)
 except:
log.error(NUM)
 try:
frequency_dict.pop(AT)
 except:
log.error(AT)
 try:
frequency_dict.pop(http)
 except:
log.error(http)
 try:
frequency_dict.pop(html)
 except:
log.error(html)
 try:
frequency_dict.pop(UNK)
 except:
log.error(UNK)
 wc =(
 wordcloud.WordCloud(
 width=1000, height=1000,
background_color='white',
min_font_size=10
).generate_from_frequencies(frequency_dict)
)
 fig = plt.figure(figsize=wc_size, facecolor=None)
 ax = fig.add_subplot()
ax.set_title("Words in the disaster tweets")
 _ = ax.imshow(wc )<count_values>
numeric_feats = ['LotFrontage', 'LotArea', 'BsmtFinSF1', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch',
'3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'TotalSF']
skewed_feats = all_data[numeric_feats].apply(lambda x: st.skew(x.dropna())).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew Before Transformation' :skewed_feats})
 skewness = skewness[abs(skewness)> 1].dropna(axis=0)
 skewed_features = skewness.index
for feat in skewed_features:
all_data[feat] = np.log1p(all_data[feat]+1)
 skewed_feats = all_data[skewed_features].apply(lambda x: st.skew(x.dropna())).sort_values(ascending=False)
 skewness['Skew After Transformation'] = skewed_feats
skewness
def bigrams_count(df, col, top_n=10):

 words = [x.split() for x in df[col]]
bigrams = [x[i]+"_"+x[i+1] for x in words for i in range(len(x)-1)]
uniq_pairs, counts = np.unique(np.array(bigrams), return_counts=True)
 return np.array([uniq_pairs, counts] )<data_type_conversions>
all_data = pd.get_dummies(all_data)
 print(all_data.shape)
 all_data.head(3 )
df_bi_0 = pd.DataFrame(
 bigrams_count(df_train[df_train[target]==0], text ).T,
columns=["bigram", "count"] 
)
 df_bi_1 = pd.DataFrame(
 bigrams_count(df_test[df_train[target]==0], text ).T,
columns=["bigram", "count"] 
)
 df_bi = df_bi_0.merge(df_bi_1, how="outer", left_on="bigram",
suffixes=("_0", "_1"), right_on="bigram")
 df_bi.fillna(0, inplace=True)
 df_bi = df_bi.astype({"count_0": int, "count_1": int})
 df_bi["total"] = df_bi["count_0"] + df_bi["count_1"]<count_duplicates>
final_y_train = all_data['SalePrice'][~all_data['SalePrice'].isnull() ]
final_X_train = all_data[all_data['Dataset_Train'] == 1].drop(['Dataset_Train', 'Dataset_Test', 'SalePrice'], axis=1)
 final_X_test = all_data[all_data['Dataset_Test'] == 1].drop(['Dataset_Train', 'Dataset_Test', 'SalePrice'], axis=1 )
top_n = 100
top_bigrams = df_bi.nlargest(top_n, "total" )<string_transform>
n_folds = 10
def rmse_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=1234 ).get_n_splits(final_X_train.values)
 rmse= np.sqrt(-cross_val_score(model, final_X_train.values, final_y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
def tokenize_dataframe(df, col, max_len=20):

 df_tmp = pd.DataFrame(
 df[col].apply(lambda x: reversed(x.split())).tolist() 
)
 orig_len = len(df_tmp.columns)
 df_tmp = df_tmp.rename(
 lambda x: f'{col}_{max_len-1-x:02d}',
axis=1 
)
 enum_cols = [f'{col}_{i:02d}' for i in range(max_len)]
if orig_len < max_len:
compl_cols = [x for x in enum_cols if x not in df_tmp.columns]
df_tmp[compl_cols] = np.nan
df[enum_cols] = df_tmp[enum_cols]
return enum_cols<categorify>
lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.0005, random_state = 1234))
def transform_data(df, col=text):

 word_cols = tokenize_dataframe(df, col, max_len=25)
 df[word_cols] = df[word_cols].fillna('')
 lemmatizer = nltk.stem.WordNetLemmatizer()
ps = nltk.stem.PorterStemmer()
df[word_cols] = df[word_cols].applymap(lambda x: ps.stem(x))
df[word_cols] = df[word_cols].applymap(lambda x: lemmatizer.lemmatize(x))
_ = filter_infrequent(df, word_cols, cutoff=10)
 word_enc = sklearn.preprocessing.LabelEncoder()
df[word_cols] =(word_enc 
.fit_transform(df[word_cols].values.flatten())
.reshape(df[word_cols].shape)
)
 num_unique_words = word_enc.classes_.shape[0]
return word_enc, num_unique_words, word_cols<string_transform>
lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.00025, random_state = 1234))
lasso.fit(final_X_train, final_y_train)
 lasso_preds = np.expm1(lasso.predict(final_X_test))
sub = pd.DataFrame()
sub['Id'] = test_id
sub['SalePrice'] = lasso_preds
def bert_tokenize(df, col):

 gs_folder_bert = "gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-12_H-768_A-12"
tf.io.gfile.listdir(gs_folder_bert)
 tokenizer = tokenization.FullTokenizer(
 vocab_file=os.path.join(gs_folder_bert, "vocab.txt"),
do_lower_case=True 
)
 bert_token =(lambda x: tokenizer 
.convert_tokens_to_ids(['[CLS]'] + tokenizer.tokenize(x)+ ['[SEP]'])
)
 num_unique_words = len(tokenizer.vocab)
 pre_words_ids = df[col].apply(bert_token)
 pre_masks = df[col].apply(lambda x: [1]*(len(tokenizer.tokenize(x)) +2))
words_ids = tf.keras.preprocessing.sequence.pad_sequences(pre_words_ids)
 masks = tf.keras.preprocessing.sequence.pad_sequences(pre_masks)
 type_ids = np.zeros(words_ids.shape, dtype=np.int)
 word_cols = [f"{text}_{i:02d}" for i in range(words_ids.shape[1])]
mask_cols = [f"mask_{i:02d}"
for i in range(words_ids.shape[1])
 ]
type_cols = [f"type_{i:02d}"
for i in range(words_ids.shape[1])
 ]
df[word_cols] = words_ids
df[mask_cols] = masks
df[type_cols] = type_ids
return tokenizer, num_unique_words, word_cols, mask_cols, type_cols<string_transform>
ridge = make_pipeline(RobustScaler() , Ridge(alpha = 6, random_state = 1234))
ridge.fit(final_X_train, final_y_train)
 ridge_preds = np.expm1(ridge.predict(final_X_test))
sub = pd.DataFrame()
sub['Id'] = test_id
sub['SalePrice'] = ridge_preds
def tf_tokenizer(df, col, num_words):

 tokenizer =(
 tf.keras.preprocessing.text.Tokenizer(num_words=num_words)
)
 tokenizer.fit_on_texts(df[col].values)
 word_ar = tf.keras.preprocessing.sequence.pad_sequences(
 tokenizer.texts_to_sequences(df[col].values)
)
 word_cols = [f"text_{i:02d}" for i in range(word_ar.shape[1])]
df[word_cols] = word_ar
return tokenizer, num_words, word_cols<concatenate>
xg_X_train, xg_X_test, xg_y_train, xg_y_test = train_test_split(final_X_train, final_y_train, test_size=0.33, random_state=1234)
 xg_model = XGBRegressor(n_estimators=100, seed = 1234)
 param_dict = {'max_depth': [3,4,5],
'min_child_weight': [2,3,4],
'learning_rate': [0.05, 0.1,0.15],
'gamma': [0.0, 0.1, 0.2]
}
start = time()
grid_search = GridSearchCV(xg_model, param_dict)
 grid_search.fit(xg_X_train, xg_y_train)
 print("GridSearch took %.2f seconds to complete." %(time() -start))
display(grid_search.best_params_ )
df_full = pd.concat([df_train, df_test], ignore_index=True)
 tokenizer, num_unique_words, word_cols, mask_cols, type_cols = bert_tokenize(df_full, text)
 log.info(f"Vocab size: {num_unique_words}" )<choose_model_class>
xg_model = XGBRegressor(n_estimators = 1000,
learning_rate = 0.1,
max_depth = 4,
min_child_weight = 4,
gamma = 0,
seed = 1234)
 start = time()
xg_model.fit(xg_X_train, xg_y_train)
 xg_preds = xg_model.predict(xg_X_test)
 print("Model took %.2f seconds to complete." %(time() -start))
print("RMSE: %.4f" % sqrt(mean_squared_error(xg_y_test, xg_preds)) )
hub_url_bert = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2"
bert_layer = hub.KerasLayer(hub_url_bert, trainable=True )<choose_model_class>
xg_model = XGBRegressor(n_estimators = 1000,
learning_rate = 0.1,
max_depth = 4,
min_child_weight = 4,
gamma = 0,
seed = 1234)
 xg_model.fit(final_X_train, final_y_train)
 xg_preds = np.expm1(xg_model.predict(final_X_test))
class TwolayerModel(tf.keras.Model):

 def __init__(self,
batch_size=32,
units=40,
embed_dim=100,
sequence_length=len(word_cols), 
):

 super(TwolayerModel, self ).__init__()
self.inps =(None, sequence_length)
 self.bs = batch_size
out_dim = 2
self._embed1 = tf.keras.layers.Embedding(
 num_unique_words,
embed_dim,
input_length=self.inps[1],
name="word_embedding", 
)
 self._lstm1 = tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(
 units,
name="lstm1",
return_sequences=True, 
)
)
 self._lstm2 = tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(
 units,
name="lstm2", 
)
)
 self._dense2 = tf.keras.layers.Dense(
 out_dim,
activation=tf.nn.softmax,
name="final", 
)
 self._optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)
 self._metrics = [tf.keras.metrics.BinaryAccuracy() ]
self._loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 self.compile(
 optimizer=self._optimizer,
loss=self._loss,
metrics=self._metrics, 
)
 def call(self, inputs):
x1 = self._embed1(inputs)
 y1 = self._lstm1(x1)
 y1 = self._lstm2(y1)
 out = self._dense2(y1)
 return out<choose_model_class>
sub = pd.DataFrame()
sub['Id'] = test_id
sub['SalePrice'] = xg_preds
<choose_model_class><EOS> 
lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.00025, random_state = 1234))
lasso.fit(final_X_train, final_y_train)
 lasso_preds = np.expm1(lasso.predict(final_X_test))
ridge = make_pipeline(RobustScaler() , Ridge(alpha = 6, random_state = 1234))
ridge.fit(final_X_train, final_y_train)
 ridge_preds = np.expm1(ridge.predict(final_X_test))
xg_model = XGBRegressor(n_estimators = 1000,
learning_rate = 0.1,
max_depth = 4,
min_child_weight = 4,
gamma = 0,
seed = 1234)
 xg_model.fit(final_X_train, final_y_train)
 xg_preds = np.expm1(xg_model.predict(final_X_test))
weights = [0.5, 0.25, 0.25]
sub = pd.DataFrame()
sub['Id'] = test_id
sub['SalePrice'] =(ridge_preds*weights[0])+(lasso_preds*weights[1])+(xg_preds*weights[2])
 sub.to_csv('Ensemble Submission.csv',index=False )