Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_search_model>
import numpy as np
import os
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
import optuna
from sklearn import linear_model
from sklearn import metrics
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from sklearn import preprocessing
from sklearn.model_selection import cross_val_score
from optuna.samplers import RandomSampler, GridSampler, TPESampler
import sklearn
import xgboost as xgb
from scipy.misc import derivative
from sklearn.metrics import mean_squared_error
import pickle
import category_encoders as ce
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
import random
import lightgbm as lgb
import random
vectorizer = TfidfVectorizer(ngram_range=(1,1), min_df=2, max_features=5000, lowercase=False, stop_words=None)
 scaler = MaxAbsScaler()
classifier = BernoulliNB(binarize=0, alpha=1 )<choose_model_class>
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed )
pl = make_pipeline(vectorizer, classifier )<train_model>
seed_everything(0 )
pl.fit(X_train, y_train )<predict_on_test>
df_train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 df_test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 y_train = np.log(df_train['SalePrice'])
 X_train = df_train.drop(['SalePrice','Id'],axis=1)
 X_test = df_test.drop('Id',axis=1)
 print('Train data: ',X_train.shape)
 print('Test data: ',X_test.shape )
preds_prob = pl.predict_proba(X_test)
 preds_class = pl.predict(X_test )<compute_test_metric>
cat_nominal_features = pickle.load(open('.. /input/features-housing/cat_nominal_features.p', "rb"))
cat_ordinal_features = pickle.load(open('.. /input/features-housing/cat_ordinal_features.p', "rb"))
num_features = pickle.load(open('.. /input/features-housing/num_features.p', "rb"))
cat_features = cat_nominal_features + cat_ordinal_features
print('Number of numeric features: ',len(num_features))
print('Number of ordinal features: ',len(cat_ordinal_features))
print('Number of nominal featuures: ',len(cat_nominal_features))
nb_loss = log_loss(y_test, preds_prob)
 print(nb_loss )<compute_test_metric>
X_train[cat_ordinal_features] = X_train[cat_ordinal_features].fillna(np.nan)
 X_train[cat_nominal_features] = X_train[cat_nominal_features].fillna(np.nan)
 X_test[cat_ordinal_features] = X_test[cat_ordinal_features].fillna(np.nan)
 X_test[cat_nominal_features] = X_test[cat_nominal_features].fillna(np.nan )
print(confusion_matrix(y_test, preds_class))
print('
')
 print(classification_report(y_test, preds_class))<install_modules>
ord_mapping=[{'col': 'Street', 'mapping': {'Grvl': 1, 'Pave': 2}},
{'col': 'Alley', 'mapping': {np.nan:0,'Grvl': 1, 'Pave': 2}},
{'col': 'Utilities', 'mapping': {'NoSeWa': 1, 'AllPub':2}},
{'col': 'ExterQual', 'mapping': {'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'ExterCond', 'mapping': {'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'BsmtCond', 'mapping': {np.nan:0,'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'BsmtQual', 'mapping': {np.nan:0,'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'BsmtExposure', 'mapping': {np.nan:0,'No':1,'Mn':2,'Av':3,
'Gd':4}},
{'col': 'BsmtFinType1', 'mapping': {np.nan:0,'Unf':1,'LwQ':2,'Rec':3,
'BLQ':4,'ALQ':5,'GLQ':6}},
{'col': 'BsmtFinType2', 'mapping': {np.nan:0,'Unf':1,'LwQ':2,'Rec':3,
'BLQ':4,'ALQ':5,'GLQ':6}},
{'col': 'HeatingQC', 'mapping': {'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'CentralAir', 'mapping': {'Y':1,'N':0}},
{'col': 'KitchenQual', 'mapping': {'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'Functional', 'mapping': {'Typ':8,'Min1':7,'Min2':6,
'Mod':5,'Maj1':4,'Maj2':3,
'Sev':2,"Sal":1}},
{'col': 'FireplaceQu', 'mapping': {np.nan:0,'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'GarageFinish', 'mapping': {np.nan:0,'Unf':1,'RFn':2,'Fin':3}},
{'col': 'GarageQual', 'mapping': {np.nan:0,'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'GarageCond', 'mapping': {np.nan:0,'Po':1,'Fa':2,'TA':3,
'Gd':4,'Ex':5}},
{'col': 'PavedDrive', 'mapping': {'N':1,'P':2,'Y':3}},
{'col': 'PoolQC', 'mapping': {np.nan:0,'Fa':1,'TA':2,
'Gd':3,'Ex':4}},
{'col': 'Fence', 'mapping': {np.nan:0,'MnWw':1,'GdWo':2,'MnPrv':3,
'GdPrv':4}}]
mapping_cols = ['Street','Alley','Utilities','ExterQual','ExterCond','BsmtCond','BsmtQual','BsmtExposure',
'BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir','KitchenQual',
'Functional','FireplaceQu','GarageFinish','GarageQual','GarageCond','PavedDrive',
'PoolQC','Fence']
def construct_ord_nom(ordinal_features,nominal_features):
mapp = []
ord_features = []
for c in ordinal_features:
if c not in mapping_cols:
continue
idx = mapping_cols.index(c)
 mapp.append(ord_mapping[idx])
 ord_features.append(c)
 ce_ord = ce.OrdinalEncoder(cols=ord_features,mapping=mapp,
handle_unknown='return_nan',handle_missing='return_nan')
 ce_nom = ce.OneHotEncoder(cols=nominal_features,handle_unknown='return_nan',handle_missing='return_nan')
 return ce_ord,ce_nom
def get_CT(ord_features,nom_features,num_features_new,ce_ord,ce_nom):
numeric_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='constant',fill_value=-1)) ,
])
 ce_ord, ce_nom = construct_ord_nom(ord_features,nom_features)
 ct1 = ColumnTransformer(
 transformers=[ 
('nominal',ce_nom,nom_features), 
('ordinal',ce_ord,ord_features), 
('num',numeric_transformer,num_features_new)
 ],remainder = 'passthrough')
 return ct1
!pip install tensorflow_text==2.3<import_modules>
ce_ord, ce_nom = construct_ord_nom(cat_ordinal_features,cat_nominal_features)
 CT = get_CT(cat_ordinal_features,cat_nominal_features,num_features,ce_ord,ce_nom )
import tensorflow
import tensorflow_hub as hub
import tensorflow_text as text<define_variables>
X_train_new = CT.fit_transform(X_train)
 X_test_new = CT.transform(X_test)
 print(X_train_new.shape)
 print(X_test_new.shape)
 dtrain = lgb.Dataset(X_train_new,y_train )
URL_PREPROCESSOR = "http://tfhub.dev/tensorflow/albert_en_preprocess/2"
URL_ENCODER = "https://tfhub.dev/tensorflow/albert_en_base/2"<categorify>
class cparams() :
def __init__(self):
self.seed = 0
self.num_iterations = 100
self.learning_rate = 0.1
self.num_leaves = 31
self.min_child_samples = 20
self.min_child_weight = 0.001
self.bagging_fraction = 1.0
self.feature_fraction = 1.0
self.bagging_freq = 0
self.alpha = 0.25
self.gamma = 2.0
self.l1 = 0.0
self.l2 = 0.0
def calibrate(self,num_round):
param = {'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'learning_rate': self.learning_rate,
'num_leaves': self.num_leaves,
'min_data_in_leaf': self.min_child_samples,
'min_sum_hessian_in_leaf':self.min_child_weight,
'bagging_fraction': self.bagging_fraction,
'bagging_freq': self.bagging_freq,
'feature_fraction': self.feature_fraction,
'lambda_l1': self.l1,
'lambda_l2': self.l2,
'seed': self.seed
}
bst = lgb.cv(param, dtrain,num_boost_round=num_round, stratified=False, \
shuffle=True,early_stopping_rounds=100,verbose_eval=10,seed=0)
 return bst
def get_param(self):
param = {'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'learning_rate': self.learning_rate,
'num_leaves': self.num_leaves,
'min_data_in_leaf': self.min_child_samples,
'min_sum_hessian_in_leaf':self.min_child_weight,
'bagging_fraction': self.bagging_fraction,
'bagging_freq': self.bagging_freq,
'feature_fraction': self.feature_fraction,
'lambda_l1': self.l1,
'lambda_l2': self.l2,
'seed': self.seed
}
return param
def build_classifier_model() :
text_input = tensorflow.keras.layers.Input(shape=() , dtype=tensorflow.string, name='text')
 preprocessing_layer = hub.KerasLayer(URL_PREPROCESSOR, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(URL_ENCODER, trainable=True, name='ALBERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tensorflow.keras.layers.Dropout(0.50 )(net)
 net = tensorflow.keras.layers.Dense(3, activation='softmax', name='classifier_b' )(net)
 return tensorflow.keras.Model(text_input, net )<choose_model_class>
current_model = cparams()
classifier_model = build_classifier_model()<install_modules>
params = {
'objective': 'regression',
'metric': 'rmse',
"verbosity": 1,
"boosting_type": "gbdt",
'seed':0
}
eval_history = lgb.cv(
 params, dtrain, verbose_eval=20,
stratified=False, num_boost_round=1000, early_stopping_rounds=100,
nfold=5,seed=0 )
!pip install -q tf-models-official<import_modules>
print('Best score: ', eval_history['rmse-mean'][-1])
 print('Number of estimators: ', len(eval_history['rmse-mean']))
current_model.num_iterations = len(eval_history['rmse-mean'] )
from official.nlp import optimization<split>
study_name2 = 'lgb_leaves'
study_leaves = optuna.create_study(study_name=study_name2,direction='maximize',sampler=TPESampler(0))
train_tf, test_tf = train_test_split(df, test_size=0.30, stratify=df['author'], random_state=1234)
 print(train_tf.shape, test_tf.shape )<create_dataframe>
def opt_leaves(trial):
params = {
'objective': 'regression',
'metric': 'rmse',
"verbosity": 1,
"boosting_type": "gbdt",
'seed':0,
'num_leaves':int(trial.suggest_loguniform("num_leaves", 3,32))
}
score = lgb.cv(
 params, dtrain, verbose_eval=0,
stratified=False, num_boost_round=current_model.num_iterations,
nfold=5,seed=0)
 return -score['rmse-mean'][-1]
def df_to_dataset(dataframe, batch_size=32):
dataframe = dataframe.copy()
labels = dataframe.pop('author')
 ds = tensorflow.data.Dataset.from_tensor_slices(( dict(dataframe), labels))
ds = ds.batch(batch_size)
 return ds<create_dataframe>
study_leaves.optimize(opt_leaves, n_trials=50 )
train_ds = df_to_dataset(train_tf, batch_size=32 )<create_dataframe>
print('Total number of trials: ',len(study_leaves.trials))
trial_leaves = study_leaves.best_trial
print('Best score : {}'.format(-trial_leaves.value))
for key, value in trial_leaves.params.items() :
print(" {}: {}".format(key, value))
test_ds = df_to_dataset(test_tf, batch_size=32 )<choose_model_class>
current_model.num_leaves = int(trial_leaves.params['num_leaves'] )
epochs = 5
steps_per_epoch = tensorflow.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * epochs
num_warmup_steps = int(0.1*num_train_steps)
 init_lr = 3e-5
optimizer = optimization.create_optimizer(init_lr=init_lr,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
optimizer_type='adamw' )<choose_model_class>
study_name3 = 'lgb_child_weight_sample'
study_sample_weight = optuna.create_study(study_name=study_name3,direction='maximize',sampler=TPESampler(0))
classifier_model.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] )<train_model>
def opt_sample_weight(trial):
params = {
'objective': 'regression',
'metric': 'rmse',
"verbosity": 1,
"boosting_type": "gbdt",
'seed':0,
'num_leaves':current_model.num_leaves,
'min_data_in_leaf':int(trial.suggest_discrete_uniform('data_in_leaf',4,32,q=2)) ,
'min_sum_hessian_in_leaf':trial.suggest_discrete_uniform('min_hessian',0.001,0.003,q=0.0005)
 }
score = lgb.cv(
 params, dtrain, verbose_eval=0,
stratified=False, num_boost_round=current_model.num_iterations,
nfold=5,seed=0)
 return -score['rmse-mean'][-1]
EPOCHS=3
with tensorflow.device('/device:GPU:0'):
history = classifier_model.fit(x=train_ds, validation_data=test_ds, batch_size=32, epochs=EPOCHS )<compute_train_metric>
study_sample_weight.optimize(opt_sample_weight, n_trials=50 )
classifier_model.evaluate(train_ds, batch_size=32 )<compute_test_metric>
print('Total number of trials: ',len(study_sample_weight.trials))
trial_sample_weight = study_sample_weight.best_trial
print('Best score : {}'.format(-trial_sample_weight.value))
for key, value in trial_sample_weight.params.items() :
print(" {}: {}".format(key, value))
classifier_model.evaluate(test_ds, batch_size=32 )<predict_on_test>
current_model.min_child_samples = int(trial_sample_weight.params['data_in_leaf'])
 current_model.min_child_weight = trial_sample_weight.params['min_hessian']
tf_preds = classifier_model.predict(test_ds )<predict_on_test>
study_name4 = 'lgb_bagging'
study_bagging = optuna.create_study(study_name=study_name4,direction='maximize',sampler=TPESampler(0))
classifier_model.predict(X_test)[0:5]<compute_test_metric>
def opt_bagging(trial):
params = {
'objective': 'regression',
'metric': 'rmse',
"verbosity": 1,
"boosting_type": "gbdt",
'seed':0,
'num_leaves':current_model.num_leaves,
'min_data_in_leaf':current_model.min_child_samples,
'min_sum_hessian_in_leaf':current_model.min_child_weight,
'bagging_fraction': trial.suggest_discrete_uniform('bfrac',0.4,1.0,q=0.05),
'bagging_freq': int(trial.suggest_discrete_uniform('bfreq',1,7,q=1.0)) ,
'feature_fraction':trial.suggest_discrete_uniform('feature',0.4,1.0,q=0.05)
 }
score = lgb.cv(
 params, dtrain, verbose_eval=0,
stratified=False, num_boost_round=current_model.num_iterations,
nfold=5,seed=0)
 return -score['rmse-mean'][-1]
albert_loss = log_loss(y_test, tf_preds)
 print(albert_loss )<train_model>
study_bagging.optimize(opt_bagging, n_trials=50 )
diff_loss = nb_loss - albert_loss
if albert_loss < nb_loss:
print("ALBERT improved the loss metric by {}".format(diff_loss))
else: print("loss metric was not improved by ALBERT compared to the base NB model" )<prepare_output>
print('Total number of trials: ',len(study_bagging.trials))
trial_bagging = study_bagging.best_trial
print('Best score : {}'.format(trial_bagging.value))
for key, value in trial_bagging.params.items() :
print(" {}: {}".format(key, value))
tf_preds_class = np.argmax(tf_preds, axis=1 )<compute_test_metric>
current_model.bagging_fraction = trial_bagging.params['bfrac']
current_model.bagging_freq = int(trial_bagging.params['bfreq'])
 current_model.feature_fraction = trial_bagging.params['feature']
print(confusion_matrix(y_test, tf_preds_class))
print('
')
 print(classification_report(y_test, tf_preds_class))<load_from_csv>
study_name5 = 'l1_l2'
study_reg = optuna.create_study(study_name=study_name5,direction='maximize',sampler=TPESampler(0))
dep = pd.read_csv('/kaggle/input/spooky-author-identification/test.zip' )<predict_on_test>
def opt_reg(trial):
params = {
'objective': 'regression',
'metric': 'rmse',
"verbosity": 1,
"boosting_type": "gbdt",
'seed':0,
'num_leaves':current_model.num_leaves,
'min_data_in_leaf':current_model.min_child_samples,
'min_sum_hessian_in_leaf':current_model.min_child_weight,
'bagging_fraction': current_model.bagging_fraction,
'bagging_freq': current_model.bagging_freq,
'feature_fraction':current_model.feature_fraction,
'lambda_l1': trial.suggest_loguniform("lambda_l1", 1e-7, 10),
'lambda_l2': trial.suggest_loguniform("lambda_l2", 1e-7, 10)
 }
score = lgb.cv(
 params, dtrain, verbose_eval=0,
stratified=False, num_boost_round=current_model.num_iterations,
nfold=5,seed=0)
 return -score['rmse-mean'][-1]
dep_preds = classifier_model.predict(dep['text'].values)
 dep_df = pd.DataFrame(data=dep_preds, columns=['EAP','MWS','HPL'] )<save_to_csv>
study_reg.optimize(opt_reg, n_trials=50 )
submit_df = pd.concat([dep,dep_df], axis='columns')
 submit_df.drop(columns='text', inplace=True)
 submit_df.to_csv('submit.csv', index=False, index_label=False )<load_from_url>
print('Total number of trials: ',len(study_reg.trials))
trial_reg = study_reg.best_trial
print('Best score : {}'.format(trial_reg.value))
for key, value in trial_reg.params.items() :
print(" {}: {}".format(key, value))
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<set_options>
current_model.l1 = trial_reg.params['lambda_l1']
current_model.l2 = trial_reg.params['lambda_l2']
sns.set_style("whitegrid")
 notebookstart = time.time()
pd.options.display.max_colwidth = 500
print("Tensorflow Version: ", tf.__version__)
 print("TF-Hub version: ", hub.__version__)
 print("Eager mode enabled: ", tf.executing_eagerly())
 print("GPU available: ", tf.test.is_gpu_available() )<categorify>
current_model.learning_rate = 0.05
lr1 = current_model.calibrate(10000)
 print('Best score: ', lr1['rmse-mean'][-1])
 print('Number of estimators: ', len(lr1['rmse-mean']))
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
current_model.learning_rate = 0.01
lr2 = current_model.calibrate(10000)
 print('Best score: ', lr2['rmse-mean'][-1])
 print('Number of estimators: ', len(lr2['rmse-mean']))
def build_model(bert_layer, out_channels, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(out_channels, activation='softmax' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='categorical_crossentropy',
metrics=['accuracy'])
 return model<choose_model_class>
current_model.learning_rate = 0.005
lr3 = current_model.calibrate(10000)
 print('Best score: ', lr3['rmse-mean'][-1])
 print('Number of estimators: ', len(lr3['rmse-mean']))
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<define_variables>
current_model.learning_rate = 0.005
current_param = current_model.get_param()
for key, value in current_param.items() :
print(" {}: {}".format(key, value))
MAX_LEN = 64*3
BATCH_SIZE = 8
EPOCHS = 15
SEED = 42
NROWS = None
TEXTCOL = "text"
TARGETCOL = "author"
NCLASS = 3<load_from_csv>
def cv_training(train_data,y_train_data):
kFold = KFold(n_splits=5, random_state=0, shuffle=True)
 models = []
eval_history = []
oof_pred = []
oof_target = []
scores = []
for fold,(trn_idx, val_idx)in enumerate(kFold.split(train_data)) :
X_train = train_data[trn_idx]
X_val = train_data[val_idx]
y_train = y_train_data[trn_idx]
y_val = y_train_data[val_idx]
dtrain = lgb.Dataset(X_train,y_train)
 dval = lgb.Dataset(X_val,y_val)
 evals_result = {}
model = lgb.train(current_param, dtrain,num_boost_round=5000,
evals_result=evals_result,valid_sets=dval,verbose_eval=20,early_stopping_rounds=100)
 models.append(model)
 y_pred = model.predict(X_val)
 score_temp = np.sqrt(mean_squared_error(y_pred,y_val))
scores.append(score_temp)
 oof_pred.append(y_pred)
 oof_target.append(y_val)
 eval_history.append(evals_result)
 oof_pred = np.concatenate(( oof_pred[0],oof_pred[1],oof_pred[2],
oof_pred[3],oof_pred[4]))
oof_target = np.concatenate(( oof_target[0],oof_target[1],oof_target[2],
oof_target[3],oof_target[4]))
oof_df = pd.DataFrame({'predictions':oof_pred,'target':oof_target})
 return models, eval_history, scores, oof_df
train = pd.read_csv(".. /input/spooky-author-identification/train.zip")
 test = pd.read_csv(".. /input/spooky-author-identification/test.zip")
 testdex = test.id
submission = pd.read_csv(".. /input/spooky-author-identification/sample_submission.zip")
 sub_cols = submission.columns
print("Train Shape: {} Rows, {} Columns".format(*train.shape))
print("Test Shape: {} Rows, {} Columns".format(*test.shape))
length_info = [len(x)for x in np.concatenate([train[TEXTCOL].values, test[TEXTCOL].values])]
print("Train Sequence Length - Mean {:.1f} +/- {:.1f}, Max {:.1f}, Min {:.1f}".format(
 np.mean(length_info), np.std(length_info), np.max(length_info), np.min(length_info)) )<feature_engineering>
models, eval_history,scores,oof_df = cv_training(X_train_new,y_train )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
oof_df.to_csv('oof_df.csv',index=False)
 oof_df.head()
train_input = bert_encode(train[TEXTCOL].values, tokenizer, max_len=MAX_LEN)
 test_input = bert_encode(test[TEXTCOL].values, tokenizer, max_len=MAX_LEN)
 label_mapper = {name: i for i,name in enumerate(set(train[TARGETCOL].values)) }
num_label = np.vectorize(label_mapper.get )(train[TARGETCOL].values)
 train_labels = to_categorical(num_label )<choose_model_class>
score = 0.0
for i in range(len(models)) :
print(models[i].best_iteration)
 print(eval_history[i]['valid_0']['rmse'][-1])
 score = score + eval_history[i]['valid_0']['rmse'][-1]
print('Average score: ',score/5 )
model = build_model(bert_layer, NCLASS, max_len=MAX_LEN)
 model.summary()<train_model>
model_final = lgb.train(current_param, dtrain,num_boost_round=3920)
 ypred = model_final.predict(X_test_new )
checkpoint = callbacks.ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001,
patience=4, verbose=1, mode='min', baseline=None,
restore_best_weights=False)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=EPOCHS,
callbacks=[checkpoint, es],
batch_size=BATCH_SIZE
)<import_modules>
sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission = pd.DataFrame({
'Id': sub['Id'],
'SalePrice': np.exp(ypred)
 })
 submission.head()
<predict_on_test><EOS> 
submission.to_csv('submission.csv',index=False )