Kaggle competition: House Prices - Advanced Regression Techniques
sub = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 sub<define_variables>
def label_encoding(train: pd.DataFrame, test: pd.DataFrame, col_definition: dict):

 n_train = len(train)
 train = pd.concat([train, test], sort=False ).reset_index(drop=True)
 for f in col_definition['encode_col']:
try:
lbl = preprocessing.LabelEncoder()
train[f] = lbl.fit_transform(list(train[f].values))
except:
print(f)
 test = train[n_train:].reset_index(drop=True)
 train = train[:n_train]
return train, test
model_name = 'jplu/tf-xlm-roberta-large'
epochs = 4
maxlen = 80
AUTO = tf.data.experimental.AUTOTUNE<load_pretrained>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 sub = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )
tokenizer = AutoTokenizer.from_pretrained(model_name )<categorify>
unique_cols = list(train.columns[train.nunique() == 1])
 print(unique_cols )
def get_training_dataset(idx, df = train, is_train = True):
text = df[['premise', 'hypothesis']].values[idx].tolist()
text_enc = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = True,
max_length = maxlen 
)
 dataset = tf.data.Dataset.from_tensor_slices(( text_enc['input_ids'], df['label'][idx].values))
dataset = dataset.repeat()
dataset = dataset.shuffle(2020)
 dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_valid_dataset(idx, df = train, is_train = False):
text = df[['premise', 'hypothesis']].values[idx].tolist()
text_enc = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = True,
max_length = maxlen 
)
 dataset = tf.data.Dataset.from_tensor_slices(( text_enc['input_ids'], df['label'][idx].values))
dataset = dataset.batch(batch_size)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(df = test, is_train = False):
text = df[['premise', 'hypothesis']].values.tolist()
text_enc = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = True,
max_length = maxlen 
)
 dataset = tf.data.Dataset.from_tensor_slices(text_enc['input_ids'])
 dataset = dataset.batch(batch_size)
 return dataset<choose_model_class>
duplicated_cols = list(train.columns[train.T.duplicated() ])
 print(duplicated_cols )
def build_model(maxlen, model_name):
with strategy.scope() :
base_model = TFAutoModel.from_pretrained(model_name)
 input_word_ids = tf.keras.Input(shape =(maxlen,), dtype = tf.int32, name = "input_word_ids")
 embedding = base_model(input_word_ids)[0]
out_tokens = embedding[:, 0, :]
output = tf.keras.layers.Dense(3, activation = 'softmax' )(out_tokens)
 model = tf.keras.Model(inputs = input_word_ids, outputs = output)
 model.compile(tf.keras.optimizers.Adam(lr = 1e-5),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 return model<train_model>
train, test = label_encoding(train, test, col_definition={'encode_col':categorical_cols} )
folds = 3
kf = KFold(n_splits = folds, shuffle = True, random_state = 777)
 models = []
histories = []
predictions = np.zeros(( test.shape[0], 3))
for fold,(trn_idx, val_idx)in enumerate(kf.split(np.arange(train['label'].shape[0]))):
print('
')
 print('-'*50)
 print(f'Training fold {fold + 1}')
 train_dataset = get_training_dataset(trn_idx, df = train, is_train = True)
 valid_dataset = get_valid_dataset(val_idx, df = train, is_train = False)
 K.clear_session()
model = build_model(maxlen, model_name)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(
 'XLM-R_fold-%i.h5'%fold, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'min', save_freq = 'epoch' 
)
 print('Model Training.....')
 STEPS_PER_EPOCH = len(trn_idx)// batch_size
history = model.fit(
 train_dataset, epochs = epochs, verbose = 1,
steps_per_epoch = STEPS_PER_EPOCH,
batch_size = batch_size,
validation_data = valid_dataset 
)
 display_training_curves(
 history.history['loss'],
history.history['val_loss'],
'loss', 311 
)
 display_training_curves(
 history.history['accuracy'],
history.history['val_accuracy'],
'accuracy', 312 
)
 histories.append(history)
 models.append(model)
 print('Prediting on test data.. ')
 test_dataset = get_test_dataset(test, is_train = False)
 pred = model.predict(test_dataset, verbose = 1)
 predictions += pred / folds
del history, train_dataset, valid_dataset, model
gc.collect()
print('
')
 print('-'*50 )<prepare_output>
X_train = train.drop(['Id', 'SalePrice'], axis=1)
 y_train = np.log1p(train['SalePrice'])
 X_test = test.drop(['Id', 'SalePrice'], axis=1 )
sub['prediction'] = np.argmax(predictions, axis = 1 )<save_to_csv>
y_preds_lgb = []
models_lgb = []
oof_train_lgb = np.zeros(( len(X_train),))
y_preds_xgb = []
models_xgb = []
oof_train_xgb = np.zeros(( len(X_train),))
y_preds = []
oof_train = np.zeros(( len(X_train),))
cv = KFold(n_splits=5, shuffle=True, random_state=0)
 params_lgb = {
'num_leaves': 29,
'max_depth': 6,
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.05
}
params_xgb = {
'objective': 'reg:squarederror',
'eval_metric': 'rmse'
}
for fold_id,(train_index, valid_index)in enumerate(cv.split(X_train)) :
X_tr = X_train.loc[train_index, :]
X_val = X_train.loc[valid_index, :]
y_tr = y_train[train_index]
y_val = y_train[valid_index]
lgb_train = lgb.Dataset(X_tr,
y_tr,
categorical_feature=categorical_cols)
 lgb_eval = lgb.Dataset(X_val,
y_val,
reference=lgb_train,
categorical_feature=categorical_cols)
 model_lgb = lgb.train(params_lgb,
lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=10,
num_boost_round=1000,
early_stopping_rounds=10)
 oof_train_lgb[valid_index] = model_lgb.predict(X_val,
num_iteration=model_lgb.best_iteration)
 y_pred_lgb = model_lgb.predict(X_test,
num_iteration=model_lgb.best_iteration)
 y_preds_lgb.append(y_pred_lgb)
 models_lgb.append(model_lgb)
 dtrain = xgb.DMatrix(X_tr, label=y_tr)
 dvalid = xgb.DMatrix(X_val, label=y_val)
 dtest = xgb.DMatrix(X_test)
 evals = [(dtrain, 'train'),(dvalid, 'eval')]
model_xgb = xgb.train(params_xgb,
dtrain,
num_boost_round=1000,
early_stopping_rounds=10,
evals=evals 
)
 oof_train_xgb[valid_index] = model_xgb.predict(dvalid)
 y_pred_xgb = model_xgb.predict(dtest, ntree_limit=model_xgb.best_ntree_limit)
 y_preds_xgb.append(y_pred_xgb)
 models_xgb.append(model_xgb)
 y_preds_ensemble =(y_pred_lgb + y_pred_xgb)/ 2
y_preds.append(y_preds_ensemble)
 oof_train_ensemble =(oof_train_lgb[valid_index] + oof_train_xgb[valid_index])/ 2
oof_train[valid_index] = oof_train_ensemble
sub.to_csv('./submission.csv', index = False)
 sub<import_modules>
print(f'CV: {np.sqrt(mean_squared_error(y_train, oof_train)) }' )
import math, re, os
import numpy as np
import tensorflow as tf<import_modules>
y_sub = sum(y_preds)/ len(y_preds)
 y_sub = np.expm1(y_sub)
 sub['SalePrice'] = y_sub
sub.head()
print(tf.__version__ )<find_best_params>
sub_elastic = pd.read_csv('.. /input/houseprice-elasticnet-finish/submission.csv')
 sub_elastic.head()
<define_variables><EOS> 
result =(sub.SalePrice + sub_elastic.SalePrice)/ 2
sub['SalePrice'] = result
sub.to_csv('submission_3_ensemble.csv', index=False)
 sub.head()