Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
import numpy as np
import pandas as pd
from sklearn import preprocessing
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
import sklearn.metrics as metrics
import math
import seaborn as sns
from sklearn.ensemble import RandomForestRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
n_steps = len(x_twitter_train1)// batch_size
n_steps_val = len(x_valid)// batch_size
model.fit(
 twitter_dataset_ssl1,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
validation_steps=n_steps_val,
epochs=2 
)
 test_preds2 = model.predict(test_dataset, verbose=1 )<train_model>
train_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
n_steps = len(x_twitter_train2)// batch_size
n_steps_val = len(x_valid)// batch_size
model.fit(
 twitter_dataset_ssl2,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
validation_steps=n_steps_val,
epochs=2 
)
 test_preds3 = model.predict(test_dataset, verbose=1 )<prepare_output>
labels = train_df['SalePrice']
labels
test_preds =(0.92)*test_preds1 +(0.05)*test_preds2 +(0.03)*test_preds3
test_preds = test_preds.argmax(axis=1)
 submission['prediction'] = test_preds<save_to_csv>
train_df.drop(['SalePrice'], axis=1, inplace=True )
submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
train_idx = train_df.shape[0]
train_idx
os.environ["WANDB_API_KEY"] = "0"
%matplotlib inline
print(f"Using Tensorflow version: {tf.__version__}")
 tfa.register_all(custom_kernels=False)
 !pip install -q transformers==3.1.0
print(f"Using Transformers version: {transformers.__version__}")
<train_on_grid>
new_df = pd.concat([train_df, test_df], ignore_index=True, axis=0 )
DEVICE = 'TPU'
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else: DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print('REPLICAS: {}'.format(REPLICAS)) 
<define_variables>
nan_pct = [new_df[col].isna().sum() * 100/new_df.shape[0] for col in new_df.columns]
nan_df = pd.DataFrame(columns=['column', 'percentage'])
 nan_df['column'] = new_df.columns
nan_df['percentage'] = nan_pct
nan_df[nan_df['percentage'] > 50]
SEED = 777
DATA_PATH = ".. /input/contradictory-my-dear-watson/"
MODEL = 'jplu/tf-xlm-roberta-large'
TOKENIZER = transformers.AutoTokenizer.from_pretrained(MODEL)
 MAX_LEN = 96
BATCH_SIZE = 16 * REPLICAS
EPOCHS= 10
FOLDS = 5
NUM_CLASSES = 3
OPT_TYPE = "Adam"
LR = 1e-5 
<load_from_csv>
new_df.drop(nan_df[nan_df['percentage'] > 50]['column'].unique() , axis=1, inplace=True )
train = pd.read_csv(DATA_PATH+"train.csv")
 test = pd.read_csv(DATA_PATH+"test.csv" )<feature_engineering>
nan_df[(nan_df['percentage'] < 50)&(nan_df['percentage'] > 5)]
train.loc[:, 'lang_label'] = train.loc[:, ['lang_abv', 'label']].apply(lambda row: row['lang_abv']+'-'+str(row['label']), axis=1 )<choose_model_class>
new_df.drop(['FireplaceQu'], axis=1, inplace=True )
out = TOKENIZER(train.loc[2,'premise'],train.loc[2,'hypothesis'],add_special_tokens=True, max_length=40, padding='max_length',truncation = True,return_token_type_ids=True)
 out<categorify>
new_df[obj_cols] = new_df[obj_cols].fillna('None' )
def preprocess_input_data(idx, tokenizer, label = False, dataset = 'train',max_len = MAX_LEN):
if dataset == 'train':
enc = TOKENIZER(train.loc[idx,['premise', 'hypothesis']].values.tolist() , max_length=max_len, padding= 'max_length', add_special_tokens=True,truncation = True,return_token_type_ids=True)
 elif dataset == 'test':
enc = TOKENIZER(test.loc[idx,['premise', 'hypothesis']].values.tolist() , max_length=max_len, padding= 'max_length', add_special_tokens=True,truncation = True,return_token_type_ids=True)
 attention_mask = enc.attention_mask
input_ids = enc.input_ids
token_type_ids = enc.token_type_ids
input_tokens = [tokenizer.convert_ids_to_tokens(enc.input_ids[i])for i in range(len(enc.input_ids)) ]
output_dict = {'token_type_ids': np.array(token_type_ids ).astype('int32'),
'input_ids': np.array(input_ids ).astype('int32'),
'input_tokens': input_tokens,
'attention_mask': np.array(attention_mask ).astype('int32'),
}
if label:
output_dict['labels'] = to_categorical(train.loc[idx, 'label'], num_classes = NUM_CLASSES ).astype('int32')
 return output_dict
processed_dict = preprocess_input_data(np.arange(train.shape[0]), tokenizer=TOKENIZER, label=True, dataset = 'train',max_len = MAX_LEN)
 input_ids, attention_mask, token_type_ids,labels = processed_dict['input_ids'],processed_dict['attention_mask'], processed_dict['token_type_ids'], processed_dict['labels']
processed_dict = preprocess_input_data(np.arange(test.shape[0]), tokenizer=TOKENIZER, label=False, dataset = 'test',max_len = MAX_LEN)
 input_ids_test, attention_mask_test, token_type_ids_test = processed_dict['input_ids'],processed_dict['attention_mask'], processed_dict['token_type_ids']<create_dataframe>
for col in float_cols:
new_df[col] = new_df[col].fillna(new_df[col].mode().iloc[0] )
def data_loader(idx,data= 'train', labelled=True, return_ids=False, repeat=True, shuffle=True,cache=True, batch_size = BATCH_SIZE):
if data == 'train':
if labelled:
dataset = tf.data.Dataset.from_tensor_slices(((input_ids[idx,],attention_mask[idx,],token_type_ids[idx,]),
labels[idx,]))
else:
dataset = tf.data.Dataset.from_tensor_slices(((input_ids[idx,],attention_mask[idx,],token_type_ids[idx,]),))
if return_ids:
dataset = tf.data.Dataset.from_tensor_slices(((input_ids[idx,],attention_mask[idx,],token_type_ids[idx,], train.loc[idx,'id'].values),))
elif data == 'test':
dataset = tf.data.Dataset.from_tensor_slices(((input_ids_test,attention_mask_test,token_type_ids_test),))
if cache:
dataset = dataset.cache()
if shuffle:
dataset = dataset.shuffle(2048)
 if repeat:
dataset = dataset.repeat()
dataset = dataset.batch(batch_size ).prefetch(AUTO)
 return dataset
for out in data_loader(np.arange(10),batch_size=5 ).unbatch().take(1):
print(out[0], out[1] )<init_hyperparams>
new_df_1 = pd.get_dummies(new_df )
def get_lr_callback(PLOT_LR = False):
lr_start = 1e-5
lr_max = 1.5e-5
lr_min = 1e-5
lr_ramp_ep = 3
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
if PLOT_LR:
plt.figure(figsize=(15, 5))
plt.subplot(1, 2, 1)
 plt.plot([lrfn(e)for e in range(EPOCHS)]);
plt.xlabel('Epoch'); plt.ylabel('LR');
plt.subplot(1, 2, 2);
plt.plot([lrfn(e)for e in range(EPOCHS)]);
plt.xlabel('Epoch'); plt.ylabel('Log LR');
plt.yscale('log');
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
get_lr_callback(PLOT_LR = True)
 def display_training_curves(training, validation, title, subplot):
if subplot%10==1:
plt.subplots(figsize=(10,10), facecolor='
plt.tight_layout()
ax = plt.subplot(subplot)
 ax.set_facecolor('
ax.plot(training)
 ax.plot(validation)
 ax.set_title('model '+ title)
 ax.set_ylabel(title)
 ax.set_xlabel('epoch')
 ax.legend(['train', 'valid.'])
 def build_model(opt = OPT_TYPE):
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 transformer_embedding = transformers.TFAutoModel.from_pretrained(MODEL)
 x = transformer_embedding({'input_ids':ids,'attention_mask': att, 'token_type_ids':tok})[0][:,0,:]
x = tf.keras.layers.Dropout(0.3 )(x)
 x = tf.keras.layers.Dense(128, activation="relu" )(x)
 x = tf.keras.layers.Dense(3 )(x)
 x = tf.keras.layers.Activation('softmax' )(x)
 model = tf.keras.Model(inputs = [ids,att,tok], outputs = x)
 if opt == 'Adam':
optimizer = tf.keras.optimizers.Adam(learning_rate=LR)
 elif OPT_TYPE == "RAdam":
optimizer = tfa.optimizers.RectifiedAdam(lr=LR)
 loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True)
 model.compile(optimizer = optimizer, loss=loss_fn, metrics = ['accuracy'])
 return model<normalization>
train_df_1 = new_df_1[:train_idx]
test_df_1 = new_df_1[train_idx:]
def memory() :
pid = os.getpid()
py = psutil.Process(pid)
 memoryUse = py.memory_info() [0]/2.**30
print('memory use:', memoryUse )<train_model>
rf = RandomForestRegressor(max_depth=40, min_samples_leaf=3, min_samples_split=8,
n_estimators=5000, random_state=17 )
display_training_curve=True
oof_preds = np.zeros(shape =(train.shape[0],3))
ypreds_test = np.zeros(shape =(test.shape[0],3))
val_ids = []
print(f'DEVICE: {DEVICE}')
 skf = StratifiedKFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(train,train.lang_label.values)) :
print('
print(f'
print('
print(f"Training on {len(idxT)} examples with batch size = {BATCH_SIZE}, validate on {len(idxV)} examples")
 if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 memory()
train_dataset = data_loader(idxT,labelled=True,repeat=True, shuffle=True)
 valid_dataset = data_loader(idxV,labelled=True,repeat=False, shuffle=False, cache=False)
 K.clear_session()
with strategy.scope() :
model = build_model()
mod_checkpoint = tf.keras.callbacks.ModelCheckpoint("fold{}.h5".format(fold+1), monitor="val_accuracy",
verbose=1, save_best_only=True,
save_weights_only=True, mode='max', save_freq='epoch')
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', factor=0.5,verbose=1,mode='max',
patience=2, min_lr=5e-6)
 history = model.fit(train_dataset, epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=1,
steps_per_epoch= len(idxT)//BATCH_SIZE, callbacks=[mod_checkpoint,get_lr_callback() ],
validation_data=valid_dataset)
 print("-"*5 +" Loading model weights from best epoch "+"-"*5)
 try:
model.load_weights("fold{}.h5".format(fold+1))
print('Done')
 except OSError:
print(f'Unable to load model!')
 valid_dataset_unlabelled = data_loader(idxV,labelled=False,return_ids=False,repeat=False, shuffle=False, cache=False)
 oof_preds[idxV,] = model.predict(valid_dataset_unlabelled, steps = len(idxV)/BATCH_SIZE)
 oof_acc = accuracy_score(np.argmax(oof_preds[idxV,],axis=1), train.label.values[idxV])
 print(f' Out-of-fold accuracy score: {oof_acc}')
 valid_dataset_unlabelled = data_loader(idxV,labelled=False,return_ids=True,repeat=False, shuffle=False)
 val_ids.extend([sample[0][3].numpy().decode('utf-8')for sample in valid_dataset_unlabelled.unbatch() ])
 test_dataset = data_loader(_,data='test', labelled=False,repeat=False, shuffle=False, cache=False)
 ypreds_test += model.predict(test_dataset, steps = test.shape[0]/BATCH_SIZE)/FOLDS
os.remove("fold{}.h5".format(fold+1))
if display_training_curve:
display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)
 display_training_curves(history.history['accuracy'], history.history['val_accuracy'], 'Accuracy', 212)
 gc.collect()<save_to_csv>
rf.fit(train_df_1, labels )
ypred_oof = np.argmax(oof_preds,axis=1)
 print(f"{FOLDS}-fold CV accuracy score = {accuracy_score(ypred_oof, train.label.values)}")
 oof_df = pd.DataFrame(list(zip(val_ids,ypred_oof.tolist())) ,columns = ['id','pred'])
 oof_df.to_csv('oof.csv', index=False)
 oof_df.head()<save_to_csv>
lgbm = LGBMRegressor(bagging_fraction=0.75, bagging_freq=5, bagging_seed=7,
feature_fraction=0.4, learning_rate=0.01, max_bin=200,
n_estimators=5000, num_leaves=8, objective='regression' )
ypred_sub = np.argmax(ypreds_test, axis=1)
 sub_df = pd.read_csv(".. /input/contradictory-my-dear-watson/sample_submission.csv")
 sub_df.loc[:,'prediction'] = ypred_sub
sub_df.to_csv('submission.csv',index=False)
 sub_df.head()<install_modules>
lgbm.fit(train_df_1, labels, eval_metric='rmse' )
!pip install --upgrade pip > /dev/null
!pip install --upgrade transformers > /dev/null
!pip install nlp > /dev/null<import_modules>
model = CatBoostRegressor(random_seed=17, depth=6, iterations=5000, learning_rate=0.01 )
import os
import gc
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import plotly.express as px
from tensorflow.keras.layers import Dense, Input, GlobalAveragePooling1D, GlobalMaxPooling1D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
import tensorflow as tf
from transformers import TFAutoModel, AutoTokenizer
import nlp<choose_model_class>
model.fit(train_df_1, labels )
def init_strategy() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Init TPU strategy")
 except ValueError:
strategy = tf.distribute.get_strategy()
print("Init CPU/GPU strategy")
 return strategy
def build_model(model_name, maxlen, head="avg_pooling"):
input_ids = Input(shape=(maxlen,), dtype=tf.int32, name="input_ids")
 encoder = TFAutoModel.from_pretrained(model_name)
 encoder_output = encoder(input_ids)[0]
if head == "cls":
features = encoder_output[:, 0, :]
elif head == "avg_pooling":
features = GlobalAveragePooling1D()(encoder_output)
 elif head == "max_pooling":
features = GlobalMaxPooling1D()(encoder_output)
 else:
raise NotImplementedError
out = Dense(3, activation='softmax' )(features)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model
def tokenize_dataframe(df, tokenizer, max_length):
text = df[['premise', 'hypothesis']].values.tolist()
encoded = tokenizer.batch_encode_plus(text, padding=True, max_length=max_length, truncation=True)
 x = encoded['input_ids']
y = None
if 'label' in df.columns:
y = df.label.values
return x, y
def build_dataset(x, y, mode, batch_size):
if mode == "train":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 elif mode == "valid":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 elif mode == "test":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(x)
.batch(batch_size)
)
 else:
raise NotImplementedError
return dataset
def load_mnli(use_validation=True):
result = []
dataset = nlp.load_dataset(path='glue', name='mnli')
 keys = ['train', 'validation_matched','validation_mismatched'] if use_validation else ['train']
for k in keys:
for record in dataset[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result.append(( c1,c2,c3,'en'))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
def load_xnli() :
result = []
dataset = nlp.load_dataset(path='xnli')
 for k in dataset.keys() :
for record in dataset[k]:
hp, pr, lb = record['hypothesis'], record['premise'], record['label']
if hp and pr and lb in {0,1,2}:
for lang, translation in zip(hp['language'], hp['translation']):
pr_lang = pr.get(lang, None)
 if pr_lang is None:
continue
result.append(( pr_lang, translation, lb,lang))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result 
<load_pretrained>
xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', max_delta_step=0,
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1, learning_rate=0.1,
n_estimators=2500 )
MODEL = 'jplu/tf-xlm-roberta-large'
MAXLEN = 120
strategy = init_strategy()
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 auto = tf.data.experimental.AUTOTUNE
def preprocess(df):
return tokenize_dataframe(df, tokenizer, MAXLEN )<load_from_csv>
xgb.fit(train_df_1, labels )
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 x, y = preprocess(train)
 x_test, _ = preprocess(test)
 test_dataset = build_dataset(x_test, None, "test", BATCH_SIZE)
 mnli = load_mnli()
xnli = load_xnli()<feature_engineering>
preds1 = rf.predict(test_df_1 )
punct = '[' + ''.join([c for c in string.punctuation if c != "'"])+ ']'
def preprocess_query(q):
q = q.lower()
q = re.sub(punct, ' ', q)
 q = re.sub('[ ]{2,}', ' ', q)
 return q
def search_in_base(q, kb):
q = preprocess_query(q)
 return int(q in kb)
<concatenate>
pred2 = lgbm.predict(test_df_1 )
premises = pd.concat([train[['premise', 'lang_abv']], test[['premise', 'lang_abv']]] )<feature_engineering>
pred3 = model.predict(test_df_1 )
knowledge_base = set(mnli['premise'].apply(preprocess_query))
premises['mnli'] = premises['premise'].apply(lambda q: search_in_base(q, knowledge_base))
print(f"fraction of train set english premises occurence in MNLI = {premises.loc[premises.lang_abv=='en', 'mnli'].mean() * 100}%" )<feature_engineering>
pred4 = xgb.predict(test_df_1 )
knowledge_base = set(xnli['premise'].apply(preprocess_query))
premises['xnli'] = premises['premise'].apply(lambda q: search_in_base(q, knowledge_base))
print(f"fraction of train set non-english premises occurence in XNLI = {premises.loc[premises.lang_abv!='en', 'xnli'].mean() * 100}%" )<save_to_csv>
preds =(preds1 + pred2 + pred3 + pred4)/ 4
strategy = init_strategy()
with strategy.scope() :
model = build_model(MODEL, MAXLEN)
 model.load_weights(".. /input/watson-xlmr-models/XLMR_mnlixnli_ep6.h5")
 dataset = build_dataset(x, y, "valid", BATCH_SIZE)
 pr = np.argmax(model.predict(dataset), axis=1)
 print(f"accuracy {accuracy_score(y, pr):.4f}")
 test_preds = model.predict(test_dataset, verbose=0)
 submission['prediction'] = test_preds.argmax(axis=1)
 submission.to_csv('submission.csv', index=False )<install_modules>
sub_csv = pd.DataFrame({
"Id": test_df["Id"],
"SalePrice": preds
} )
<import_modules><EOS> 
sub_csv.to_csv('submission.csv', index=False )