Kaggle competition: House Prices - Advanced Regression Techniques
x_small_train = tokenize(x_small_train, tokenizer, max_len)
 small_train_dataset = build_dataset(x_small_train, y_small_train, "train", batch_size )<train_model>
pd.options.mode.chained_assignment = None
%matplotlib inline
history_small_train = model.fit(small_train_dataset,
steps_per_epoch=100,
epochs=3 )<choose_model_class>
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv" )
model = build_model(encoder_handle, random_seed, learning_rate, loss, metrics, max_len )<train_model>
X_train, X_test, y_train, y_test = train_test_split(train, train["SalePrice"], test_size=0.3, random_state=42)
 X_train.shape , X_test.shape
history = model.fit(train_dataset,
validation_data=valid_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=[early_stopping] )<create_dataframe>
cat_var_na = []
for label, content in X_train.items() :
if pd.api.types.is_string_dtype(content)and train[label].isnull().sum() *100/ len(X_train)>0:
cat_var_na.append(label)
 X_train[cat_var_na].isnull().sum() *100/len(X_train )
x_test = tokenize(test[['premise', 'hypothesis']].values.tolist() , tokenizer, max_len)
 test_dataset = build_dataset(x_test, None, "test", batch_size )<prepare_output>
cat_var_40 = []
cat_var_10 = []
for var in cat_var_na:
if X_train[var].isnull().sum() *100/len(X_train)>=5:
cat_var_40.append(var)
 else:
cat_var_10.append(var )
predictions_prob = model.predict(test_dataset)
 final = predictions_prob.argmax(axis=-1)
 submission = pd.DataFrame()
submission['id'] = test['id']
submission['prediction'] = final.astype(np.int32 )<save_to_csv>
X_train[cat_var_40] = X_train[cat_var_40].fillna("Missing")
 X_test[cat_var_40] = X_test[cat_var_40].fillna("Missing")
 test[cat_var_40] = test[cat_var_40].fillna("Missing" )
submission.to_csv("submission.csv", index = False )<install_modules>
for var in cat_var_10:
mode_val = X_train[var].mode() [0]
X_train[var] = X_train[var].fillna(mode_val)
 X_test[var] = X_test[var].fillna(mode_val)
 test[var] = test[var].fillna(mode_val )
!pip install sentence-transformers
!pip install pandas
!pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html<set_options>
cat_var = []
for var in X_train.columns:
if X_train[var].dtype == "O":
cat_var.append(var )
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler() ])
 dataset_path = '.. /input/contradictory-my-dear-watson/train.csv'
logging.info("Read AllNLI train dataset")
 label2int = {"contradiction": 0, "entailment": 1, "neutral": 2}
train_samples = []
dev_samples = []
train_data = pandas.read_csv(dataset_path)
 train_data['label'] = train_data['label'].replace([0, 2], [2, 0])
 for id, row in train_data.iterrows() :
label_id = int(row['label'])
 train_samples.append(InputExample(texts=[row['premise'], row['hypothesis']], label=label_id))
train_batch_size = 16
num_epochs = 20
model_save_path = './training_allnli-' + datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
 model = CrossEncoder('sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens', num_labels=len(label2int))
train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=train_batch_size)
 evaluator = CESoftmaxAccuracyEvaluator.from_input_examples(dev_samples, name='AllNLI-dev')
 warmup_steps = math.ceil(len(train_dataloader)* num_epochs * 0.1)
 logging.info("Warmup-steps: {}".format(warmup_steps))
model.fit(train_dataloader=train_dataloader,
epochs=num_epochs,
warmup_steps=warmup_steps,
output_path=model_save_path)
 test_dataset = '.. /input/contradictory-my-dear-watson/test.csv'
df = pandas.read_csv(test_dataset)
 sentence_pairs = []
ids = []
for id, row in df.iterrows() :
label_id = 0
ids.append(row['id'])
 sentence_pairs.append([row['premise'], row['hypothesis']])
 pred_scores = model.predict(sentence_pairs, convert_to_numpy=True, show_progress_bar=False,
batch_size=train_batch_size)
 pred_labels = np.argmax(pred_scores, axis=1)
 out_df = pandas.DataFrame([ids, pred_labels] ).transpose()
out_df = out_df.rename(columns={0: 'id', 1: 'prediction'})
 out_df['prediction'] = out_df['prediction'].replace([0, 2],[2, 0])
 out_df.to_csv('submission.csv', index=False)
<install_modules>
def find_frequent(df, var, rare):

 df=df.copy()
temp = df[var].value_counts() /len(df)
 return temp[temp>rare].index
!pip install googletrans textAugment<feature_engineering>
for var in cat_var:
frequent_ls = find_frequent(X_train, var, 0.03)
 X_train[var] = np.where(X_train[var].isin(frequent_ls), X_train[var], "Rare")
 X_test[var] = np.where(X_test[var].isin(frequent_ls), X_test[var], "Rare")
 test[var] = np.where(test[var].isin(frequent_ls), test[var], "Rare" )
os.environ["WANDB_API_KEY"] = "0" 
<find_best_params>
def encode(train, test, c_test, target):
ordered_label = train.groupby([var])[target].mean().sort_values().index
ordinal_label = {k: i for i,k in enumerate(ordered_label,0)}
train[var] = train[var].map(ordinal_label)
 test[var] = test[var].map(ordinal_label)
 c_test[var] = c_test[var].map(ordinal_label)
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync )<load_from_csv>
for var in cat_var:
encode(X_train, X_test, test, "SalePrice" )
df_train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 df_test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv' )<feature_engineering>
num_var_na = []
for label, content in X_train.items() :
if pd.api.types.is_numeric_dtype(content)and X_train[label].isnull().sum() >0:
num_var_na.append(label)
 X_train[num_var_na].isnull().sum() *100/len(X_train)
def clean_word(value):
language = value[0]
word = value[1]
if language != 'English':
word = word.lower()
return word
word = word.lower()
word = re.sub(r'\?\?', 'e', word)
 word = re.sub('\.\.\.', '.', word)
 word = re.sub('\/', ' ', word)
 word = re.sub('--', ' ', word)
 word = re.sub('/\xad', '', word)
 word = word.strip(' ')
 return word
df_train['premise'] = df_train[['language', 'premise']].apply(lambda v: clean_word(v), axis=1)
 df_train['hypothesis'] = df_train[['language', 'hypothesis']].apply(lambda v: clean_word(v), axis=1)
 df_test['premise'] = df_test[['language', 'premise']].apply(lambda v: clean_word(v), axis=1)
 df_test['hypothesis'] = df_test[['language', 'hypothesis']].apply(lambda v: clean_word(v), axis=1)
 languages = [ 'zh-cn' if lang == 'zh' else lang for lang in df_train['lang_abv'].unique() ]<load_pretrained>
for var in num_var_na:
var_mode = X_train[var].mode() [0]
X_train[var + "_na"] = np.where(X_train[var].isnull() ,1,0)
 X_test[var + "_na"] = np.where(X_test[var].isnull() ,1,0)
 test[var + "_na"] = np.where(test[var].isnull() ,1,0)
 X_train[var] = X_train[var].fillna(var_mode)
 X_test[var] = X_test[var].fillna(var_mode)
 test[var] = test[var].fillna(var_mode)
 X_train[num_var_na].isnull().sum() *100/len(X_train )
seed = 42
tf.random.set_seed(seed)
 model_name = 'jplu/tf-xlm-roberta-large'
tokenizer = XLMRobertaTokenizer.from_pretrained(model_name)
 def build_model() :
with strategy.scope() :
bert_encoder = TFXLMRobertaModel.from_pretrained(model_name)
 input_word_ids = tf.keras.Input(shape=(None,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(None,), dtype=tf.int32, name="input_mask")
 embedding = bert_encoder([input_word_ids, input_mask])[0]
output_layer = tf.keras.layers.Dropout(0.25 )(embedding)
 output_layer = tf.keras.layers.GlobalAveragePooling1D()(output_layer)
 output_dense_layer = tf.keras.layers.Dense(64, activation='relu' )(output_layer)
 output_dense_layer = tf.keras.layers.Dense(32, activation='relu' )(output_dense_layer)
 output = tf.keras.layers.Dense(3, activation='softmax' )(output_dense_layer)
 model = tf.keras.Model(inputs=[input_word_ids, input_mask], outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 return model
with strategy.scope() :
model = build_model()
model.summary()<define_variables>
cont_var = ["LotFrontage", "LotArea", "1stFlrSF", "GrLivArea"]
batch_size = 8 * strategy.num_replicas_in_sync
num_splits = 5
test_input = None<categorify>
def quantile_boundary(df, var, lower_bound, upper_bound):

 lower = df[var].quantile(lower_bound)
 upper = df[var].quantile(upper_bound)
 return lower, upper
auto = tf.data.experimental.AUTOTUNE
def make_dataset(train_input, train_label):
dataset = tf.data.Dataset.from_tensor_slices(
(
 train_input,
train_label 
)
 ).repeat().shuffle(batch_size ).batch(batch_size ).prefetch(auto)
 return dataset
def xlm_roberta_encode(hypotheses, premises, src_langs, augmentation=False):
num_examples = len(hypotheses)
 sentence_1 = [tokenizer.encode(s)for s in premises]
sentence_2 = [tokenizer.encode(s)for s in hypotheses]
input_word_ids = list(map(lambda x: x[0]+x[1], list(zip(sentence_1,sentence_2))))
input_mask = [np.ones_like(x)for x in input_word_ids]
inputs = {
'input_word_ids': tf.keras.preprocessing.sequence.pad_sequences(input_word_ids, padding='post'),
'input_mask': tf.keras.preprocessing.sequence.pad_sequences(input_mask, padding='post')
 }
return inputs
train_df, validation_df = train_test_split(df_train, test_size=0.1)
 df_train['prediction'] = 0
num_augmentation = 1
train_input = xlm_roberta_encode(train_df.hypothesis.values,train_df.premise.values, train_df.lang_abv.values, augmentation=False)
 train_label = train_df.label.values
train_sequence = make_dataset(train_input, train_label)
 validation_input = xlm_roberta_encode(validation_df.hypothesis.values, validation_df.premise.values,validation_df.lang_abv.values, augmentation=False)
 validation_label = validation_df.label.values
tf.keras.backend.clear_session()<categorify>
lower,upper = quantile_boundary(X_train, "LotFrontage",0.065,0.95)
 X_train["LotFrontage"] = np.where(X_train["LotFrontage"]<lower,lower,
np.where(X_train["LotFrontage"]>upper,upper,X_train["LotFrontage"]))
lower,upper = quantile_boundary(X_train, "LotArea",0.0065,0.955)
 X_train["LotArea"] = np.where(X_train["LotArea"]<lower,lower,
np.where(X_train["LotArea"]>upper,upper,X_train["LotArea"]))
lower,upper = quantile_boundary(X_train, "1stFlrSF",0.0065,0.985)
 X_train["1stFlrSF"] = np.where(X_train["1stFlrSF"]<lower,lower,
np.where(X_train["1stFlrSF"]>upper,upper,X_train["1stFlrSF"]))
lower,upper = quantile_boundary(X_train, "GrLivArea",0.0065,0.98)
 X_train["GrLivArea"] = np.where(X_train["GrLivArea"]<lower,lower,
np.where(X_train["GrLivArea"]>upper,upper,X_train["GrLivArea"]))
train_df, validation_df = train_test_split(df_train, test_size=0.1)
 df_train['prediction'] = 0
num_augmentation = 1
train_input = xlm_roberta_encode(train_df.hypothesis.values,train_df.premise.values, train_df.lang_abv.values, augmentation=False)
 train_label = train_df.label.values
train_sequence = make_dataset(train_input, train_label)
 validation_input = xlm_roberta_encode(validation_df.hypothesis.values, validation_df.premise.values,validation_df.lang_abv.values, augmentation=False)
 validation_label = validation_df.label.values
tf.keras.backend.clear_session()<train_model>
X_train["SalePrice"]=np.log(X_train["SalePrice"] )
n_steps =(len(train_label)) // batch_size
with strategy.scope() :
history = model.fit(
 train_sequence, shuffle=True, steps_per_epoch=n_steps,
validation_data =(validation_input, validation_label), epochs=50, verbose=1,
callbacks=[
tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10),
tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=5),
tf.keras.callbacks.ModelCheckpoint(
 'model.h5', monitor='val_accuracy', save_best_only=True,save_weights_only=True)
 ] 
)
 model.load_weights('model.h5')
 validation_predictions = model.predict(validation_input)
 validation_predictions = np.argmax(validation_predictions, axis=-1)
 validation_df['predictions'] = validation_predictions
acc = accuracy_score(validation_label, validation_predictions)
 print('Accuracy: {}'.format(acc))<save_to_csv>
def year_to_age(df, var):

 df[var] = df["YrSold"]-df[var]
return df
if test_input is None:
test_input = xlm_roberta_encode(df_test.hypothesis.values, df_test.premise.values, df_test.lang_abv.values,augmentation=False)
 test_split_predictions = model.predict(test_input)
 predictions = np.argmax(test_split_predictions, axis=-1)
 submission = df_test.id.copy().to_frame()
submission['prediction'] = predictions
submission.head()
submission.to_csv("submission.csv", index = False )<import_modules>
year_var = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']
for var in year_var:
X_train = year_to_age(X_train, var)
 X_test = year_to_age(X_test, var)
 test = year_to_age(test, var)
import os
import numpy as np
import pandas as pd
from kaggle_datasets import KaggleDatasets
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
import plotly.express as px<set_options>
lower,upper = quantile_boundary(X_train, "YearBuilt",0.065,0.97)
 X_train["YearBuilt"] = np.where(X_train["YearBuilt"]<lower,lower,
np.where(X_train["YearBuilt"]>upper,upper,X_train["YearBuilt"]))
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
feature_scale = [var for var in X_train.columns if var not in ['Id', 'SalePrice']]
len(feature_scale )
model_name = 'jplu/tf-xlm-roberta-large'
n_epochs = 10
max_len = 80
batch_size = 16 * strategy.num_replicas_in_sync<load_from_csv>
for var in feature_scale:
mode_val = X_train[var].mode() [0]
test[var] = test[var].fillna(mode_val )
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv' )<load_from_csv>
scaler=MinMaxScaler()
X_train[feature_scale] = scaler.fit_transform(X_train[feature_scale])
 X_test[feature_scale] = scaler.transform(X_test[feature_scale])
 test[feature_scale] = scaler.transform(test[feature_scale] )
twitter = pd.read_csv('/kaggle/input/sentiment140/training.1600000.processed.noemoticon.csv',
encoding = 'latin1',
names=['polarity','id','date','flag','user','text'] )<split>
y_train = X_train['SalePrice'].values
y_test = X_test['SalePrice'].values
X_train.drop(['Id', 'SalePrice'], axis=1, inplace=True)
 X_test.drop(['Id', 'SalePrice'], axis=1, inplace=True )
twitter_premise, twitter_hypothesis = train_test_split(twitter['text'], test_size=0.5, random_state=2020 )<prepare_output>
sfm = SelectFromModel(Lasso(alpha = 0.001, random_state = 0))
sfm.fit(X_train,y_train)
 feature_selected = X_train.columns[(sfm.get_support())]
len(feature_selected )
twitter_df = pd.DataFrame()
twitter_df['premise'] = twitter_premise.values
twitter_df['hypothesis'] = twitter_hypothesis.values
twitter_df.head()<load_pretrained>
X_train=X_train[feature_selected]
X_test = X_test[feature_selected]
test = test[feature_selected]
tokenizer = AutoTokenizer.from_pretrained(model_name )<categorify>
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 14))
return(rmse )
%%time
train_text = train[['premise', 'hypothesis']].values.tolist()
test_text = test[['premise', 'hypothesis']].values.tolist()
twitter_text = twitter_df[['premise', 'hypothesis']].values.tolist()
train_encoded = tokenizer.batch_encode_plus(
 train_text,
pad_to_max_length=True,
max_length=max_len 
)
 test_encoded = tokenizer.batch_encode_plus(
 test_text,
pad_to_max_length=True,
max_length=max_len 
)
 twitter_encoded = tokenizer.batch_encode_plus(
 twitter_text,
pad_to_max_length=True,
max_length=max_len
)<split>
model_ridge = Ridge()
alphas = [0.08,0.05, 0.1,0.001, 0.3, 1, 3, 5, 10,11, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]
cv_ridge = pd.Series(cv_ridge, index = alphas)
 cv_ridge.min()
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded['input_ids'], train.label.values,
test_size=0.2, random_state=2020 
)
 x_test = test_encoded['input_ids']
x_twitter_train1, x_twitter_train2 = train_test_split(
 twitter_encoded['input_ids'],
test_size=0.5, random_state=2020
)<define_variables>
model_lasso = LassoCV(alphas = [0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100] ).fit(X_train, y_train)
 rmse_cv(model_lasso ).min()
%%time
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
)
 train_twitter_dataset1 =(
 tf.data.Dataset 
.from_tensor_slices(x_twitter_train1)
.batch(batch_size)
)
 train_twitter_dataset2 =(
 tf.data.Dataset 
.from_tensor_slices(x_twitter_train2)
.batch(batch_size)
 )<categorify>
model_xgb = xgb.XGBRegressor(n_estimators=450, max_depth=3, learning_rate=0.06)
 model_xgb.fit(X_train, y_train )
with strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(model_name)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 last_hidden_states = transformer_encoder(input_ids)[0]
cls_token = last_hidden_states[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()<train_model>
xgb_preds = np.expm1(model_xgb.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))
linear_preds = 0.4*xgb_preds+0.6*lasso_preds
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds,"actual":y_test,
"linear_preds": linear_preds })
 predictions["Xgb_error"]=predictions["actual"]-predictions["xgb"]
predictions["Lasso_error"]=predictions["actual"]-predictions["lasso"]
predictions["Linear_error"]=predictions["linear_preds"]-predictions["lasso"]
n_steps = len(x_train)// batch_size
train_history1 = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)<predict_on_test>
print(mean_squared_error(y_test,predictions["linear_preds"])**(1/2))
print(r2_score(y_test,predictions["linear_preds"]))
test_preds1 = model.predict(test_dataset, verbose=1 )<predict_on_test>
test_prediction= 0.4*np.expm1(model_xgb.predict(test)) + 0.6*np.expm1(model_lasso.predict(test))
<define_variables><EOS> 
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": test_prediction
})
 submission.to_csv('submission.csv', index=False )