Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<install_modules>
import numpy as np
import pandas as pd
!pip install git+https://github.com/ssut/py-googletrans.git<import_modules>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 y = train["SalePrice"]
X = train.drop(labels=["SalePrice"], axis=1)
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
from googletrans import Translator
from dask import bag, diagnostics
import numpy as np<categorify>
cat_features = ["MSZoning", "MSSubClass", "Street", "Alley",
"LotShape", "LandContour", "Utilities", "LotConfig",
"LandSlope", "Neighborhood","Condition1",
"Condition2", "BldgType", "HouseStyle",
"RoofStyle", "RoofMatl", "Exterior1st", "Exterior2nd",
"MasVnrType", "ExterQual", "ExterCond", "Foundation",
"BsmtQual", "BsmtCond", "BsmtExposure", "BsmtFinType1",
"BsmtFinType2", "Heating", "HeatingQC", "CentralAir",
"Electrical", "Functional", "FireplaceQu", "GarageType",
"GarageFinish", "GarageQual", "GarageCond", "PavedDrive",
"SaleType", "SaleCondition", "PoolQC", "Fence", "MiscFeature",
"KitchenQual"]
def translate(words, dest):
dest_choices = ['zh-cn',
'ar',
'fr',
'sw',
'ur',
'vi',
'ru',
'hi',
'el',
'th',
'es',
'de',
'tr',
'bg'
]
if not dest:
dest = np.random.choice(dest_choices)
 translator = Translator()
decoded = translator.translate(words, dest=dest ).text
return decoded
def trans_parallel(df, dest):
premise_bag = bag.from_sequence(df.premise.tolist() ).map(translate, dest)
 hypo_bag = bag.from_sequence(df.hypothesis.tolist() ).map(translate, dest)
 with diagnostics.ProgressBar() :
premises = premise_bag.compute()
hypos = hypo_bag.compute()
df[['premise', 'hypothesis']] = list(zip(premises, hypos))
return df
eng_trans = train.loc[train.lang_abv == "en"].copy() \ 
.pipe(trans_parallel, dest=None)
 non_eng_trans = train.loc[train.lang_abv != "en"].copy() \ 
.pipe(trans_parallel, dest='en')
 eng_trans[['lang_abv', 'language']] = [['mx', 'Mixed']]*len(eng)
 non_eng_trans[['lang_abv', 'language']] = [['en', 'English']]*len(non_eng_trans)
 train = train.append([eng_trans, non_eng_trans])
 train.reset_index
train.shape<install_modules>
ordinal = ["OverallQual", "OverallCond", "YearBuilt", "YearRemodAdd",
"GarageYrBlt", "GarageCars", "MoSold", "YrSold"]
!pip install git+https://github.com/makcedward/nlpaug<import_modules>
absolute = ["LotFrontage", "LotArea", "MasVnrArea", "BsmtFinSF1",
"BsmtFinSF2", "BsmtUnfSF", "TotalBsmtSF", "1stFlrSF", "2ndFlrSF",
"LowQualFinSF", "BsmtFullBath", "BsmtHalfBath", "FullBath",
"HalfBath", "BedroomAbvGr", "KitchenAbvGr", "TotRmsAbvGrd",
"Fireplaces", "GarageArea", "WoodDeckSF", "OpenPorchSF", "EnclosedPorch",
"3SsnPorch", "ScreenPorch", "PoolArea"]
import nlpaug.augmenter.word as naw
import nlpaug.flow as nafc
from nlpaug.util import Action<normalization>
X[absolute].isnull().sum()
text = train.premise.values[0]
model = 'distilbert-base-uncased'
ins_aug = naw.ContextualWordEmbsAug(
 model_path=model, action="insert")
 print("Original:")
 print(text)
 print("Augmented Text:")
 for i in range(10):
augmented_text = ins_aug.augment(text)
 print(augmented_text )<categorify>
X[cat_features].isnull().sum()
syn_aug = naw.SynonymAug(aug_src='wordnet')
 augmented_text = syn_aug.augment(text)
 print("Original:")
 print(text)
 print("Augmented Text:")
 for i in range(10):
augmented_text = syn_aug.augment(text)
 print(augmented_text )<categorify>
X[ordinal].isnull().sum()
def insert_augment(words, model):
ins_aug = naw.ContextualWordEmbsAug(
 model_path=model, action="insert")
 augmented_text = ins_aug.augment(words)
 return augmented_text
def ins_aug_parallel(df, model):
premise_bag = bag.from_sequence(df.premise.tolist() ).map(insert_augment, model)
 hypo_bag = bag.from_sequence(df.hypothesis.tolist() ).map(insert_augment, model)
 with diagnostics.ProgressBar() :
premises = premise_bag.compute()
hypos = hypo_bag.compute()
df[['premise', 'hypothesis']] = list(zip(premises, hypos))
return df
eng_ins_aug = train.loc[train.lang_abv == "en"].copy() \ 
.pipe(ins_aug_parallel, model='distilbert-base-uncased')
 train = train.append([eng_ins_aug])
 train.to_csv('train_cleaned_ins.csv',index=False)
<categorify>
!pip install category_encoders
def synonym_augment(words):
syn_aug = naw.SynonymAug(
 aug_src = 'wordnet')
 augmented_text = syn_aug.augment(words)
 return augmented_text
def syn_aug_parallel(df):
premise_bag = bag.from_sequence(df.premise.tolist() ).map(synonym_augment)
 hypo_bag = bag.from_sequence(df.hypothesis.tolist() ).map(synonym_augment)
 with diagnostics.ProgressBar() :
premises = premise_bag.compute()
hypos = hypo_bag.compute()
df[['premise', 'hypothesis']] = list(zip(premises, hypos))
return df
eng_syn_aug = train.loc[train.lang_abv == "en"].copy() \ 
.pipe(syn_aug_parallel)
 train = train.append([eng_syn_aug])
 train.to_csv('train_cleaned_ins_syn.csv',index=False)
<set_options>
cb_enc = ce.CatBoostEncoder(cols=cat_features, verbose=1)
 X = X.drop(labels=cat_features, axis=1 ).join(cb_enc.fit_transform(X[cat_features], y))
test = test.drop(labels=cat_features, axis=1 ).join(cb_enc.transform(test[cat_features]))
print(len(test))
plt.style.use('fivethirtyeight')
 warnings.filterwarnings('ignore')
<set_options>
knn_imp = KNNImputer()
simp_imp = SimpleImputer(strategy="median")
 X = pd.DataFrame(simp_imp.fit_transform(X), columns=X.columns)
 test = pd.DataFrame(simp_imp.fit_transform(test), columns=test.columns )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
X[absolute].isnull().sum()
MODEL = 'jplu/tf-xlm-roberta-large'
EPOCHS = 8
MAX_LEN = 96
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE<load_pretrained>
X[ordinal].isnull().sum()
tokenizer = AutoTokenizer.from_pretrained(MODEL )<prepare_x_and_y>
from vecstack import StackingTransformer
from sklearn.pipeline import Pipeline
def quick_encode(df,maxlen=100):
values = df[['premise','hypothesis']].values.tolist()
tokens=tokenizer.batch_encode_plus(values,max_length=maxlen,pad_to_max_length=True)
 return np.array(tokens['input_ids'])
 x_train = quick_encode(train)
 x_test = quick_encode(test)
 y_train = train.label.values<define_variables>
from catboost import CatBoostRegressor
from xgboost import XGBRegressor
from sklearn.ensemble import RandomForestRegressor
def create_dist_dataset(X, y,val,batch_size= BATCH_SIZE):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(len(X))
if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(AUTO)
 else:
dataset = dataset.batch(batch_size ).prefetch(AUTO)
 return dataset
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_test)) 
.batch(BATCH_SIZE)
 )<choose_model_class>
d1 = 3
d2 = 7
d3 = 11
d4 = 13
rs = 42
L1_stack = StackingTransformer(
 estimators=[ 
(f'CatBoost depth={d1}', CatBoostRegressor(verbose=False, eval_metric="RMSE", random_seed=rs, depth=d1)) , 
(f'CatBoost depth={d2}', CatBoostRegressor(verbose=False, eval_metric="RMSE", random_seed=rs, depth=d2)) , 
(f'CatBoost depth={d3}', CatBoostRegressor(verbose=False, eval_metric="RMSE", random_seed=rs, depth=d3)) , 
(f'XGBoost depth={d1}', XGBRegressor(seed=rs, max_depth=d1)) , 
(f'XGBoost depth={d2}', XGBRegressor(seed=rs, max_depth=d2)) , 
(f'XGBoost depth={d3}', XGBRegressor(seed=rs, max_depth=d3)) , 
(f'RandomForest depth={d1}', RandomForestRegressor(random_state=rs, max_depth=d1)) , 
(f'RandomForest depth={d2}', RandomForestRegressor(random_state=rs, max_depth=d2)) , 
(f'RandomForest depth={d3}', RandomForestRegressor(random_state=rs, max_depth=d3))
],
regression=True,
verbose=2,
random_state=42,
n_folds=5,
variant="A"
)
def build_model(transformer,max_len):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model<train_model>
final_estimator = LinearRegression()
n_steps = len(x_train)// batch_size
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)<split>
model = Pipeline(
 steps=[ 
("L1_stack", L1_stack), 
("Final_estimator", final_estimator)
 ]
)
pred_test=np.zeros(( test.shape[0],3))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 val_score=[]
history=[]
for fold,(train_ind,valid_ind)in enumerate(skf.split(x_train,y_train)) :
if fold < 4:
print("fold",fold+1)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 train_data = create_dist_dataset(x_train[train_ind],y_train[train_ind],val=False)
 valid_data = create_dist_dataset(x_train[valid_ind],y_train[valid_ind],val=True)
 Checkpoint=tf.keras.callbacks.ModelCheckpoint(f"roberta_base.h5", monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min')
 with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 n_steps = len(train_ind)//BATCH_SIZE
print("training model {} ".format(fold+1))
train_history = model.fit(
 train_data,
steps_per_epoch=n_steps,
validation_data=valid_data,
epochs=EPOCHS,callbacks=[Checkpoint],verbose=0)
 print("Loading model...")
 model.load_weights(f"roberta_base.h5")
 print("fold {} validation acc {}".format(fold+1,np.mean(train_history.history['val_accuracy'])))
 print("fold {} validation acc {}".format(fold+1,np.mean(train_history.history['val_loss'])))
 history.append(train_history)
 val_score.append(np.mean(train_history.history['val_accuracy']))
print('predict on test.... ')
 preds=model.predict(test_dataset,verbose=1)
 pred_test+=preds/4<load_from_csv>
model.fit(X, y )
submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 submission['prediction'] = np.argmax(pred_test,axis=1)
 submission.head()<save_to_csv>
y_pred = model.predict(test )
<import_modules><EOS> 
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission["SalePrice"] = y_pred
submission.to_csv('submission.csv', index=False )