Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
from concurrent.futures import ProcessPoolExecutor
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync )<load_from_csv>
from concurrent.futures import ProcessPoolExecutor
import itertools
import functools
import time
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
import sklearn
from sklearn.linear_model import LinearRegression, Ridge, Lasso, LassoCV, ElasticNet, ElasticNetCV
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold, cross_val_score, GridSearchCV
from sklearn.metrics import roc_curve, make_scorer, mean_squared_error
from sklearn.preprocessing import RobustScaler
from mlxtend.regressor import StackingCVRegressor
import lightgbm as lgb
import xgboost as xgb
train_csv = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 test_csv = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv" )<load_pretrained>
%%script false --no-raise-error
train = pd.read_csv('./train.csv')
 test = pd.read_csv('./test.csv' )
AUTO = tf.data.experimental.AUTOTUNE
MODEL_NAME = "jplu/tf-xlm-roberta-base"
REPLICAS = strategy.num_replicas_in_sync
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)
 BATCH_SIZE = 16 * REPLICAS
MAX_LEN = 192
EPOCHS = 8
SEED = 48
FONT_DIR = ".. /input/font-dataset/FontScripts/"
np.random.seed(SEED)
 random.seed(SEED )<categorify>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
def prepare_input_v2(sentences):

 sen_enc = TOKENIZER.batch_encode_plus(sentences,
pad_to_max_length=True,
return_attention_mask=False,
return_token_type_ids=False,
max_length=MAX_LEN)
 return np.array(sen_enc["input_ids"])
 def get_dataset(features, labels=None, labelled=True, batch_size=8, repeat=True, shuffle=True):

 if labelled:
ds = tf.data.Dataset.from_tensor_slices(( features, labels))
else:
ds = tf.data.Dataset.from_tensor_slices(features)
 if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(2048)
 ds = ds.batch(batch_size*REPLICAS)
 ds.prefetch(AUTO)
 return ds
def build_model() :

 encoder = TFAutoModel.from_pretrained(MODEL_NAME)
 input_word_ids = tf.keras.layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name="input_word_ids")
 embeddings = encoder(input_word_ids)[0]
x = embeddings[:, 0, :]
output = tf.keras.layers.Dense(3, activation="softmax" )(x)
 model = tf.keras.models.Model(inputs=input_word_ids, outputs=output)
 model.compile(loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),
optimizer=tf.keras.optimizers.Adam(lr=1e-5),
metrics=["accuracy"])
 return model
def ratio_languages(df):

 languages = np.unique(df.language)
 total = df.language.value_counts().sum()
ratios = {}
for e in languages:
ratios[e] = round(( df.language.value_counts().loc[e] / total), 2)*100
ratios = sorted(ratios.items() , key=lambda x:(x[1],x[0]), reverse=True)
 languages = []
values = []
for e in ratios:
languages.append(e[0])
 values.append(e[1])
 _, texts, _ = plt.pie(values, explode=[0.2]*(len(values)) , labels=languages, autopct="%.2i%%", radius=2,
rotatelabels=True)
 for e in texts:
e.set_fontsize(15)
 e.set_fontfamily('fantasy')
 plt.show()
def get_lr_callback(batch_size):
lr_start = 0.000001
lr_max = 0.00000125 * batch_size
lr_min = 0.00000001
lr_sus_epoch = 0
lr_decay = 0.80
lr_ramp_ep = 5
lr = lr_start
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max- lr_start)/lr_ramp_ep * epoch + lr_start
elif epoch <(lr_ramp_ep + lr_sus_epoch):
lr = lr_max
else:
lr =(lr_max - lr_min)*lr_decay**(epoch - lr_ramp_ep - lr_sus_epoch)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 return lr_callback
def plot_wordcloud(df, col):

 words = " "
font_path = None
fig, ax = plt.subplots(nrows=2, ncols=2)
 fig.set_size_inches(12, 12)
 res = []
for i in range(2):
for j in range(2):
res.append([i,j])
 for i,lang in enumerate(["English",
"Hindi",
"Urdu",
"German" ,
]):
for line in df[df.language==lang][col].values:
tokens = line.split()
tokens = [word.lower() for word in tokens]
words += " ".join(tokens)+" "
fig.add_subplot(ax[res[i][0]][res[i][1]])
 if lang=="Hindi":
font_path = FONT_DIR + "Hindi.ttf"
if lang=="French":
font_path = FONT_DIR + "French.ttf"
if lang=="Russian":
font_path= FONT_DIR + "Russian.ttf"
if lang=="Arabic":
font_path = FONT_DIR + "Arabic.ttf"
if lang=="Chinese":
font_path = FONT_DIR + "Chinese.otf"
if lang=="Swahili":
font_path = FONT_DIR + "Swahili.ttf"
if lang=="Urdu":
font_path = FONT_DIR + "Urdu.ttf"
if lang=="Vietnamese":
font_path = FONT_DIR + "Vietnamese.ttf"
if lang=="Greek":
font_path = FONT_DIR + "Greek.ttf"
if lang=="Thai":
font_path = FONT_DIR + "Thai.ttf"
if lang=="Spanish":
font_path = FONT_DIR + "Spanish.ttf"
if lang=="German":
font_path = FONT_DIR + "German.ttf"
if lang=="Turkish":
font_path = FONT_DIR + "Turkish.ttf"
if lang=="Bulgarian":
font_path = FONT_DIR + "Bulgarian.ttf"
s_words = STOPWORDS
wordcloud = WordCloud(font_path=font_path, width=800, height=800,
background_color="black",
min_font_size=10,
stopwords=s_words ).generate(words)
 ax[res[i][0]][res[i][1]].imshow(wordcloud)
 ax[res[i][0]][res[i][1]].axis("off")
 ax[res[i][0]][res[i][1]].set_title(f"Language: {lang}", fontsize=14 )<count_values>
train_ids = train.Id
test_ids = test.Id
train_y = train.pop('SalePrice' )
print(train_csv["language"].value_counts())
 print()
ratio_languages(train_csv )<feature_engineering>
def allyficy(train, test):
return pd.concat([train, test] ).reset_index(drop=True )
xnli = nlp.load_dataset(path="xnli")
 buff = {}
buff["premise"] = []
buff["hypothesis"] = []
buff["label"] = []
buff["language"] = []
uniq_lang = set()
for e in xnli["test"]:
for i in e["hypothesis"]["language"]:
uniq_lang.add(i)
 language_map = {}
for e in uniq_lang:
language_map[e] = test_csv.loc[test_csv.lang_abv==e, "language"].iloc[0]
for x in xnli['test']:
label = x['label']
for idx, lang in enumerate(x['hypothesis']['language']):
if lang=="en":
continue
hypothesis = x['hypothesis']['translation'][idx]
premise = x['premise'][lang]
buff['premise'].append(premise)
 buff['hypothesis'].append(hypothesis)
 buff['label'].append(label)
 buff['language'].append(language_map[lang])
 xnli_df = pd.DataFrame(buff )<count_values>
class Get_rid_of_Nans(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def Stringificy(self, X):
X['MSZoning'] = X.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
X['YrSold'] = X.YrSold.astype(str)
 X['YearRemodAdd'] = X.YearRemodAdd.astype(str)
 X['MoSold']= X['MoSold'].astype(str)
 X['YearBuilt']= X['YearBuilt'].astype(str)
 def Nans_from_str(self, X):
missing_val_col = ["Alley", "PoolQC", "MiscFeature", "Fence", "FireplaceQu", "GarageType", "GarageFinish", "GarageQual",
"GarageCond", 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType']
for i in missing_val_col:
X[i] = X[i].fillna('None')
 X['Electrical'] = X['Electrical'].fillna("SBrkr")
 X['Exterior1st'] = X['Exterior1st'].fillna(X['Exterior1st'].mode() [0])
 X['Exterior2nd'] = X['Exterior2nd'].fillna(X['Exterior2nd'].mode() [0])
 X['Functional'] = X['Functional'].fillna(X['Functional'].mode() [0])
 X['SaleType'] = X['SaleType'].fillna(X['SaleType'].mode() [0])
 X['Utilities'] = X['Utilities'].fillna(X['Utilities'].mode() [0])
 X['KitchenQual'] = X['KitchenQual'].fillna(X['KitchenQual'].mode() [0])
 def Nans_from_int(self, X):
missing_val_col2 = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'GarageYrBlt',
'GarageArea', 'GarageCars', 'MasVnrArea']
for i in missing_val_col2:
X[i] = X[i].fillna(0)
 def Nans_from_float(self, X):
X['LotFrontage'] = X.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))
def transform(self, X, y=None):
self.X = X.copy()
self.Stringificy(self.X)
 self.Nans_from_int(self.X)
 self.Nans_from_str(self.X)
 self.Nans_from_float(self.X)
 return self.X
def fit_transform(self, X, y=None):
return self.fit(X ).self.transform(X )
xnli_df["language"].value_counts()<concatenate>
class Create_dummies(TransformerMixin):
def __init__(self):
pass
def fit(self):
return self
def transform(self, df):
return pd.get_dummies(df ).reset_index(drop=True )
new_df = pd.concat([train_df, xnli_df], axis=0)
 new_df.sample(5 )<merge>
class Overfit_reducer(TransformerMixin):
def __init__(self):
pass
def overfit_reducer(self, df):

 overfit = []
for i in df.columns:
counts = df[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 print('list of overfitted features: ',overfit)
 return overfit
def fit(self):
return self
def transform(self, df):
return df.drop(self.overfit_reducer(df), axis=1 )
pd.merge(new_df, test_csv, how="inner" )<merge>
class Add_new_features(TransformerMixin):
def __init__(self):
pass
def add_features(self, df):
df['TotalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['YrBltAndRemod'] = df['YearBuilt']+df['YearRemodAdd']
df['Total_sqr_footage'] =(df['BsmtFinSF1'] + df['BsmtFinSF2'] +
df['1stFlrSF'] + df['2ndFlrSF'])
 df['Total_Bathrooms'] =(df['FullBath'] +(0.5 * df['HalfBath'])+
df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath']))
df['Total_porch_sf'] =(df['OpenPorchSF'] + df['3SsnPorch'] +
df['EnclosedPorch'] + df['ScreenPorch'] +
df['WoodDeckSF'])
 df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['hasbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 return df
def fit(self):
return self
def transform(self, df):
return self.add_features(df )
new_df = new_df.merge(pd.merge(new_df, test_csv, how="inner"), how="left", indicator=True)
 new_df = new_df[new_df._merge=="left_only"]
new_df = new_df.drop(["id", "lang_abv", "_merge"], axis=1)
 new_df.info() 
<merge>
class Separate(TransformerMixin):
def __init__(self, y):
self.y =y
def fit(self):
return self
def transform(self, X):
return {'train': X.iloc[:len(self.y),:], 'test': X.iloc[len(train_y):,:]}
class Numerificy(TransformerMixin):
def __init__(self):
pass
def fit(self):
return self
def transform(self, X):
train_num = X['train'].select_dtypes(exclude="object")
 test_num = X['test'].select_dtypes(exclude='object')
 return {'train': train_num, 'test': test_num}
pd.merge(new_df, test_csv, how="inner" )<feature_engineering>
model_1st_without_nans_and_strings = Pipeline([('without_nans', Get_rid_of_Nans()), 
('dummyficeted', Create_dummies()), 
('overfit_reduced', Overfit_reducer()), 
('separated', Separate(train_y)) , 
('numyficyed', Numerificy())])
 all_data = allyficy(train, test)
 train_X, test_X = model_1st_without_nans_and_strings.transform(all_data ).values()
ratio_languages(new_df )<count_values>
r = RobustScaler().fit(train_X)
 train_X_robust, test_X_robust = r.transform(train_X), r.transform(test_X )
new_df.language.value_counts()<prepare_x_and_y>
def rmsle(clf, X, y):
y_ = clf.predict(X)
 return -np.sqrt(mean_squared_error(np.log1p(y),np.log1p(y_)) )
X, y = new_df[["premise", "hypothesis"]], new_df.label<data_type_conversions>
predict_LM = Pipeline([('model', LinearRegression())])
 predict_model_Elastic = Pipeline([('model', ElasticNet())])
X["language_label"] = new_df.language.astype(str)+ "_" + new_df.label.astype(str )<categorify>
class Tune_XGB:
def __init__(self, train_X, train_y, test_X, base_params=None):
self.train_X, self.train_y, self.test_X = train_X.copy() , train_y.copy() , test_X.copy()
FOLDS = 5
EARLY_STOP = 50
MAX_ROUNDS = 5
if base_params == None:
self.base_params = {
'n_estimators': 100,
'max_depth': -1,
'min_child_weight': 0,
'gamma': 10,
'subsample': 0.8,
'colsample_bytree': 0.5,
'reg_alpha': 0.0,
'reg_lambda': 1,
'learning_rate': 0.01,
'objective': 'reg:linear',
'silent': 1,
'seed': 42,
'verbosity':0}
else:
self.base_params = base_params
def update(self, base_dict, update_copy):
for key in update_copy.keys() :
base_dict[key] = update_copy[key]
def Step_0_find_n_estimators(self, n_estimators):
print(n_estimators)
 test0 = {
'n_estimators': n_estimators,
}
return self.grid_search(self.base_params, test0)
 def Step_05_find_n_estimators(self, test05):
curent = self.base_params['n_estimators']
test05 = {
'n_estimators': [int(curent/1.3), int(curent/1.1), curent, int(curent/0.9), int(curent/0.7)]
}
return self.grid_search(self.base_params, test05)
 def Step_1_find_depth_and_child(self, test1):
return self.grid_search(self.base_params, test1)
 def Step_2_narrow_depth(self, test2):
max_depth = self.base_params['max_depth']
test2 = {
'max_depth': [max_depth-1,max_depth,max_depth+1]
}
return self.grid_search(self.base_params, test2)
 def Step_3_gamma(self, test3):
return self.grid_search(self.base_params, test3)
 def Step_4_sample(self, test4):
return self.grid_search(self.base_params, test4)
 def Step_5_reg1(self, test5):
return self.grid_search(self.base_params, test5)
 def Step_6_eta_nround(self, test6):
return self.grid_search(self.base_params, test6)
 def train_test(self, X, y):
train_X_v, test_X_v, train_y_v, test_y_v = train_test_split(X, y, test_size=0.2)
 return train_X_v, test_X_v, train_y_v, test_y_v
def validate(self, model):
X,x, Y,y = self.train_test(self.train_X, self.train_y)
 model.fit(X, Y,
eval_set=[(X, Y),(x, y)],
eval_metric='rmse',
verbose=False)
 train_pred = model.predict(X)
 test_pred = model.predict(x)
 train_score = mean_squared_error(train_pred, Y)
 test_score = mean_squared_error(test_pred, y)
 error = model.evals_result()
eval_score = error['validation_0']['rmse'][-1]
return test_score, train_score, eval_score
def grid_search(self, base_params, grid):
print("Starn Tuning: ", grid)
 if self.mode==1:
return self.grid_search_parallel(base_params, grid)
 else:
keys = set(grid.keys())
 l = [grid[x] for x in keys]
perm = list(itertools.product(*l))
jobs = []
for i in perm:
jobs.append({k:v for k,v in zip(keys,i)})
 test_score = []
train_score = []
eval_score = []
for i, job in enumerate(jobs):
base_params = self.base_params.copy()
print(job)
 self.update(base_params, job)
 model = xgb.XGBRegressor(**base_params)
 score = self.validate(model)
 test_score.append(( job, score[0]))
train_score.append(( job, score[1]))
eval_score.append(( job, score[2]))
return {'train_scores':train_score, 'test_scores':test_score, 'eval_scores':eval_score}
def grid_search_parallel(self, base_params, grid):
keys = set(grid.keys())
 l = [grid[x] for x in keys]
perm = list(itertools.product(*l))
jobs = []
for i in perm:
jobs.append({k:v for k,v in zip(keys,i)})
 test_score = []
train_score = []
eval_score = []
e = ProcessPoolExecutor()
score = list(e.map(self.build_val ,jobs))
for i, scor in enumerate(score):
test_score.append(( jobs[i], scor[0]))
train_score.append(( jobs[i], scor[1]))
eval_score.append(( jobs[i], scor[2]))
return {'train_scores':train_score, 'test_scores':test_score, 'eval_scores':eval_score}
def build_val(self, job):
base_params = self.base_params.copy()
self.update(base_params, job)
 model = xgb.XGBRegressor(**base_params)
 score = self.validate(model)
 return(score)
 def draw(self, errors=None):
fig, ax = plt.subplots(figsize=(6,4))
ax.plot(range(len(errors['validation_0']['rmse'])) , errors['validation_0']['rmse'])
 plt.show()
def draw_all(self, errors):
fig, ax = plt.subplots(figsize=(6,4))
ax.plot(range(len(errors['train_scores'])) ,[i[1] for i in errors['train_scores']], label='train')
 ax.plot(range(len(errors['test_scores'])) ,[i[1] for i in errors['test_scores']], label='test')
 ax.plot(range(len(errors['eval_scores'])) ,[i[1] for i in errors['eval_scores']], label='eval_scores')
 ax.legend()
plt.show()
def start_tuning(self, num, mode=0):
self.mode=mode
print("loading")
 start_time = time.time()
base_params = dict(self.base_params)
 steps = {self.Step_0_find_n_estimators:[100, 200, 400, 800,1600,3200],
self.Step_05_find_n_estimators:None,
self.Step_1_find_depth_and_child:{'max_depth': [10, 14, 18, 22, -1],
'min_child_weight': list(range(1,10,2)) },
self.Step_2_narrow_depth: None,
self.Step_3_gamma:{'gamma': list([i/10.0 for i in range(0,5)])},
self.Step_4_sample: {'subsample': list([i/10.0 for i in range(6,10)]),
'colsample_bytree': list([i/10.0 for i in range(6,10)])},
self.Step_5_reg1:{'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100],
'reg_lambda': [1e-5, 1e-2, 0.1, 1, 100]},
self.Step_6_eta_nround:{'learning_rate':[0.001, 0.005, 0.01, 0.05, 0.1, 0.3]},
}
for i,key in enumerate(steps.keys()):
if i in range(num):
e = ProcessPoolExecutor
result = key(steps[key])
 self.draw_all(result)
 best_param = min(result['test_scores'], key=lambda x:x[1])
 print('best selected param:', best_param)
 self.update(self.base_params, best_param[0])
 print(f'params after step {i}:', self.base_params)
 print("FINAL RESUTL: {}".format(self.base_params))
elapsed_time = time.time() - start_time
print('done')
 tune = Tune_XGB(train_X_robust, train_y, test_X_robust )
print("Splitting Data...")
 x_train, x_test, y_train, y_test = train_test_split(X, y, stratify=X.language_label, test_size=0.2, random_state=SEED)
 y_train = tf.one_hot(y_train, depth=3)
 y_test = tf.one_hot(y_test, depth=3)
 print("Prepairing Input...")
 train_input = prepare_input_v2(x_train[["premise", "hypothesis"]].values.tolist())
 valid_input = prepare_input_v2(x_test[["premise", "hypothesis"]].values.tolist())
 print("Preparing Dataset...")
 train_dataset = get_dataset(train_input, y_train, labelled=True, batch_size=BATCH_SIZE, repeat=True,
shuffle=True)
 valid_dataset = get_dataset(valid_input, y_test, labelled=True, batch_size=BATCH_SIZE//REPLICAS, repeat=False,
shuffle=False)
 print("Downloading and Building Model...")
 with strategy.scope() :
model = build_model()
lr_callback = get_lr_callback(BATCH_SIZE)
 checkpoint = tf.keras.callbacks.ModelCheckpoint("XLM-R-base.h5", save_weights_only=True,
save_best_only=True, save_freq="epoch", monitor="val_loss",
mode="min")
 print("Training...")
 model.fit(train_dataset,
steps_per_epoch= x_train.shape[0]/BATCH_SIZE,
validation_data=valid_dataset,
epochs=EPOCHS,
callbacks=[lr_callback, checkpoint] )<create_dataframe>
class Tune_ligthGBM:
def __init__(self, train_X, train_y, test_X, base_params=None):
self.train_X, self.train_y, self.test_X = train_X.copy() , train_y.copy() , test_X.copy()
if base_params == None:
self.base_params = {'n_estimators':100,
'max_depth':7,
'num_leaves': 31,
'min_data_in_leaf':20,
'bagging_fraction':0.75,
'bagging_freq':5,
'bagging_seed':7,
'feature_fraction':0.8,
'feature_fraction_seed':7,
'reg_alpha': 0.0,
'reg_lambda': 1,
'learning_rate':0.01,
'objective':'regression',
'num_leaves':4,
'max_bin':200,
'verbose':-1,}
else:
self.base_params = base_params
def update(self, base_dict, update_copy):
for key in update_copy.keys() :
base_dict[key] = update_copy[key]
def Step_0_find_n_estimators(self, n_estimators):
print(n_estimators)
 test0 = {
'n_estimators': n_estimators,
}
return self.grid_search(self.base_params, test0)
 def Step_05_find_n_estimators(self, test05):
curent = self.base_params['n_estimators']
test05 = {
'n_estimators': [int(curent/1.3), int(curent/1.1), curent, int(curent/0.9), int(curent/0.7)]
}
return self.grid_search(self.base_params, test05)
 def Step_06_min_child_weight(self, teset06):
print(teset06)
 return self.grid_search(self.base_params, teset06)
 def Step_07_min_child_weight_closer(self, teset07):
curent = self.base_params['min_child_weight']
if curent>=4:
teset07={
'min_child_weight':[int(curent/1.3),curent, int(curent/0.7)]
}
else:
teset07={'min_child_weight': [curent]}
return self.grid_search(self.base_params, teset07)
 def Step_1_find_depth_and_child(self, test1):
return self.grid_search(self.base_params, test1)
 def Step_2_narrow_depth(self, test2):
max_depth = self.base_params['max_depth']
test2 = {
'max_depth': [max_depth-1,max_depth,max_depth+1]
}
return self.grid_search(self.base_params, test2)
 def Step_4_sample(self, test4):
return self.grid_search(self.base_params, test4)
 def Step_5_reg1(self, test5):
return self.grid_search(self.base_params, test5)
 def Step_6_eta_nround(self, test6):
return self.grid_search(self.base_params, test6)
 def train_test(self, X, y):
train_X_v, test_X_v, train_y_v, test_y_v = train_test_split(X, y, test_size=0.2)
 return train_X_v, test_X_v, train_y_v, test_y_v
def validate(self, model):
X,x, Y,y = self.train_test(self.train_X, self.train_y)
 model.fit(X, Y,
eval_set=[(X, Y),(x, y)],
eval_metric='rmse',
verbose=False)
 train_pred = model.predict(X)
 test_pred = model.predict(x)
 train_score = mean_squared_error(train_pred, Y)
 test_score = mean_squared_error(test_pred, y)
 return test_score, train_score
def grid_search(self, base_params, grid):
print("Starn Tuning: ", grid)
 if self.mode==1:
return self.grid_search_parallel(base_params, grid)
 else:
keys = set(grid.keys())
 l = [grid[x] for x in keys]
perm = list(itertools.product(*l))
jobs = []
for i in perm:
jobs.append({k:v for k,v in zip(keys,i)})
 test_score = []
train_score = []
for i, job in enumerate(jobs):
base_params = self.base_params.copy()
print(job)
 self.update(base_params, job)
 model = lgb.LGBMRegressor(**base_params)
 score = self.validate(model)
 test_score.append(( job, score[0]))
train_score.append(( job, score[1]))
eval_score.append(( job, score[2]))
return {'train_scores':train_score, 'test_scores':test_score}
def grid_search_parallel(self, base_params, grid):
keys = set(grid.keys())
 l = [grid[x] for x in keys]
perm = list(itertools.product(*l))
jobs = []
for i in perm:
jobs.append({k:v for k,v in zip(keys,i)})
 test_score = []
train_score = []
e = ProcessPoolExecutor()
score = list(map(self.build_val ,jobs))
for i, scor in enumerate(score):
test_score.append(( jobs[i], scor[0]))
train_score.append(( jobs[i], scor[1]))
return {'train_scores':train_score, 'test_scores':test_score}
def build_val(self, job):
base_params = self.base_params.copy()
self.update(base_params, job)
 model = lgb.LGBMRegressor(**base_params)
 score = self.validate(model)
 return(score)
 def draw(self, errors=None):
fig, ax = plt.subplots(figsize=(6,4))
ax.plot(range(len(errors['validation_0']['rmse'])) , errors['validation_0']['rmse'])
 plt.show()
def draw_all(self, errors):
fig, ax = plt.subplots(figsize=(6,4))
ax.plot(range(len(errors['train_scores'])) ,[i[1] for i in errors['train_scores']], label='train')
 ax.plot(range(len(errors['test_scores'])) ,[i[1] for i in errors['test_scores']], label='test')
 ax.legend()
plt.show()
def start_tuning(self, num, mode=0):
self.mode=mode
print("loading")
 start_time = time.time()
base_params = dict(self.base_params)
 steps = {self.Step_0_find_n_estimators:[100, 200, 400, 800, 1600, 3200],
self.Step_05_find_n_estimators:None,
self.Step_1_find_depth_and_child:{'max_depth': list(range(10,20,2)) ,
'num_leaves': [20, 31, 40],
'min_data_in_leaf':[10, 15, 20, 25, 30]},
self.Step_2_narrow_depth: None,
self.Step_4_sample: {'bagging_fraction':[0.2, 0.4, 0.6, 0.8],
'feature_fraction': [0.2, 0.4, 0.6, 0.8]},
self.Step_5_reg1:{'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100],
'reg_lambda':[1e-5, 1e-2, 0.1, 1, 100]},
self.Step_6_eta_nround:{'learning_rate':[0.001, 0.005, 0.01, 0.05, 0.1, 0.3]}
}
for i,key in enumerate(steps.keys()):
if i in range(num):
e = ProcessPoolExecutor
result = key(steps[key])
 self.draw_all(result)
 best_param = min(result['test_scores'], key=lambda x:x[1])
 print('best selected param:', best_param)
 self.update(self.base_params, best_param[0])
 print(f'params after step {i}:', self.base_params)
 print("FINAL RESUTL: {}".format(self.base_params))
elapsed_time = time.time() - start_time
print('done')
 tune_l = Tune_ligthGBM(train_X_robust, train_y, test_X_robust )
test_input = prepare_input_v2(test_csv[["premise", "hypothesis"]].values.tolist())
 test_dataset = get_dataset(test_input, None, labelled=False, batch_size=BATCH_SIZE, repeat=False, shuffle=False )<predict_on_test>
%%script false --no-raise-error
tune_l.start_tuning(10, mode=1 )
preds = model.predict(test_dataset )<prepare_output>
%%script false --no-raise-error
tune.start_tuning(10, mode=1)
preds = preds.argmax(axis=1 )<load_from_csv>
my_local1 = {'objective': 'reg:linear',
'learning_rate': 0.01,
'seed': 42,
'n_estimators': 3200,
'max_depth': 5,
'min_child_weight': 9,
'gamma': 0.0,
'subsample': 0.6,
'colsample_bytree': 0.6,
'reg_alpha': 0.01}
my_local_eval={'n_estimators': 4571, 'num_round': 50, 'max_depth': 14, 'min_child_weight': 1, 'gamma': 0.0, 'subsample': 0.9, 'colsample_bytree': 0.6, 'reg_alpha': 0.01,
'reg_lambda': 1e-05, 'learning_rate': 0.0005, 'objective': 'reg:linear', 'seed': 42, 'verbosity': 0}
my_local_test_1_log = {'n_estimators': 4571, 'num_round': 50, 'max_depth': 6, 'min_child_weight': 11, 'gamma': 0.1, 'subsample': 0.8, 'colsample_bytree': 0.6, 'reg_alpha': 0.01,
'reg_lambda': 0.1, 'learning_rate': 0.01, 'objective': 'reg:linear', 'seed': 42, 'verbosity': 0}
submission = pd.read_csv(".. /input/contradictory-my-dear-watson/sample_submission.csv")
 submission.head()<prepare_output>
my_local_l2 = {'n_estimators': 6400,
'max_depth': -1,
'num_leaves': 40,
'min_data_in_leaf': 15,
'bagging_fraction': 0.6,
'bagging_freq': 5,
'bagging_seed': 7,
'feature_fraction': 0.8,
'feature_fraction_seed': 7,
'reg_alpha': 1e-05,
'reg_lambda': 1,
'learning_rate': 0.01,
'objective': 'regression',
'max_bin': 200,
'verbose': -1}
my_local_l4 = {'n_estimators': 2909, 'max_depth': 8, 'num_leaves': 20, 'min_data_in_leaf': 20,
'bagging_fraction': 0.8, 'bagging_freq': 5, 'bagging_seed': 7,
'feature_fraction': 0.6, 'feature_fraction_seed': 7, 'reg_alpha': 0.1,
'reg_lambda': 1, 'learning_rate': 0.01,
'objective': 'regression', 'max_bin': 200, 'verbose': -1}
my_local_l5 = {'n_estimators': 1777, 'max_depth': 10, 'num_leaves': 40, 'min_data_in_leaf': 10, 'bagging_fraction': 0.6, 'bagging_freq': 5, 'bagging_seed': 7,
'feature_fraction': 0.8, 'feature_fraction_seed': 7, 'reg_alpha': 1e-05, 'reg_lambda': 1e-05, 'learning_rate': 0.005, 'objective': 'regression',
'max_bin': 200, 'verbose': -1}
test_l = {'n_estimators': 6400, 'max_depth': 17, 'num_leaves': 40, 'min_data_in_leaf': 15, 'bagging_fraction': 0.8, 'bagging_freq': 5, 'bagging_seed': 7,
'feature_fraction': 0.4, 'feature_fraction_seed': 7, 'reg_alpha': 0.1, 'reg_lambda': 0.1, 'learning_rate': 0.05, 'objective': 'regression',
'max_bin': 200, 'verbose': -1}
submission["prediction"] = preds<save_to_csv>
base = 0.39823
my_first = {'l1_ratio': 0.17, 'alpha': 0.01, 'max_iter': 12}
my_second = {'max_iter': 500.0, 'l1_ratio': 0.16, 'alpha': 0.01}
my_therd = {'max_iter': 50.0, 'l1_ratio': 0.16, 'alpha': 0.01}
test_el = {'max_iter': 50.0, 'l1_ratio': 0.16, 'alpha': 0.01}
submission.to_csv("submission.csv", header=True, index=False )<set_options>
%%script false --no-raise-error
parametersGrid = {"max_iter": [5, 10],
"alpha": np.arange(0.14, 0.18, 0.005),
"l1_ratio": np.arange(0.5, 0.62, 0.005)}
model = ElasticNet()
grid_search = GridSearchCV(model, parametersGrid, verbose=0)
 grid_search.fit(train_X, train_y)
 best_params = grid_search.best_params_
all_params = grid_search.cv_results_['params']
scores = grid_search.cv_results_['mean_test_score']
predict = grid_search.predict(test_X)
 plot_functions_from_params(all_params, scores)
 best_params
os.environ["WANDB_API_KEY"] = "0"
!pip uninstall -y transformers
!pip install transformers
!pip install nlp
strategy = None
def seed_all(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed_all(2020 )<load_from_csv>
%%script false --no-raise-error
parametersGrid = {"max_iter": [5, 10],
"alpha": np.arange(0.1, 0.2, 0.005),
"l1_ratio": np.arange(0.3, 0.45, 0.005)}
model = ElasticNet()
grid_search_robust = GridSearchCV(model, parametersGrid, verbose=0)
 grid_search_robust.fit(train_X_robust, train_y)
 best_params_rubost = grid_search_robust.best_params_
all_params_rubost = grid_search_robust.cv_results_['params']
scores_rubost = grid_search_robust.cv_results_['mean_test_score']
predict = grid_search_robust.predict(test_X_robust)
 plot_functions_from_params(all_params_rubost, scores_rubost)
 best_params_rubost
original_train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 original_train = shuffle(original_train)
 original_valid = original_train[:len(original_train)// 5]
original_train = original_train[len(original_train)// 5:]<train_model>
%%script false --no-raise-error
xg_model = xgb.XGBRegressor(**my_local_test_1_log, n_jobs = 4)
print(f"original - training: {len(original_train)} examples")
 original_train.head(10 )<train_model>
lightgbm = lgb.LGBMRegressor(**test_l, n_jobs = 4)
 lightgbm.fit(train_X, np.log1p(train_y))
predict_l = lightgbm.predict(test_X)
 predict_l = np.expm1(predict_l )
print(f"original - validation: {len(original_valid)} examples")
 original_valid.head(10 )<load_from_csv>
%%script false --no-raise-error
elastic = ElasticNet(**test_el)
 elastic_pipe = make_pipeline(RobustScaler() , elastic)
original_test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 print(f"original - test: {len(original_test)} examples")
 original_test.head(10 )<create_dataframe>
%%script false --no-raise-error
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def cv_rmse(model, X=train_X, y=np.log1p(train_y)) :
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse)
 alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
lasso_pipe = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds,n_jobs=4))
mnli = nlp.load_dataset(path='glue', name='mnli' )<train_model>
%%script false --no-raise-error
meta_xgboost = xgb.XGBRegressor(n_jobs=4)
 stack = StackingCVRegressor(regressors=[elastic_pipe, lightgbm, xg_model],meta_regressor=meta_xgboost)
 stack.fit(np.array(train_X), np.log1p(train_y))
predict_s = stack.predict(np.array(test_X))
predict_s = np.expm1(predict_s )
print(mnli, '
')
 print('The split names in MNLI dataset:')
 for k in mnli:
print(' ', k)
 print("
mnli['train'] is ", type(mnli['train']))
mnli['train']<define_variables>
predict = predict_l
print('The number of training examples in mnli dataset:', mnli['train'].num_rows)
 print('The number of validation examples in mnli dataset - part 1:', mnli['validation_matched'].num_rows)
 print('The number of validation examples in mnli dataset - part 2:', mnli['validation_mismatched'].num_rows, '
')
 print('The class names in mnli dataset:', mnli['train'].features['label'].names)
 print('The feature names in mnli dataset:', list(mnli['train'].features.keys()), '
')
 for elt in mnli['train']:
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', mnli['train'].features['label'].names[elt['label']])
 print('idx', elt['idx'])
 print('-' * 80)
 if elt['idx'] >= 10:
break<define_variables>
sub = pd.concat([test_ids, pd.Series(predict)], axis=1)
 sub = sub.rename(columns={0:'SalePrice'})
 sub.to_csv('sumbission.csv', index=False, header=True )