Kaggle competition: House Prices - Advanced Regression Techniques
gc.enable()
folds = KFold(n_splits=4, shuffle=True, random_state=546789)
 oof_preds = np.zeros(data.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in data.columns if f not in ['SK_ID_CURR']]
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(data)) :
trn_x, trn_y = data[feats].iloc[trn_idx], y.iloc[trn_idx]
val_x, val_y = data[feats].iloc[val_idx], y.iloc[val_idx]
clf = LGBMClassifier(
 n_estimators=10000,
learning_rate=0.03,
num_leaves=34,
colsample_bytree=0.9,
subsample=0.8,
max_depth=8,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.01,
min_child_weight=300,
silent=-1,
verbose=-1, 
)
 clf.fit(trn_x, trn_y,
eval_set= [(trn_x, trn_y),(val_x, val_y)],
eval_metric='auc', verbose=100, early_stopping_rounds=100 
)
 oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
test['TARGET'] = sub_preds
test[['SK_ID_CURR', 'TARGET']].to_csv('submission1LGBM1.csv', index=False)
<load_from_csv>
from fastai_structured import *
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import ExtraTreesRegressor
from catboost import CatBoostRegressor
from sklearn.model_selection import RandomizedSearchCV
import matplotlib.pyplot as plt
import seaborn as sns
import altair as alt
M1 = pd.read_csv('.. /input/gogomaster/Think1.csv')
 M2 = pd.read_csv('.. /input/gogomaster/11111.csv')
 M3 = pd.read_csv('.. /input/gogomaster/22222.csv')
 M4 = pd.read_csv('.. /input/gogomaster/33333.csv')
 M5 = pd.read_csv('.. /input/gogomaster/44444.csv')
 M6 = pd.read_csv('.. /input/gogomaster/55555.csv' )<merge>
df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df.shape
def merge_dataframes(dfs, merge_keys):
dfs_merged = reduce(lambda left,right: pd.merge(left, right, on=merge_keys), dfs)
 return dfs_merged<merge>
df.dtypes.value_counts()
dfs = [M1,M2,M3,M4,M5,M6]
merge_keys=['SK_ID_CURR']
df = merge_dataframes(dfs, merge_keys=merge_keys )<rename_columns>
df.select_dtypes(['object'] ).isna().sum()
df.columns = ['SK_ID_CURR','T1','T2','T3','T4','T5','T6']
df.head()<define_variables>
df.select_dtypes(['float64'] ).isna().sum()
pred_prob = 0.7 * df['T3'] + 0.3 * df['T1']
pred_prob.head()<prepare_output>
df['TotalSqFeet'] = df.GrLivArea + df.TotalBsmtSF
df['TotBathrooms'] = df.FullBath +(df.HalfBath)*0.5 + df.BsmtFullBath +(df.BsmtHalfBath)*0.5
df.set_index('Id',inplace=True)
sub = pd.DataFrame()
sub['SK_ID_CURR'] = df['SK_ID_CURR']
sub['target']= pred_prob<save_to_csv>
df.groupby("Neighborhood")["SalePrice"].mean().sort_values(ascending=False ).index
sub.to_csv('ldit.csv', index=False )<define_variables>
df.groupby("MSSubClass")["SalePrice"].mean().sort_values(ascending=False )
B_prob = 0.6 * df['T3'] + 0.2 * df['T1'] + 0.2 * df['T2']<save_to_csv>
train_cats(df)
SUB = pd.DataFrame()
SUB['SK_ID_CURR'] = df['SK_ID_CURR']
SUB['TARGET'] = B_prob
SUB.to_csv('Blendss.csv', index=False )<define_variables>
test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 test['TotalSqFeet'] = test.GrLivArea + test.TotalBsmtSF
test['TotBathrooms'] = test.FullBath +(test.HalfBath)*0.5 + test.BsmtFullBath +(test.BsmtHalfBath)*0.5
test.set_index('Id', inplace=True)
 apply_cats(test, df )
corr_pred = 0.6 * df['T2'] + 0.05 * df['T3'] + 0.05 * df['T4'] + 0.1 * df['T5'] + 0.2 * df['T1']
corr_pred.head()<save_to_csv>
test.Neighborhood = test.Neighborhood.cat.codes
df.Neighborhood = df.Neighborhood.cat.codes
SuB = pd.DataFrame()
SuB['SK_ID_CURR'] = df['SK_ID_CURR']
SuB['TARGET'] = corr_pred
SuB.to_csv('corr_blend.csv', index=False )<import_modules>
display_all(df.isna().sum() /len(df)*100 )
import numpy as np
import pandas as pd
import seaborn as sns
import tensorflow as tf
import itertools
import os
import tflearn
from matplotlib import pyplot as plt
from sklearn.decomposition.kernel_pca import KernelPCA
from sklearn.metrics import classification_report
from sklearn.preprocessing import PolynomialFeatures<set_options>
X, y, nas = proc_df(df, 'SalePrice')
 X_test, _, _ = proc_df(test, na_dict=nas )
%matplotlib inline<load_from_csv>
X.skew(axis=0 ).sort_values(ascending=False )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission["type"] = "Unknown"<choose_model_class>
numeric_feats = df.select_dtypes(include=np.number)
 skewed_feats = [numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10)
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )
poly_features = PolynomialFeatures(interaction_only=True )<feature_engineering>
y_first = np.log1p(df.SalePrice )
try_comb = pd.DataFrame(
 poly_features.fit_transform(train.drop(["id", "color", "type"], axis=1)) [:,5:],
columns=["boneXrotting", "boneXhair", "boneXsoul",
"rottingXhair", "rottingXsoul",
"hairXsoul"] 
)
 try_comb["type"] = train.type<prepare_output>
y = np.log1p(y_first )
for i in ["boneXhair", "boneXsoul", "hairXsoul"]:
train[i] = try_comb[i].copy()
try_comb = None<create_dataframe>
display_all(X.isna().sum() /len(X)*100 )
try_comb = pd.DataFrame(
 poly_features.fit_transform(test.drop(["id", "color"], axis=1)) [:,5:],
columns=["boneXrotting", "boneXhair", "boneXsoul",
"rottingXhair", "rottingXsoul",
"hairXsoul"] 
)
 for i in ["boneXhair", "boneXsoul", "hairXsoul"]:
test[i] = try_comb[i].copy()
try_comb = None<data_type_conversions>
xgb = XGBRegressor(n_estimators=10000)
 xgb.fit(X, y, verbose=False)
 xgb.score(X, y )
train.loc[train.type == "Ghost", "type"] = 0
train.loc[train.type == "Ghoul", "type"] = 1
train.loc[train.type == "Goblin", "type"] = 2
train["type"] = train["type"].astype("int" )<define_variables>
m = CatBoostRegressor(random_state=0, verbose=1000, n_estimators=100000, learning_rate=0.007, early_stopping_rounds=5)
 m.fit(X, y)
 m.score(X, y )
is_ghost =(train.type == 0 ).values
is_ghoul =(train.type == 1 ).values
is_goblin =(train.type == 2 ).values<filter>
m = CatBoostRegressor(random_state=0, verbose=1000, n_estimators=10000)
 m.fit(X, y)
 m.score(X, y )
ghost_ghoul = train.loc[is_ghost | is_ghoul].copy()
ghost_goblin = train.loc[is_ghost | is_goblin].copy()
ghoul_goblin = train.loc[is_ghoul| is_goblin].copy()<feature_engineering>
m.save_model('hp_cb',
format="cbm",
export_parameters=None,
pool=None )
ghost_ghoul["KPCA_0"] = ghost_ghoul_transf[:,0]
ghost_ghoul["KPCA_1"] = ghost_ghoul_transf[:,1]
ghost_goblin["KPCA_0"] = ghost_goblin_transf[:,0]
ghost_goblin["KPCA_1"] = ghost_goblin_transf[:,1]
ghoul_goblin["KPCA_0"] = ghoul_goblin_transf[:,0]
ghoul_goblin["KPCA_1"] = ghoul_goblin_transf[:,1]<compute_test_metric>
m = RandomForestRegressor(n_jobs=-1, n_estimators=60, random_state=0)
 m.fit(X, y)
 m.score(X, y )
def neural_net(X_train, y_train, X_test, layers=[1024], dropout=0.8, n_epoch=30):
if isinstance(X_train, pd.DataFrame):
X_train = X_train.values
if isinstance(y_train, pd.DataFrame):
y_train = y_train.values
if isinstance(X_test, pd.DataFrame):
X_test = X_test.values
with tf.Graph().as_default() :
net = tflearn.input_data(shape=[None, X_train.shape[1]])
 for layer_size in layers:
net = tflearn.fully_connected(net, layer_size,
activation='relu',
weights_init='xavier',
regularizer='L2')
 net = tflearn.dropout(net, dropout)
 net = tflearn.fully_connected(net, y_train.shape[1], activation='softmax')
 net = tflearn.regression(net)
 model = tflearn.DNN(net, tensorboard_verbose=0)
 model.fit(X_train, y_train, validation_set=0.2, n_epoch=n_epoch)
 probs = np.array(model.predict(X_test))
return probs<prepare_x_and_y>
et = ExtraTreesRegressor(n_jobs=-1, n_estimators=500, random_state=0, max_features=0.5)
 et.fit(X, y)
 et.score(X, y )
X_train = ghost_ghoul.drop(["id", "color", "type"], axis=1)
 y_train = pd.get_dummies(ghost_ghoul["type"])
 X_test = test.drop(["id", "color"], axis=1)
 transf = ghost_ghoul_KPCA.transform(X_test)
 X_test["KPCA_0"] = transf[:,0]
X_test["KPCA_1"] = transf[:,1]<compute_train_metric>
m = ExtraTreesRegressor(n_jobs=-1, n_estimators=500, random_state=0, max_features=0.5)
 m.fit(X, y)
 m.score(X, y )
ghost_ghoul_probs = neural_net(X_train, y_train, X_test, dropout=0.8, n_epoch=40 )<prepare_x_and_y>
import shap
from catboost import Pool
X_train = ghost_goblin.drop(["id", "color", "type"], axis=1)
 y_train = pd.get_dummies(ghost_goblin["type"])
 X_test = test.drop(["id", "color"], axis=1)
 transf = ghost_goblin_KPCA.transform(X_test)
 X_test["KPCA_0"] = transf[:,0]
X_test["KPCA_1"] = transf[:,1]<compute_train_metric>
pd.set_option('display.max_rows', 100 )
ghost_goblin_probs = neural_net(X_train, y_train, X_test, layers=[512], dropout=0.8, n_epoch=60 )<prepare_x_and_y>
shap_values = m.get_feature_importance(Pool(X, y), type='ShapValues' )
X_train = ghoul_goblin.drop(["id", "color", "type"], axis=1)
 y_train = pd.get_dummies(ghoul_goblin["type"])
 X_test = test.drop(["id", "color"], axis=1)
 transf = ghoul_goblin_KPCA.transform(X_test)
 X_test["KPCA_0"] = transf[:,0]
X_test["KPCA_1"] = transf[:,1]<compute_train_metric>
m.get_feature_importance(Pool(X, y), type='LossFunctionChange', prettified=True )
ghoul_goblin_probs = neural_net(X_train, y_train, X_test, dropout=0.5, n_epoch=40 )<define_variables>
max_features = ['sqrt', 'log2', 0.5, None]
min_samples_leaf = [1, 3, 5, 10, 25, 100]
random_grid = {'min_samples_leaf': min_samples_leaf,
'max_features': max_features}
print(random_grid )
global_predictions = np.zeros(( X_test.values.shape[0], 3))
global_predictions[:,[0,1]] += ghost_ghoul_probs
global_predictions[:,[0,2]] += ghost_goblin_probs
global_predictions[:,[1,2]] += ghoul_goblin_probs<data_type_conversions>
rf = RandomForestRegressor(n_estimators=100)
 rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid,
n_iter = 20, cv = 5, verbose=2, random_state=10, n_jobs=-1 )
test["global_pred"] = np.argmax(global_predictions, axis=1 ).astype("str")
 test.loc[test.global_pred == "0", "global_pred"] = "Ghost"
test.loc[test.global_pred == "1", "global_pred"] = "Ghoul"
test.loc[test.global_pred == "2", "global_pred"] = "Ghoblin"<save_to_csv>
search = rf_random.fit(X, y)
 print('Par:', search.best_params_)
 print('Score:', search.best_score_ )
submission["type"] = test["global_pred"]
submission.to_csv("sub.csv", index=False )<prepare_x_and_y>
y_pred_xgb = xgb.predict(X_test )
y = df["type"]
indexes_test = df_test["id"]
df = df.drop(["type","color","id"],axis=1)
 df_test = df_test.drop(["color","id"],axis=1 )<categorify>
y_pred_cb = m.predict(X_test )
df = pd.get_dummies(df)
 df_test = pd.get_dummies(df_test )<split>
y_pred = 9/10 * y_pred_cb + 1/10 * y_pred_et
X_train, X_test, y_train, y_test = train_test_split(df, y, test_size=0.3, random_state=0 )<train_model>
y_pred = y_pred_cb
lr = LogisticRegression(penalty='l2',C=1000000)
 lr.fit(X_train,y_train)
 y_pred= lr.predict(X_test)
 print(classification_report(y_pred,y_test))<save_to_csv>
y_pred = rf_random.predict(X_test )
y_pred = lr.predict(df_test)
 Y = pd.DataFrame()
Y["id"] = indexes_test
Y["type"] = y_pred
Y.to_csv("submission.csv",index=False )<set_options>
y_first = np.expm1(y_pred )
% matplotlib inline 
<load_from_csv>
y_pred = np.expm1(y_first )
test = pd.read_csv('.. /input/nomad2018-predict-transparent-conductors/test.csv')
 test_id = test.id
train = pd.read_csv('.. /input/nomad2018-predict-transparent-conductors/train.csv' )<create_dataframe>
my_submission = pd.DataFrame({'Id': test.index, 'SalePrice': y_pred} )
ga_cols = []
al_cols = []
o_cols = []
in_cols = []
warnings.filterwarnings("ignore")
 for i in range(6):
ga_cols.append("Ga_"+str(i))
for i in range(6):
al_cols.append("Al_"+str(i))
for i in range(6):
o_cols.append("O_"+str(i))
for i in range(6):
in_cols.append("In_"+str(i))
ga_df= pd.DataFrame(columns=ga_cols)
 al_df = pd.DataFrame(columns=al_cols)
 o_df = pd.DataFrame(columns= o_cols)
 in_df = pd.DataFrame(columns=in_cols)
 def get_xyz_data(filename):
pos_data = []
lat_data = []
with open(filename)as f:
for line in f.readlines() :
x = line.split()
if x[0] == 'atom':
pos_data.append([np.array(x[1:4], dtype=np.float),x[4]])
 elif x[0] == 'lattice_vector':
lat_data.append(np.array(x[1:4], dtype=np.float))
return pos_data, np.array(lat_data)
 for i in train.id.values:
fn = ".. /input/nomad2018-predict-transparent-conductors/train/{}/geometry.xyz".format(i)
 train_xyz, train_lat = get_xyz_data(fn)
 ga_list = []
al_list = []
o_list = []
in_list = []
for li in train_xyz:
try:
if li[1] == "Ga":
ga_list.append(li[0])
 except:
pass
try:
if li[1] == "Al":
al_list.append(li[0])
 except:
pass
try:
if li[1] == "In":
in_list.append(li[0])
 except:
pass
try:
if li[1] == "O":
o_list.append(li[0])
 except:
pass
try:
model = PCA(n_components=2)
 ga_list = np.array(ga_list)
 temp_ga = model.fit_transform(ga_list.transpose())
 temp_ga = [item for sublist in temp_ga for item in sublist]
except:
temp_ga = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 al_list = np.array(al_list)
 temp_al = model.fit_transform(al_list.transpose())
 temp_al = [item for sublist in temp_al for item in sublist]
except:
temp_al = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 o_list = np.array(o_list)
 temp_o = model.fit_transform(o_list.transpose())
 temp_o = [item for sublist in temp_o for item in sublist]
except:
temp_o = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 in_list = np.array(in_list)
 temp_in = model.fit_transform(in_list.transpose())
 temp_in = [item for sublist in temp_in for item in sublist]
except:
temp_in = [0,0,0,0,0,0]
temp_ga = pd.DataFrame(temp_ga ).transpose()
temp_ga.columns = ga_cols
temp_ga.index = np.array([i])
 temp_al = pd.DataFrame(temp_al ).transpose()
temp_al.columns = al_cols
temp_al.index = np.array([i])
 temp_o = pd.DataFrame(temp_o ).transpose()
temp_o.columns = o_cols
temp_o.index = np.array([i])
 temp_in = pd.DataFrame(temp_in ).transpose()
temp_in.columns = in_cols
temp_in.index = np.array([i])
 ga_df = pd.concat([ga_df,temp_ga])
 al_df = pd.concat([al_df,temp_al])
 o_df = pd.concat([o_df,temp_o])
 in_df = pd.concat([in_df,temp_in])
 ga_df["id"] = ga_df.index
al_df["id"] = al_df.index
o_df["id"] = o_df.index
in_df["id"] = in_df.index
train = pd.merge(train,ga_df,on = ["id"],how = "left")
 train = pd.merge(train,al_df,on = ["id"],how = "left")
 train = pd.merge(train,o_df,on = ["id"],how = "left")
 train = pd.merge(train,in_df,on = ["id"],how = "left")
 ga_df= pd.DataFrame(columns=ga_cols)
 al_df = pd.DataFrame(columns=al_cols)
 o_df = pd.DataFrame(columns= o_cols)
 in_df = pd.DataFrame(columns=in_cols)
 for i in test.id.values:
fn = ".. /input/nomad2018-predict-transparent-conductors/test/{}/geometry.xyz".format(i)
 test_xyz, test_lat = get_xyz_data(fn)
 ga_list = []
al_list = []
o_list = []
in_list = []
for li in test_xyz:
try:
if li[1] == "Ga":
ga_list.append(li[0])
 except:
pass
try:
if li[1] == "Al":
al_list.append(li[0])
 except:
pass
try:
if li[1] == "In":
in_list.append(li[0])
 except:
pass
try:
if li[1] == "O":
o_list.append(li[0])
 except:
pass
try:
model = PCA(n_components=2)
 ga_list = np.array(ga_list)
 temp_ga = model.fit_transform(ga_list.transpose())
 temp_ga = [item for sublist in temp_ga for item in sublist]
except:
temp_ga = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 al_list = np.array(al_list)
 temp_al = model.fit_transform(al_list.transpose())
 temp_al = [item for sublist in temp_al for item in sublist]
except:
temp_al = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 o_list = np.array(o_list)
 temp_o = model.fit_transform(o_list.transpose())
 temp_o = [item for sublist in temp_o for item in sublist]
except:
temp_o = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 in_list = np.array(in_list)
 temp_in = model.fit_transform(in_list.transpose())
 temp_in = [item for sublist in temp_in for item in sublist]
except:
temp_in = [0,0,0,0,0,0]
temp_ga = pd.DataFrame(temp_ga ).transpose()
temp_ga.columns = ga_cols
temp_ga.index = np.array([i])
 temp_al = pd.DataFrame(temp_al ).transpose()
temp_al.columns = al_cols
temp_al.index = np.array([i])
 temp_o = pd.DataFrame(temp_o ).transpose()
temp_o.columns = o_cols
temp_o.index = np.array([i])
 temp_in = pd.DataFrame(temp_in ).transpose()
temp_in.columns = in_cols
temp_in.index = np.array([i])
 ga_df = pd.concat([ga_df,temp_ga])
 al_df = pd.concat([al_df,temp_al])
 o_df = pd.concat([o_df,temp_o])
 in_df = pd.concat([in_df,temp_in])
 ga_df["id"] = ga_df.index
al_df["id"] = al_df.index
o_df["id"] = o_df.index
in_df["id"] = in_df.index
test = pd.merge(test,ga_df,on = ["id"],how = "left")
 test = pd.merge(test,al_df,on = ["id"],how = "left")
 test = pd.merge(test,o_df,on = ["id"],how = "left")
 test = pd.merge(test,in_df,on = ["id"],how = "left" )<rename_columns>
my_submission.to_csv('submission.csv', index=False )
train.rename(columns={
'spacegroup' : 'sg',
'number_of_total_atoms' : 'Natoms',
'percent_atom_al' : 'x_Al',
'percent_atom_ga' : 'x_Ga',
'percent_atom_in' : 'x_In',
'lattice_vector_1_ang' : 'a',
'lattice_vector_2_ang' : 'b',
'lattice_vector_3_ang' : 'c',
'lattice_angle_alpha_degree' : 'alpha',
'lattice_angle_beta_degree' : 'beta',
'lattice_angle_gamma_degree' : 'gamma',
'formation_energy_ev_natom' : 'E',
'bandgap_energy_ev' : 'Eg'}, inplace=True)
 test.rename(columns={
'spacegroup' : 'sg',
'number_of_total_atoms' : 'Natoms',
'percent_atom_al' : 'x_Al',
'percent_atom_ga' : 'x_Ga',
'percent_atom_in' : 'x_In',
'lattice_vector_1_ang' : 'a',
'lattice_vector_2_ang' : 'b',
'lattice_vector_3_ang' : 'c',
'lattice_angle_alpha_degree' : 'alpha',
'lattice_angle_beta_degree' : 'beta',
'lattice_angle_gamma_degree' : 'gamma',
}, inplace=True)
 target = [
'E',
'Eg']
all_data = pd.concat(( train, test))<define_variables>
processed = X.copy()
processed['SalePrice'] = y
def get_prop_list(path_to_element_data):

 return [f[:-4] for f in os.listdir(path_to_element_data)]
path_to_element_data = '.. /input/elemental-properties/'
properties = get_prop_list(path_to_element_data)
 print(sorted(properties))<define_variables>
def split_vals(a,n): return a[:n].copy() , a[n:].copy()
n_valid = int(len(processed)*0.2)
 n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(processed, n_trn)
 X_train, X_valid = split_vals(X, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 X_train.shape, y_train.shape, X_valid.shape
def get_prop(prop, path_to_element_data):

 fin = os.path.join(path_to_element_data, prop+'.csv')
 with open(fin)as f:
all_els = {line.split(',')[0] : float(line.split(',')[1][:-1])for line in f}
my_els = ['Al', 'Ga', 'In']
return {el : all_els[el] for el in all_els if el in my_els}
prop_dict = {prop : get_prop(prop, path_to_element_data)for prop in properties}
print('The mass of aluminum is %.2f amu' % prop_dict['mass']['Al'] )<groupby>
def print_score(m):
res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )
def avg_prop(x_Al, x_Ga, x_In, prop):

 els = ['Al', 'Ga', 'In']
concentration_dict = dict(zip(els, [x_Al, x_Ga, x_In]))
return np.sum(prop_dict[prop][el] * concentration_dict[el] for el in els)
 for prop in properties:
all_data['_'.join(['avg', prop])] = avg_prop(all_data['x_Al'],
all_data['x_Ga'],
all_data['x_In'],
prop )<compute_test_metric>
print_score(m )
def get_vol(a, b, c, alpha, beta, gamma):

 return a*b*c*np.sqrt(1 + 2*np.cos(alpha)*np.cos(beta)*np.cos(gamma)
 - np.cos(alpha)**2
- np.cos(beta)**2
- np.cos(gamma)**2)
 lattice_angles = ['alpha', 'beta', 'gamma']
for lang in lattice_angles:
all_data['_'.join([lang, 'r'])] = np.pi * all_data[lang] / 180
all_data['vol'] = get_vol(all_data['a'], all_data['b'], all_data['c'],
all_data['alpha_r'], all_data['beta_r'], all_data['gamma_r'])
 all_data['atomic_density'] = all_data['Natoms'] / all_data['vol']<feature_engineering>
m = ExtraTreesRegressor(n_jobs=-1, bootstrap=True, oob_score=True)
 %time m.fit(X_train, y_train)
 print_score(m )
avg_cols = ['x_Al','x_Ga','x_In','a','b','c','avg_rs_max','avg_electronegativity',
'avg_rp_max','avg_LUMO','avg_IP','avg_rd_max','avg_EA','avg_HOMO',
'avg_mass','vol','atomic_density']
for col in avg_cols:
new_col = col + "_avg"
all_data[new_col] = np.nan
for group in all_data['sg'].unique() :
all_data.loc[(all_data['sg'] == group), new_col] = all_data[(all_data['sg'] == group)][col].mean()
print('Number of Null Values: {}'.format(pd.isnull(all_data[avg_cols] ).sum().sum()))<categorify>
m = RandomForestRegressor(n_jobs=-1, oob_score=True)
 %time m.fit(X_train, y_train)
 print_score(m )
all_data[['sg', 'Natoms']] = all_data[['sg', 'Natoms']].astype(str)
 all_data = pd.get_dummies(all_data )<define_variables>
df_trn, y_trn, nas = proc_df(df, 'SalePrice')
 X_train, _ = split_vals(df_trn, n_trn)
 y_train, _ = split_vals(y_trn, n_trn)
 X_train.shape, y_train.shape, X_valid.shape
features = ['x_Al', 'x_Ga', 'x_In', 'a', 'b', 'c', 'alpha', 'beta',
'gamma', 'vol', 'atomic_density', 'x_Al_avg','x_Ga_avg', 'x_In_avg', 'a_avg',
'b_avg', 'c_avg', 'vol_avg', 'atomic_density_avg', 'pca_abc', 'pca_AlGaInDensity',
'O_0_0','O_1_0', 'O_2_0', 'O_3_0', 'O_4_0', 'O_5_0', 'Al_0_0', 'Al_1_0', 'Al_2_0', 'Al_3_0', 'Al_4_0', 'Al_5_0', 'Ga_0_0',
'Ga_1_0', 'Ga_2_0', 'Ga_3_0', 'Ga_4_0', 'Ga_5_0', 'In_0_0', 'In_1_0',
'In_2_0', 'In_3_0', 'In_4_0', 'In_5_0',]
vector1 = all_data[['a', 'b', 'c']].values
vector2 = all_data[['x_Al', 'x_Ga', 'x_In', 'atomic_density_avg']].values
pca = PCA()
pca.fit(vector1)
 all_data['pca_abc'] = pca.transform(vector1)[:,0]
pca = PCA()
pca.fit(vector2)
 all_data['pca_AlGaInDensity'] = pca.transform(vector2)[:,0]
scale = StandardScaler()
scaled = scale.fit(all_data[features] ).transform(all_data[features])
 X_scale = scaled[:train.shape[0]]
X_scaled_test = scaled[train.shape[0]:]
X_tr = all_data[:train.shape[0]][features].values
X_te = all_data[train.shape[0]:][features].values
y1 = np.log1p(train['E'])
 y2 = np.log1p(train['Eg'])
 y12 = np.column_stack(( y1, y2))
X_tr.shape, y1.shape, y2.shape, y12.shape, X_scaled_test.shape<compute_test_metric>
??RandomForestRegressor
def rmsle(h, y):

 return np.sqrt(np.square(np.log(h + 1)- np.log(y + 1)).mean() )<import_modules>
m = RandomForestRegressor(n_estimators=10, max_depth=4, bootstrap=False, n_jobs=-1)
 m.fit(X_train, y_train)
 print_score(m )
import keras
import tensorflow as tf
import keras.backend as K
from keras.regularizers import l2
from keras.optimizers import Adam
from keras.models import Model
from keras.models import Sequential
from keras.layers import Dense,Dropout,BatchNormalization, Input<choose_model_class>
m = ExtraTreesRegressor(n_jobs=-1, n_estimators=500, bootstrap=True, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
def get_model(shape):

 X_input = Input(shape=(shape,))
X = Dense(64, activation='relu' )(X_input)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(2, activation='linear' )(X)
 return Model(inputs=X_input, outputs=X)
 def rmsle_K(y, y0):
return K.sqrt(K.mean(K.square(tf.log1p(tf.expm1(y)) - tf.log1p(tf.expm1(y0)))))
 def compile_model(shape, lr=0.001):
model = get_model(shape)
 optimizer = Adam(lr=lr, decay=0.001)
 model.compile(optimizer=optimizer, loss='mse', metrics=[rmsle_K])
 return model<train_model>
preds = np.stack([t.predict(X_valid)for t in m.estimators_])
def assess_nn(X, y, params):

 final = []
best_iter = [[], []]
kfold = KFold(n_splits=2, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = compile_model(X_scale.shape[1], lr=0.003)
 model.fit(x=X_scale, y=y12, epochs=600, batch_size=16, verbose=0)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print(e)
 out.append(e)
 final.append(np.array(out ).mean())
 print('y1 best iteration: {}'.format(np.mean(best_iter[0])))
 print('y2 best iteration: {}'.format(np.mean(best_iter[1])))
 return(np.array(final ).mean() , np.array(final ).std() )<import_modules>
from sklearn import *
from sklearn.model_selection import KFold
from sklearn.ensemble import GradientBoostingRegressor<choose_model_class>
plt.plot([metrics.r2_score(y_valid, np.mean(preds[:i+1], axis=0)) for i in range(1000)]);
grad_1 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=7,
n_estimators=1120,
max_features=7,
min_samples_leaf=43,
min_samples_split=14,
min_weight_fraction_leaf=0.01556)
 grad_2 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=6,
n_estimators=3275,
max_features=2,
min_samples_leaf=2,
min_samples_split=2,
min_weight_fraction_leaf=0.08012)
 def assess_grad(X, y_list, model_list):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_list[idx]
model.fit(X_train, y_train)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print(e)
 out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )<compute_test_metric>
??RandomForestRegressor
model = assess_grad(X_tr, [y1, y2], [grad_1, grad_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))<import_modules>
m = RandomForestRegressor(bootstrap=True, n_jobs=-1)
 m.fit(X_train, y_train)
 print_score(m )
import lightgbm as lgb
from lightgbm import Dataset<choose_model_class>
set_rf_samples(50000 )
lgb_1 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2000,
num_threads=3,
bagging_fraction=0.56369,
bagging_freq= 14,
feature_fraction= 0.88868,
lambda_l2= 0.0091689,
max_depth=20, 
)
 lgb_2 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2838,
num_threads=3,
bagging_fraction=0.50173,
bagging_freq= 14,
feature_fraction= 0.62509,
lambda_l2= 0.0086298,
max_depth=20, 
)
 def assess_light(X, y_list, model_list):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_list[idx]
model.fit(X_train, y_train)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )<compute_test_metric>
def split_vals(a,n): return a[:n].copy() , a[n:].copy()
n_valid = int(len(processed)*0.2)
 n_trn = len(df)-n_valid
raw_train, raw_valid = split_vals(processed, n_trn)
 X_train, X_valid = split_vals(X, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 X_train.shape, y_train.shape, X_valid.shape
model = assess_light(X_scale, [y1, y2], [lgb_1, lgb_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))<split>
raw_train, raw_valid = split_vals(df, n_trn )
def assess_early_stop_light(X, y_list, params):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train)
 model = lgb.train(params[idx],
lgb_train,
num_boost_round=1000000,
valid_sets=[lgb_valid],
early_stopping_rounds=100,
verbose_eval=0)
 best_iter[idx].append(model.best_iteration)
 h = model.predict(X_test, num_iteration=model.best_iteration)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 print('y1 best iteration: {}'.format(np.mean(best_iter[0])))
 print('y2 best iteration: {}'.format(np.mean(best_iter[1])))
 return(np.array(final ).mean() , np.array(final ).std() )<init_hyperparams>
def get_preds(t): return t.predict(X_valid)
 %time preds = np.stack(parallel_trees(m, get_preds))
np.mean(preds[:,0]), np.std(preds[:,0] )
params1 = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'rmse', 'rmsle'},
'max_depth': 10,
'learning_rate': 0.010,
'feature_fraction': 0.8632,
'bagging_fraction': 0.8759,
'bagging_freq': 4,
'verbose': 0,
'verbose_eval':0,
'num_threads': 3,
'lambda_l2': 0.0005597442104287973,
'lambda_l1': 0.00015997163552092318
}
params2 = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'rmse', 'rmsle'},
'learning_rate': 0.010,
'verbose': 0,
'verbose_eval':0,
'num_threads': 3,
'bagging_fraction': 0.9311539021934098,
'bagging_freq': 15,
'feature_fraction': 0.9989744117209727,
'lambda_l1': 8.337666829263869e-05,
'lambda_l2': 0.005541689229153562,
'max_depth': 19
}
model = assess_early_stop_light(X_tr, [y1, y2], [params1, params2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))<import_modules>
x['pred_std'] = np.std(preds, axis=0)
 x['pred'] = np.mean(preds, axis=0)
 flds = ['Neighborhood', 'SalePrice', 'pred', 'pred_std']
nei_summ = x[flds].groupby('Neighborhood', as_index=False ).mean()
nei_summ.head()
from xgboost import XGBRegressor<init_hyperparams>
m = ExtraTreesRegressor(n_jobs=-1, n_estimators=100, bootstrap=True, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
<choose_model_class>
fi = rf_feat_importance(m, X )
xgb_1 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 1804,
gamma= 0.0,
subsample= 0.222159,
colsample_bytree= 0.5359,
colsample_bylevel= 0.19958,
max_delta_step= 64,
max_depth=28,
min_child_weight= 10,
reg_lambda=0.33038,
silent= True, 
)
 xgb_2 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 2386,
gamma= 0.0,
subsample= 0.90919,
colsample_bytree= 0.59049,
colsample_bylevel= 0.59404,
max_delta_step= 99,
max_depth=58,
min_child_weight= 85,
reg_lambda= 0.031165789070644215,
silent= True, 
)
 def assess_xgb(X, y_list, model_num):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_num[idx]
model.fit(X_train, y_train)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )<compute_test_metric>
fi = rf_feat_importance(m, df_trn )
model = assess_xgb(X_scale, [y1, y2], [xgb_1, xgb_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))<init_hyperparams>
df_keep = df_trn[to_keep].copy()
X_train, X_valid = split_vals(df_keep, n_trn )
xgb_param_1 = {
'learning_rate': 0.005,
'n_jobs': 3,
'gamma': 0.0,
'subsample': 0.222159,
'colsample_bytree': 0.5359,
'colsample_bylevel': 0.19958,
'max_delta_step': 64,
'max_depth': 28,
'min_child_weight': 10,
'reg_lambda': 0.33038,
'silent': True
}
xgb_param_2 = {
'learning_rate': 0.005,
'n_jobs': 3,
'gamma': 0.0,
'subsample': 0.90919,
'colsample_bytree': 0.59049,
'colsample_bylevel': 0.59404,
'max_delta_step': 99,
'max_depth': 58,
'min_child_weight': 85,
'reg_lambda': 0.031165789070644215,
'silent': True
}
def assess_early_cv_stop_xgb(X, y_list, param_list):

 final = []
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_iter, X_test = X[train_index], X[test_index]
y_iter, y_test = y[train_index], y[test_index]
X_train, X_valid, y_train, y_valid = train_test_split(X_iter, y_iter, test_size=0.3)
 xgb_train = DMatrix(X_train, label=y_train)
 xgb_valid = DMatrix(X_valid, y_valid)
 xgb_test = DMatrix(X_test, y_test)
 watchlist = [(xgb_train,'train'),(xgb_valid, 'valid')]
model = xgb.train(param_list[idx], xgb_train, 1000000,
watchlist, early_stopping_rounds=20, verbose_eval=False)
 h = model.predict(xgb_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )<compute_train_metric>
df_keep = X[to_keep].copy()
model = assess_early_cv_stop_xgb(X_tr, [y1, y2], [xgb_param_1, xgb_param_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))<import_modules>
m = ExtraTreesRegressor(n_jobs=-1, n_estimators=500, random_state=0)
 m.fit(df_keep, y )
from catboost import CatBoostRegressor<choose_model_class>
m = RandomForestRegressor(max_features=0.5,
n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )
cat_1 = CatBoostRegressor(iterations=2300,
learning_rate=0.020,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
od_type='Iter',
od_wait=50, 
)
 def assess_cat(X, y_list, model_num):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_num[idx]
model.fit(X_train, y_train, verbose=False)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )<compute_test_metric>
df.Neighborhood.value_counts()
model = assess_cat(X_tr, [y1, y2], [cat_1, cat_1])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))<choose_model_class>
df.MSSubClass.value_counts()
catboost_cv = CatBoostRegressor(iterations=1200,
learning_rate=0.03,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
random_seed=99,
od_type='Iter',
od_wait=50)
 def assess_cv_catboost(X, y_list):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_iter, X_test = X[train_index], X[test_index]
y_iter, y_test = y[train_index], y[test_index]
X_train, X_valid, y_train, y_valid = train_test_split(X_iter, y_iter, test_size=0.3)
 model = catboost_cv
model.fit(X_train, y_train,
eval_set=(X_valid, y_valid),
use_best_model=True,
verbose=False)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )<compute_test_metric>
df.GarageType.value_counts()
model = assess_cv_catboost(X_tr, [y1, y2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))<train_model>
from scipy.cluster import hierarchy as hc
class StackingAveragedModels() :
def __init__(self, base_models, meta_model, n_folds=15):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 try:
instance.fit(X[train_index], y[train_index], verbose=False)
 except:
instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<train_model>
from scipy.stats import spearmanr as sp
class StackingStoppingAveragedModels() :
def __init__(self, base_models, meta_model, n_folds=10):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
X_train, X_valid, y_train, y_valid = train_test_split(X[train_index],
y[train_index],
test_size=0.3)
 instance = clone(model)
 if type(instance)== sklearn.ensemble.gradient_boosting.GradientBoostingRegressor:
instance.min_impurity_decrease = 0.01
instance.fit(X[train_index], y[train_index])
 elif type(instance)== xgboost.sklearn.XGBRegressor:
instance.n_estimators = 10000000
instance.fit(X_train, y_train, early_stopping_rounds=50, eval_metric="rmse",
eval_set=[(X_valid, y_valid)], verbose=False)
 elif type(instance)== catboost.core.CatBoostRegressor:
instance.fit(X_train, y_train, eval_set=[X_valid, y_valid],
use_best_model=True, verbose=False)
 elif type(instance)== lightgbm.sklearn.LGBMRegressor:
instance.n_estimators = 10000000
instance.fit(X_train, y_train, eval_set=[(X_valid, y_valid)],
early_stopping_rounds=100, verbose=False)
 self.base_models_[i].append(instance)
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<choose_model_class>
def get_oob(df):
m = RandomForestRegressor(n_estimators=500, n_jobs=-1, oob_score=True)
 x, _ = split_vals(df, n_trn)
 m.fit(x, y_train)
 return m.oob_score_
cat_1 = CatBoostRegressor(iterations=1400,
learning_rate=0.03,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
od_type='Iter',
od_wait=50)
 xgb_1 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 1804,
gamma= 0.0,
subsample= 0.222159,
colsample_bytree= 0.5359,
colsample_bylevel= 0.19958,
max_delta_step= 64,
max_depth=28,
min_child_weight= 10,
reg_lambda=0.33038,
silent= True)
 lgb_1 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2000,
num_threads=3,
bagging_fraction=0.56369,
bagging_freq= 14,
feature_fraction= 0.88868,
lambda_l2= 0.0091689,
max_depth=96, 
)
 grad_1 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=7,
n_estimators=1120,
max_features=7,
min_samples_leaf=43,
min_samples_split=14,
min_weight_fraction_leaf=0.01556)
 linear = LinearRegression()<choose_model_class>
get_oob(df_keep )
cat_2 = CatBoostRegressor(iterations=1200,
learning_rate=0.03,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
od_type='Iter',
od_wait=50)
 lgb_2 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2838,
num_threads=3,
bagging_fraction=0.50173,
bagging_freq= 14,
feature_fraction= 0.62509,
lambda_l2= 0.0086298,
max_depth=20 
)
 grad_2 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=6,
n_estimators=3275,
max_features=2,
min_samples_leaf=2,
min_samples_split=2,
min_weight_fraction_leaf=0.08012)
 xgb_2 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 2386,
gamma= 0.0,
subsample= 0.90919,
colsample_bytree= 0.59049,
colsample_bylevel= 0.59404,
max_delta_step= 99,
max_depth=58,
min_child_weight= 85,
reg_lambda= 0.031165789070644215,
silent= True,
)<train_model>
stacked_1 = StackingAveragedModels(base_models=[xgb_1, lgb_1, grad_1, cat_1], meta_model=linear)
 stack_1 = stacked_1.fit(X_scale, y1)
 stacked_2 = StackingAveragedModels(base_models=[xgb_2, lgb_2, grad_2, cat_1], meta_model=linear)
 stack_2 = stacked_2.fit(X_scale, y2 )<train_model>
conda install -c conda-forge hdbscan
<prepare_output>
data = df_keep
clusterer = hdbscan.HDBSCAN(min_cluster_size=3, gen_min_span_tree=True)
 clusterer.fit(data )
one = stack_1.predict(X_scaled_test)
 two = stack_2.predict(X_scaled_test)
 _id = test['id']
submit = pd.DataFrame({'id':_id,
'formation_energy_ev_natom':np.expm1(one ).flatten() ,
'bandgap_energy_ev':np.expm1(two ).flatten() } )<save_to_csv>
from pdpbox import pdp
from plotnine import *
submit.to_csv('submission.csv', index=False )<import_modules>
df_all = df_keep
df_all["SalePrice"] = y
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam, Adagrad
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
import random
import kerastuner as kt<define_variables>
!pip install scikit-misc
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 tf.random.set_seed(seed )<set_options>
!pip install treeinterpreter
class BlockingTimeSeriesSplitWithGap() :
def __init__(self,n_splits=5,gap=0,group_fold_size=None):

 self.n_splits = n_splits
self.gap = gap
self.group_fold_size = group_fold_size
def get_n_splits(self,X,y,groups):
return self.n_splits
def split(self,X,groups,y=None,test_size=0.2):

 group_indices = {i:np.where(groups==i)for i in np.unique(groups)}
groups = np.unique(groups)
 if self.group_fold_size is None:
group_fold_size = len(groups)//self.n_splits
elif self.group_fold_size > len(groups):
raise AssertionError(f'group_fold_size={self.group_fold_size} is greater than the number of groups({len(groups)} ).')
 elif self.group_fold_size * self.n_splits < len(groups):
warnings.warn(f'group_fold_size={self.group_fold_size} is too small for a group of {len(groups)} items.Setting group_fold_size={(len(groups)) //self.n_splits}.')
 group_fold_size = len(groups)//self.n_splits
else:
group_fold_size = self.group_fold_size
splits = list(windowed(groups,group_fold_size,step=1))
idx = np.round(np.linspace(0, len(splits)-1, self.n_splits)).astype(int)
 splits = [splits[i] for i in idx]
for idx in splits:
idx = list(filter(None,idx))
mid = int(( 1-test_size)* len(idx))
group_idx_tr, group_idx_te = idx[0: mid], idx[mid + self.gap: -1]
if len(group_idx_te)< 1:
break
tr = np.concatenate([group_indices[gr_i] for gr_i in group_idx_tr],axis=1 ).ravel()
te = np.concatenate([group_indices[gr_i] for gr_i in group_idx_te],axis=1 ).ravel()
yield tr,te<split>
from treeinterpreter import treeinterpreter as ti
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=32, epochs=1,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train,
validation_data=(X_test,y_test),
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )<prepare_x_and_y>
df_train, df_valid = split_vals(df[df_keep.columns], n_trn )
TRAINING = False
USE_FINETUNE = True
FOLDS = 5
SEED = 42
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
EPSILON = {c:0.0 for c in resp_cols}
X = train[features].values
y = np.stack([(train[c] > EPSILON[c] ).astype('int')for c in resp_cols] ).T
f64_max = train['feature_64'].values.max()
f64_min = train['feature_64'].values.min()<statistical_test>
prediction, bias, contributions = ti.predict(m, row )
def calculate_vif(X,thresh=5.0,sample_size=-1,features=None):
if sample_size<0:
new_X = X.copy()
else:
new_X = X[np.random.choice(len(X), size=sample_size)]
if features and len(features)== new_X.shape[1]:
features_copy = features.copy()
else:
features_copy = list(range(new_X.shape[-1]))
dropped_idx = []
dropped = True
while dropped:
dropped = False
cc = np.corrcoef(new_X, rowvar=False)
 VIF = np.linalg.inv(cc ).diagonal()
maxloc = np.argmax(VIF)
 if max(VIF)> thresh:
print(f'Dropped {features_copy[maxloc]} with vif={max(VIF)}')
 features_copy.pop(maxloc)
 new_X = np.delete(new_X, maxloc, axis=1)
 dropped_idx.append(maxloc)
 dropped = True
del new_X
for idx in dropped_idx:
X = np.delete(X,idx,axis=1)
 return X, features_copy<set_options>
idxs = np.argsort(contributions[0] )
gc.collect()<load_pretrained>
[o for o in zip(df_keep.columns[idxs], df_valid.iloc[0][idxs], contributions[0][idxs])]
if TRAINING:
pd.to_pickle(features,'features.pkl')
 else:
features = pd.read_pickle('.. /input/jsautoencoder/features.pkl')
 f_mean = f_mean = np.mean(train[features[1:]].values,axis=0 )<choose_model_class>
columns = ['OverallQual', 'GrLivArea', 'YearBuilt', 'GarageCars',
'TotalBsmtSF', '1stFlrSF', 'BsmtFinSF1', 'GarageArea', 'LotArea']
df.loc[:, columns]
optimizer = lambda x: Adam(x )<categorify>
%whos
def create_encoder(hp,input_dim,output_dim):
i = Input(input_dim)
 x = BatchNormalization()(i)
 x = GaussianNoise(hp.Float('noise',0.1,0.5,default=0.1))(x)
 encoding_units = []
encoding_dp = []
for j in range(hp.Int('num_encoding_layers',1,3)) :
u = hp.Int(f'encoding_units_{j}',64,512)
 encoding_units.append(u)
 x = BatchNormalization()(x)
 x = Dense(u,activation='relu' )(x)
 dp = hp.Float(f'encoding_dp_{j}',0.1,0.5)
 encoding_dp.append(dp)
 x = Dropout(dp )(x)
 for u,dp in zip(encoding_units[::-1],encoding_dp[::-1]):
x = BatchNormalization()(x)
 x = Dense(u,activation='relu' )(x)
 x = Dropout(dp )(x)
 decoded = Dense(input_dim,activation='linear',name='decoded' )(x)
 for j in range(hp.Int('num_bottleneck_layers',1,3)) :
x = BatchNormalization()(x)
 x = Dense(hp.Int(f'bottleneck_units_{j}',16,64))(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'bottleneck_dp_{j}',0.1,0.5))(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=optimizer(1e-3),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder<categorify>
numeric_feats = X.dtypes[X.dtypes != "bool"].index
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)[0]
x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.1,0.5))(x)
 for i in range(hp.Int('num_layers',3,6)) :
x = BatchNormalization()(x)
 x = Dense(hp.Int(f'num_units_{i}',16,64))(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.1,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=optimizer(1e-3),loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<compute_test_metric>
skewed_feats = X.apply(lambda x: skew().sort_value
def utility_score_bincount(date, weight, resp, action):
count_i = len(np.unique(date))
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u<normalization>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )
bias = 0.1
def time_bias(f0,f64):
if f0 < 0:
correction =(1+bias/2)- bias *(f64-f64_min)/(f64_max-f64_min)
 else:
correction =(1-bias/2)+ bias *(f64-f64_min)/(f64_max-f64_min)
 return correction<define_search_space>
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape