Kaggle competition: Nomad2018 Predicting Transparent Conductors
<SOS> metric: MCRMSLE Kaggle data source: nomad2018-predicting-transparent-conductors<load_from_csv>
% matplotlib inline
!mkdir -p data
df_train = pd.read_csv(f'{dir_csv}/stage_1_train.csv')
 df_train.head()<count_missing_values>
test = pd.read_csv('.. /input/nomad2018-predict-transparent-conductors/test.csv')
 test_id = test.id
train = pd.read_csv('.. /input/nomad2018-predict-transparent-conductors/train.csv' )
df_train.Label.isna().sum()<feature_engineering>
ga_cols = []
al_cols = []
o_cols = []
in_cols = []
warnings.filterwarnings("ignore")
 for i in range(6):
ga_cols.append("Ga_"+str(i))
for i in range(6):
al_cols.append("Al_"+str(i))
for i in range(6):
o_cols.append("O_"+str(i))
for i in range(6):
in_cols.append("In_"+str(i))
ga_df= pd.DataFrame(columns=ga_cols)
 al_df = pd.DataFrame(columns=al_cols)
 o_df = pd.DataFrame(columns= o_cols)
 in_df = pd.DataFrame(columns=in_cols)
 def get_xyz_data(filename):
pos_data = []
lat_data = []
with open(filename)as f:
for line in f.readlines() :
x = line.split()
if x[0] == 'atom':
pos_data.append([np.array(x[1:4], dtype=np.float),x[4]])
 elif x[0] == 'lattice_vector':
lat_data.append(np.array(x[1:4], dtype=np.float))
return pos_data, np.array(lat_data)
 for i in train.id.values:
fn = ".. /input/nomad2018-predict-transparent-conductors/train/{}/geometry.xyz".format(i)
 train_xyz, train_lat = get_xyz_data(fn)
 ga_list = []
al_list = []
o_list = []
in_list = []
for li in train_xyz:
try:
if li[1] == "Ga":
ga_list.append(li[0])
 except:
pass
try:
if li[1] == "Al":
al_list.append(li[0])
 except:
pass
try:
if li[1] == "In":
in_list.append(li[0])
 except:
pass
try:
if li[1] == "O":
o_list.append(li[0])
 except:
pass
try:
model = PCA(n_components=2)
 ga_list = np.array(ga_list)
 temp_ga = model.fit_transform(ga_list.transpose())
 temp_ga = [item for sublist in temp_ga for item in sublist]
except:
temp_ga = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 al_list = np.array(al_list)
 temp_al = model.fit_transform(al_list.transpose())
 temp_al = [item for sublist in temp_al for item in sublist]
except:
temp_al = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 o_list = np.array(o_list)
 temp_o = model.fit_transform(o_list.transpose())
 temp_o = [item for sublist in temp_o for item in sublist]
except:
temp_o = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 in_list = np.array(in_list)
 temp_in = model.fit_transform(in_list.transpose())
 temp_in = [item for sublist in temp_in for item in sublist]
except:
temp_in = [0,0,0,0,0,0]
temp_ga = pd.DataFrame(temp_ga ).transpose()
temp_ga.columns = ga_cols
temp_ga.index = np.array([i])
 temp_al = pd.DataFrame(temp_al ).transpose()
temp_al.columns = al_cols
temp_al.index = np.array([i])
 temp_o = pd.DataFrame(temp_o ).transpose()
temp_o.columns = o_cols
temp_o.index = np.array([i])
 temp_in = pd.DataFrame(temp_in ).transpose()
temp_in.columns = in_cols
temp_in.index = np.array([i])
 ga_df = pd.concat([ga_df,temp_ga])
 al_df = pd.concat([al_df,temp_al])
 o_df = pd.concat([o_df,temp_o])
 in_df = pd.concat([in_df,temp_in])
 ga_df["id"] = ga_df.index
al_df["id"] = al_df.index
o_df["id"] = o_df.index
in_df["id"] = in_df.index
train = pd.merge(train,ga_df,on = ["id"],how = "left")
 train = pd.merge(train,al_df,on = ["id"],how = "left")
 train = pd.merge(train,o_df,on = ["id"],how = "left")
 train = pd.merge(train,in_df,on = ["id"],how = "left")
 ga_df= pd.DataFrame(columns=ga_cols)
 al_df = pd.DataFrame(columns=al_cols)
 o_df = pd.DataFrame(columns= o_cols)
 in_df = pd.DataFrame(columns=in_cols)
 for i in test.id.values:
fn = ".. /input/nomad2018-predict-transparent-conductors/test/{}/geometry.xyz".format(i)
 test_xyz, test_lat = get_xyz_data(fn)
 ga_list = []
al_list = []
o_list = []
in_list = []
for li in test_xyz:
try:
if li[1] == "Ga":
ga_list.append(li[0])
 except:
pass
try:
if li[1] == "Al":
al_list.append(li[0])
 except:
pass
try:
if li[1] == "In":
in_list.append(li[0])
 except:
pass
try:
if li[1] == "O":
o_list.append(li[0])
 except:
pass
try:
model = PCA(n_components=2)
 ga_list = np.array(ga_list)
 temp_ga = model.fit_transform(ga_list.transpose())
 temp_ga = [item for sublist in temp_ga for item in sublist]
except:
temp_ga = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 al_list = np.array(al_list)
 temp_al = model.fit_transform(al_list.transpose())
 temp_al = [item for sublist in temp_al for item in sublist]
except:
temp_al = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 o_list = np.array(o_list)
 temp_o = model.fit_transform(o_list.transpose())
 temp_o = [item for sublist in temp_o for item in sublist]
except:
temp_o = [0,0,0,0,0,0]
try:
model = PCA(n_components=2)
 in_list = np.array(in_list)
 temp_in = model.fit_transform(in_list.transpose())
 temp_in = [item for sublist in temp_in for item in sublist]
except:
temp_in = [0,0,0,0,0,0]
temp_ga = pd.DataFrame(temp_ga ).transpose()
temp_ga.columns = ga_cols
temp_ga.index = np.array([i])
 temp_al = pd.DataFrame(temp_al ).transpose()
temp_al.columns = al_cols
temp_al.index = np.array([i])
 temp_o = pd.DataFrame(temp_o ).transpose()
temp_o.columns = o_cols
temp_o.index = np.array([i])
 temp_in = pd.DataFrame(temp_in ).transpose()
temp_in.columns = in_cols
temp_in.index = np.array([i])
 ga_df = pd.concat([ga_df,temp_ga])
 al_df = pd.concat([al_df,temp_al])
 o_df = pd.concat([o_df,temp_o])
 in_df = pd.concat([in_df,temp_in])
 ga_df["id"] = ga_df.index
al_df["id"] = al_df.index
o_df["id"] = o_df.index
in_df["id"] = in_df.index
test = pd.merge(test,ga_df,on = ["id"],how = "left")
 test = pd.merge(test,al_df,on = ["id"],how = "left")
 test = pd.merge(test,o_df,on = ["id"],how = "left")
 test = pd.merge(test,in_df,on = ["id"],how = "left" )
df_train['fn'] = df_train.ID.apply(lambda x: '_'.join(x.split('_')[:2])+ '.png')
 df_train.columns = ['ID', 'probability', 'fn']
df_train['label'] = df_train.ID.apply(lambda x: x.split('_')[-1])
 df_train.drop_duplicates('ID', inplace=True )<save_to_csv>
train.rename(columns={
'spacegroup' : 'sg',
'number_of_total_atoms' : 'Natoms',
'percent_atom_al' : 'x_Al',
'percent_atom_ga' : 'x_Ga',
'percent_atom_in' : 'x_In',
'lattice_vector_1_ang' : 'a',
'lattice_vector_2_ang' : 'b',
'lattice_vector_3_ang' : 'c',
'lattice_angle_alpha_degree' : 'alpha',
'lattice_angle_beta_degree' : 'beta',
'lattice_angle_gamma_degree' : 'gamma',
'formation_energy_ev_natom' : 'E',
'bandgap_energy_ev' : 'Eg'}, inplace=True)
 test.rename(columns={
'spacegroup' : 'sg',
'number_of_total_atoms' : 'Natoms',
'percent_atom_al' : 'x_Al',
'percent_atom_ga' : 'x_Ga',
'percent_atom_in' : 'x_In',
'lattice_vector_1_ang' : 'a',
'lattice_vector_2_ang' : 'b',
'lattice_vector_3_ang' : 'c',
'lattice_angle_alpha_degree' : 'alpha',
'lattice_angle_beta_degree' : 'beta',
'lattice_angle_gamma_degree' : 'gamma',
}, inplace=True)
 target = [
'E',
'Eg']
all_data = pd.concat(( train, test))
pivot = df_train.pivot(index='fn', columns='label', values='probability')
 pivot.reset_index(inplace=True)
 pivot.to_csv('data/train_pivot.csv', index=False)
 pivot.head()<save_to_csv>
def get_prop_list(path_to_element_data):

 return [f[:-4] for f in os.listdir(path_to_element_data)]
path_to_element_data = '.. /input/elemental-properties/'
properties = get_prop_list(path_to_element_data)
 print(sorted(properties))
ks, vs = [], []
for k, v in d.items() :
ks.append(k), vs.append(' '.join(v))
pd.DataFrame(data = {'fn': ks, "labels": vs} ).to_csv('data/train_labels_as_string.csv', index = False )<load_from_csv>
def get_prop(prop, path_to_element_data):

 fin = os.path.join(path_to_element_data, prop+'.csv')
 with open(fin)as f:
all_els = {line.split(',')[0] : float(line.split(',')[1][:-1])for line in f}
my_els = ['Al', 'Ga', 'In']
return {el : all_els[el] for el in all_els if el in my_els}
prop_dict = {prop : get_prop(prop, path_to_element_data)for prop in properties}
print('The mass of aluminum is %.2f amu' % prop_dict['mass']['Al'] )
class Labeller() :

 def __init__(self):
self.df = pd.read_csv("data/train_labels_as_string.csv")
 self.df.set_index("fn", inplace = True)
 def __call__(self, path):
fn = path.name
labels_txt = self.df.loc[fn].labels
if isinstance(labels_txt, float)or labels_txt == ' ': return []
return labels_txt.split(' ' )<load_from_csv>
def avg_prop(x_Al, x_Ga, x_In, prop):

 els = ['Al', 'Ga', 'In']
concentration_dict = dict(zip(els, [x_Al, x_Ga, x_In]))
return np.sum(prop_dict[prop][el] * concentration_dict[el] for el in els)
 for prop in properties:
all_data['_'.join(['avg', prop])] = avg_prop(all_data['x_Al'],
all_data['x_Ga'],
all_data['x_In'],
prop )
classes = L(pd.read_csv("data/train_pivot.csv" ).columns.tolist() [1:])
 classes<compute_train_metric>
def get_vol(a, b, c, alpha, beta, gamma):

 return a*b*c*np.sqrt(1 + 2*np.cos(alpha)*np.cos(beta)*np.cos(gamma)
 - np.cos(alpha)**2
- np.cos(beta)**2
- np.cos(gamma)**2)
 lattice_angles = ['alpha', 'beta', 'gamma']
for lang in lattice_angles:
all_data['_'.join([lang, 'r'])] = np.pi * all_data[lang] / 180
all_data['vol'] = get_vol(all_data['a'], all_data['b'], all_data['c'],
all_data['alpha_r'], all_data['beta_r'], all_data['gamma_r'])
 all_data['atomic_density'] = all_data['Natoms'] / all_data['vol']
mcat = MultiCategorize(vocab = classes)
 mcat.o2i = classes.val2idx()<import_modules>
avg_cols = ['x_Al','x_Ga','x_In','a','b','c','avg_rs_max','avg_electronegativity',
'avg_rp_max','avg_LUMO','avg_IP','avg_rd_max','avg_EA','avg_HOMO',
'avg_mass','vol','atomic_density']
for col in avg_cols:
new_col = col + "_avg"
all_data[new_col] = np.nan
for group in all_data['sg'].unique() :
all_data.loc[(all_data['sg'] == group), new_col] = all_data[(all_data['sg'] == group)][col].mean()
print('Number of Null Values: {}'.format(pd.isnull(all_data[avg_cols] ).sum().sum()))
mcat.o2i<split>
all_data[['sg', 'Natoms']] = all_data[['sg', 'Natoms']].astype(str)
 all_data = pd.get_dummies(all_data )
tfms = [PILImage.create, [Labeller() , mcat, OneHotEncode() ]]
ds_img_tfms = [ToTensor() ]
dsrc = DataSource(items, tfms, splits = RandomSplitter()(items))<define_variables>
features = ['x_Al', 'x_Ga', 'x_In', 'a', 'b', 'c', 'alpha', 'beta',
'gamma', 'vol', 'atomic_density', 'x_Al_avg','x_Ga_avg', 'x_In_avg', 'a_avg',
'b_avg', 'c_avg', 'vol_avg', 'atomic_density_avg', 'pca_abc', 'pca_AlGaInDensity',
'O_0_0','O_1_0', 'O_2_0', 'O_3_0', 'O_4_0', 'O_5_0', 'Al_0_0', 'Al_1_0', 'Al_2_0', 'Al_3_0', 'Al_4_0', 'Al_5_0', 'Ga_0_0',
'Ga_1_0', 'Ga_2_0', 'Ga_3_0', 'Ga_4_0', 'Ga_5_0', 'In_0_0', 'In_1_0',
'In_2_0', 'In_3_0', 'In_4_0', 'In_5_0',]
vector1 = all_data[['a', 'b', 'c']].values
vector2 = all_data[['x_Al', 'x_Ga', 'x_In', 'atomic_density_avg']].values
pca = PCA()
pca.fit(vector1)
 all_data['pca_abc'] = pca.transform(vector1)[:,0]
pca = PCA()
pca.fit(vector2)
 all_data['pca_AlGaInDensity'] = pca.transform(vector2)[:,0]
scale = StandardScaler()
scaled = scale.fit(all_data[features] ).transform(all_data[features])
 X_scale = scaled[:train.shape[0]]
X_scaled_test = scaled[train.shape[0]:]
X_tr = all_data[:train.shape[0]][features].values
X_te = all_data[train.shape[0]:][features].values
y1 = np.log1p(train['E'])
 y2 = np.log1p(train['Eg'])
 y12 = np.column_stack(( y1, y2))
X_tr.shape, y1.shape, y2.shape, y12.shape, X_scaled_test.shape
test_paths = get_image_files(dir_test_img)
 test_tfms = [PILImage.create, [lambda x: np.array([0,0,0,0,0,0])]]
dsrc_test = DataSource([test_paths[0]] + test_paths, test_tfms, splits=[[0], L(range(len(test_paths)) ).map(lambda x: x + 1)] )<load_pretrained>
def rmsle(h, y):

 return np.sqrt(np.square(np.log(h + 1)- np.log(y + 1)).mean() )
ds_img_tfms = [ToTensor() ]
dl_tfms = [Cuda() , IntToFloatTensor() , Normalize(*dataset_stats)]
dbch = dsrc.databunch(after_item = ds_img_tfms, after_batch = dl_tfms, bs = 128, num_workers = 4)
 dbch_test = dsrc_test.databunch(after_item = ds_img_tfms, after_batch = dl_tfms, bs = 128, num_workers = 4 )<choose_model_class>
import keras
import tensorflow as tf
import keras.backend as K
from keras.regularizers import l2
from keras.optimizers import Adam
from keras.models import Model
from keras.models import Sequential
from keras.layers import Dense,Dropout,BatchNormalization, Input
model = create_cnn_model(resnet18, 6, -2, True )<find_best_params>
def get_model(shape):

 X_input = Input(shape=(shape,))
X = Dense(64, activation='relu' )(X_input)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dropout(0.1 )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(64, activation='relu' )(X)
 X = Dense(2, activation='linear' )(X)
 return Model(inputs=X_input, outputs=X)
 def rmsle_K(y, y0):
return K.sqrt(K.mean(K.square(tf.log1p(tf.expm1(y)) - tf.log1p(tf.expm1(y0)))))
 def compile_model(shape, lr=0.001):
model = get_model(shape)
 optimizer = Adam(lr=lr, decay=0.001)
 model.compile(optimizer=optimizer, loss='mse', metrics=[rmsle_K])
 return model
model_segments = model[0][:6], model[0][6:], model[1]<normalization>
def assess_nn(X, y, params):

 final = []
best_iter = [[], []]
kfold = KFold(n_splits=2, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = compile_model(X_scale.shape[1], lr=0.003)
 model.fit(x=X_scale, y=y12, epochs=600, batch_size=16, verbose=0)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print(e)
 out.append(e)
 final.append(np.array(out ).mean())
 print('y1 best iteration: {}'.format(np.mean(best_iter[0])))
 print('y2 best iteration: {}'.format(np.mean(best_iter[1])))
 return(np.array(final ).mean() , np.array(final ).std() )
def trainable_params_mod(model): return L(trainable_params(segment)for segment in model_segments)
 opt_func = partial(Adam, wd=0.01, eps=1e-3 )<choose_model_class>
from sklearn.model_selection import KFold
from sklearn.ensemble import GradientBoostingRegressor
learn = Learner(
 dbch,
model,
loss_func=BCEWithLogitsLossFlat() ,
metrics=[accuracy_multi],
opt_func=opt_func,
splitter=trainable_params_mod
)<train_model>
grad_1 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=7,
n_estimators=1120,
max_features=7,
min_samples_leaf=43,
min_samples_split=14,
min_weight_fraction_leaf=0.01556)
 grad_2 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=6,
n_estimators=3275,
max_features=2,
min_samples_leaf=2,
min_samples_split=2,
min_weight_fraction_leaf=0.08012)
 def assess_grad(X, y_list, model_list):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_list[idx]
model.fit(X_train, y_train)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print(e)
 out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )
learn.freeze_to(-1 )<find_best_params>
model = assess_grad(X_tr, [y1, y2], [grad_1, grad_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))
learn.lr_find()<train_model>
import lightgbm as lgb
from lightgbm import Dataset
learn.fit_one_cycle(2, 2e-2 )<save_model>
lgb_1 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2000,
num_threads=3,
bagging_fraction=0.56369,
bagging_freq= 14,
feature_fraction= 0.88868,
lambda_l2= 0.0091689,
max_depth=20, 
)
 lgb_2 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2838,
num_threads=3,
bagging_fraction=0.50173,
bagging_freq= 14,
feature_fraction= 0.62509,
lambda_l2= 0.0086298,
max_depth=20, 
)
 def assess_light(X, y_list, model_list):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_list[idx]
model.fit(X_train, y_train)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )
learn.save('phase-1' )<load_pretrained>
model = assess_light(X_scale, [y1, y2], [lgb_1, lgb_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))
learn.load('phase-1');<train_model>
def assess_early_stop_light(X, y_list, params):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_valid = lgb.Dataset(X_test, y_test, reference=lgb_train)
 model = lgb.train(params[idx],
lgb_train,
num_boost_round=1000000,
valid_sets=[lgb_valid],
early_stopping_rounds=100,
verbose_eval=0)
 best_iter[idx].append(model.best_iteration)
 h = model.predict(X_test, num_iteration=model.best_iteration)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 print('y1 best iteration: {}'.format(np.mean(best_iter[0])))
 print('y2 best iteration: {}'.format(np.mean(best_iter[1])))
 return(np.array(final ).mean() , np.array(final ).std() )
learn.freeze_to(-2 )<find_best_params>
params1 = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'rmse', 'rmsle'},
'max_depth': 10,
'learning_rate': 0.010,
'feature_fraction': 0.8632,
'bagging_fraction': 0.8759,
'bagging_freq': 4,
'verbose': 0,
'verbose_eval':0,
'num_threads': 3,
'lambda_l2': 0.0005597442104287973,
'lambda_l1': 0.00015997163552092318
}
params2 = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'rmse', 'rmsle'},
'learning_rate': 0.010,
'verbose': 0,
'verbose_eval':0,
'num_threads': 3,
'bagging_fraction': 0.9311539021934098,
'bagging_freq': 15,
'feature_fraction': 0.9989744117209727,
'lambda_l1': 8.337666829263869e-05,
'lambda_l2': 0.005541689229153562,
'max_depth': 19
}
model = assess_early_stop_light(X_tr, [y1, y2], [params1, params2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))
learn.lr_find(start_lr=1e-8, end_lr=1e-1 )<train_model>
from xgboost import XGBRegressor
learn.fit_one_cycle(2, [1e-3, 1e-4, 1e-5] )<save_model>
learn.save('phase-2')
 learn.load('phase-2');<train_model>
xgb_1 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 1804,
gamma= 0.0,
subsample= 0.222159,
colsample_bytree= 0.5359,
colsample_bylevel= 0.19958,
max_delta_step= 64,
max_depth=28,
min_child_weight= 10,
reg_lambda=0.33038,
silent= True, 
)
 xgb_2 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 2386,
gamma= 0.0,
subsample= 0.90919,
colsample_bytree= 0.59049,
colsample_bylevel= 0.59404,
max_delta_step= 99,
max_depth=58,
min_child_weight= 85,
reg_lambda= 0.031165789070644215,
silent= True, 
)
 def assess_xgb(X, y_list, model_num):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_num[idx]
model.fit(X_train, y_train)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )
learn.fit_one_cycle(5, [1e-4, 5e-4, 1e-3] )<save_model>
model = assess_xgb(X_scale, [y1, y2], [xgb_1, xgb_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))
learn.save('phase-3' )<load_pretrained>
xgb_param_1 = {
'learning_rate': 0.005,
'n_jobs': 3,
'gamma': 0.0,
'subsample': 0.222159,
'colsample_bytree': 0.5359,
'colsample_bylevel': 0.19958,
'max_delta_step': 64,
'max_depth': 28,
'min_child_weight': 10,
'reg_lambda': 0.33038,
'silent': True
}
xgb_param_2 = {
'learning_rate': 0.005,
'n_jobs': 3,
'gamma': 0.0,
'subsample': 0.90919,
'colsample_bytree': 0.59049,
'colsample_bylevel': 0.59404,
'max_delta_step': 99,
'max_depth': 58,
'min_child_weight': 85,
'reg_lambda': 0.031165789070644215,
'silent': True
}
def assess_early_cv_stop_xgb(X, y_list, param_list):

 final = []
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_iter, X_test = X[train_index], X[test_index]
y_iter, y_test = y[train_index], y[test_index]
X_train, X_valid, y_train, y_valid = train_test_split(X_iter, y_iter, test_size=0.3)
 xgb_train = DMatrix(X_train, label=y_train)
 xgb_valid = DMatrix(X_valid, y_valid)
 xgb_test = DMatrix(X_test, y_test)
 watchlist = [(xgb_train,'train'),(xgb_valid, 'valid')]
model = xgb.train(param_list[idx], xgb_train, 1000000,
watchlist, early_stopping_rounds=20, verbose_eval=False)
 h = model.predict(xgb_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )
learn.load('phase-3' )<feature_engineering>
model = assess_early_cv_stop_xgb(X_tr, [y1, y2], [xgb_param_1, xgb_param_2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))
learn.dbunch = dbch_test<predict_on_test>
from catboost import CatBoostRegressor
preds, targs = learn.get_preds()<define_variables>
cat_1 = CatBoostRegressor(iterations=2300,
learning_rate=0.020,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
od_type='Iter',
od_wait=50, 
)
 def assess_cat(X, y_list, model_num):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model = model_num[idx]
model.fit(X_train, y_train, verbose=False)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )
ids = []
labels = []
for path, pred in zip(test_paths, preds):
for i, label in enumerate(classes):
ids.append(f"{path.name.split('.')[0]}_{label}")
 predicted_probability = '{0:1.10f}'.format(pred[i].item())
 labels.append(predicted_probability )<save_to_csv>
model = assess_cat(X_tr, [y1, y2], [cat_1, cat_1])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))
pd.DataFrame({'ID': ids, 'Label': labels} ).to_csv(f'submission.csv', index=False )<load_from_csv>
catboost_cv = CatBoostRegressor(iterations=1200,
learning_rate=0.03,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
random_seed=99,
od_type='Iter',
od_wait=50)
 def assess_cv_catboost(X, y_list):

 final = []
best_iter = [[], []]
for idx, y in enumerate(y_list):
kfold = KFold(n_splits=10, shuffle=True)
 out = []
for train_index, test_index in kfold.split(X):
X_iter, X_test = X[train_index], X[test_index]
y_iter, y_test = y[train_index], y[test_index]
X_train, X_valid, y_train, y_valid = train_test_split(X_iter, y_iter, test_size=0.3)
 model = catboost_cv
model.fit(X_train, y_train,
eval_set=(X_valid, y_valid),
use_best_model=True,
verbose=False)
 h = model.predict(X_test)
 e = rmsle(np.expm1(h), np.expm1(y_test))
print('RMSLE: {}'.format(e))
out.append(e)
 final.append(np.array(out ).mean())
 return(np.array(final ).mean() , np.array(final ).std() )
path = '.. /input/'
traindat = pd.read_csv(path + 'train_ver2.csv', low_memory = True)
 testdat = pd.read_csv(path + 'test_ver2.csv', low_memory = True )<define_variables>
model = assess_cv_catboost(X_tr, [y1, y2])
 print("Model RMSLE: {}, std: {}".format(model[0], model[1]))
demographic_cols = ['fecha_dato',
'ncodpers','ind_empleado','pais_residencia','sexo','age','fecha_alta','ind_nuevo','antiguedad','indrel',
'indrel_1mes','tiprel_1mes','indresi','indext','canal_entrada','indfall',
'tipodom','cod_prov','ind_actividad_cliente','renta','segmento']
notuse = ["ult_fec_cli_1t","nomprov"]
product_col = [
'ind_ahor_fin_ult1','ind_aval_fin_ult1','ind_cco_fin_ult1','ind_cder_fin_ult1','ind_cno_fin_ult1','ind_ctju_fin_ult1',
'ind_ctma_fin_ult1','ind_ctop_fin_ult1','ind_ctpp_fin_ult1','ind_deco_fin_ult1','ind_deme_fin_ult1',
'ind_dela_fin_ult1','ind_ecue_fin_ult1','ind_fond_fin_ult1','ind_hip_fin_ult1','ind_plan_fin_ult1',
'ind_pres_fin_ult1','ind_reca_fin_ult1','ind_tjcr_fin_ult1','ind_valo_fin_ult1','ind_viv_fin_ult1','ind_nomina_ult1',
'ind_nom_pens_ult1','ind_recibo_ult1']
train_cols = demographic_cols + product_col
traindat = traindat.filter(train_cols)
 testdat = testdat.filter(train_cols )<count_missing_values>
class StackingAveragedModels() :
def __init__(self, base_models, meta_model, n_folds=15):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 try:
instance.fit(X[train_index], y[train_index], verbose=False)
 except:
instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
traindat.isnull().sum()<data_type_conversions>
class StackingStoppingAveragedModels() :
def __init__(self, base_models, meta_model, n_folds=10):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
X_train, X_valid, y_train, y_valid = train_test_split(X[train_index],
y[train_index],
test_size=0.3)
 instance = clone(model)
 if type(instance)== sklearn.ensemble.gradient_boosting.GradientBoostingRegressor:
instance.min_impurity_decrease = 0.01
instance.fit(X[train_index], y[train_index])
 elif type(instance)== xgboost.sklearn.XGBRegressor:
instance.n_estimators = 10000000
instance.fit(X_train, y_train, early_stopping_rounds=50, eval_metric="rmse",
eval_set=[(X_valid, y_valid)], verbose=False)
 elif type(instance)== catboost.core.CatBoostRegressor:
instance.fit(X_train, y_train, eval_set=[X_valid, y_valid],
use_best_model=True, verbose=False)
 elif type(instance)== lightgbm.sklearn.LGBMRegressor:
instance.n_estimators = 10000000
instance.fit(X_train, y_train, eval_set=[(X_valid, y_valid)],
early_stopping_rounds=100, verbose=False)
 self.base_models_[i].append(instance)
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
traindat.age = pd.to_numeric(traindat.age, errors='coerce')
 traindat.renta = pd.to_numeric(traindat.renta, errors='coerce')
 traindat.antiguedad = pd.to_numeric(traindat.antiguedad, errors='coerce')
 traindat.loc[traindat['ind_empleado'].isnull() ,'ind_empleado'] = 'N'
traindat.loc[traindat['pais_residencia'].isnull() ,'pais_residencia'] = 'ES'
traindat.loc[traindat['sexo'].isnull() ,'sexo'] = 'V'
traindat.fecha_alta = traindat.fecha_alta.astype('datetime64[ns]')
 traindat.loc[traindat['fecha_alta'].isnull() , 'fecha_alta'] = pd.Timestamp(2011,9,1)
 traindat.loc[traindat['ind_nuevo'].isnull() , 'ind_nuevo'] = 0
traindat.loc[traindat['indrel'].isnull() , 'indrel'] = 1
traindat.indrel_1mes = traindat.indrel_1mes.astype('str' ).str.slice(0,1)
 traindat.loc[traindat['indrel_1mes'].isnull() , 'indrel_1mes'] = '1'
traindat.loc[traindat['tiprel_1mes'].isnull() , 'tiprel_1mes'] = 'I'
traindat.loc[traindat['indresi'].isnull() , 'indresi'] = 'S'
traindat.loc[traindat['indext'].isnull() , 'indext'] = 'N'
traindat.loc[traindat['canal_entrada'].isnull() , 'canal_entrada'] = 'MIS'
traindat.loc[traindat['indfall'].isnull() , 'indfall'] = 'N'
traindat.loc[traindat['tipodom'].isnull() , 'tipodom'] = 0.0
traindat.loc[traindat['cod_prov'].isnull() , 'cod_prov'] = 28.0
traindat.loc[traindat['ind_actividad_cliente'].isnull() , 'ind_actividad_cliente'] = 0.0
traindat["renta"] = traindat[['renta','cod_prov']].groupby("cod_prov" ).transform(lambda x: x.fillna(x.mean()))
traindat["age"] = traindat[['age','cod_prov']].groupby("cod_prov" ).transform(lambda x: x.fillna(x.mean()))
traindat["antiguedad"] = traindat[['antiguedad','cod_prov']].groupby("cod_prov" ).transform(lambda x: x.fillna(x.mean()))
traindat.loc[traindat['segmento'].isnull() , 'segmento'] = '02 - PARTICULARES'
traindat.loc[traindat['ind_nomina_ult1'].isnull() , 'ind_nomina_ult1'] = 0
traindat.loc[traindat['ind_nom_pens_ult1'].isnull() , 'ind_nom_pens_ult1'] = 0
testdat.age = pd.to_numeric(testdat.age, errors='coerce')
 testdat.antiguedad = pd.to_numeric(testdat.antiguedad, errors='coerce')
 testdat.renta = pd.to_numeric(testdat.renta, errors='coerce')
 testdat.loc[testdat['sexo'].isnull() ,'sexo'] = 'V'
testdat.indrel_1mes = testdat.indrel_1mes.astype('str' ).str.slice(0,1)
 testdat.loc[testdat['indrel_1mes'].isnull() , 'indrel_1mes'] = '1'
testdat.loc[testdat['tiprel_1mes'].isnull() , 'tiprel_1mes'] = 'I'
testdat.loc[testdat['canal_entrada'].isnull() , 'canal_entrada'] = 'MIS'
testdat.loc[testdat['cod_prov'].isnull() , 'cod_prov'] = 28.0
testdat.loc[testdat['segmento'].isnull() , 'segmento'] = '02 - PARTICULARES'
testdat["renta"] = testdat[['renta','cod_prov']].groupby("cod_prov" ).transform(lambda x: x.fillna(x.mean()))
testdat["age"] = testdat[['age','cod_prov']].groupby("cod_prov" ).transform(lambda x: x.fillna(x.mean()))
testdat["antiguedad"] = testdat[['antiguedad','cod_prov']].groupby("cod_prov" ).transform(lambda x: x.fillna(x.mean()))<count_missing_values>
cat_1 = CatBoostRegressor(iterations=1400,
learning_rate=0.03,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
od_type='Iter',
od_wait=50)
 xgb_1 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 1804,
gamma= 0.0,
subsample= 0.222159,
colsample_bytree= 0.5359,
colsample_bylevel= 0.19958,
max_delta_step= 64,
max_depth=28,
min_child_weight= 10,
reg_lambda=0.33038,
silent= True)
 lgb_1 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2000,
num_threads=3,
bagging_fraction=0.56369,
bagging_freq= 14,
feature_fraction= 0.88868,
lambda_l2= 0.0091689,
max_depth=96, 
)
 grad_1 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=7,
n_estimators=1120,
max_features=7,
min_samples_leaf=43,
min_samples_split=14,
min_weight_fraction_leaf=0.01556)
 linear = LinearRegression()
traindat.isnull().sum()<data_type_conversions>
cat_2 = CatBoostRegressor(iterations=1200,
learning_rate=0.03,
depth=5,
loss_function='RMSE',
eval_metric='RMSE',
od_type='Iter',
od_wait=50)
 lgb_2 = lgb.LGBMRegressor(
 objective= 'regression',
boosting_type= 'gbdt',
learning_rate= 0.002,
n_estimators = 2838,
num_threads=3,
bagging_fraction=0.50173,
bagging_freq= 14,
feature_fraction= 0.62509,
lambda_l2= 0.0086298,
max_depth=20 
)
 grad_2 = GradientBoostingRegressor(
 loss='ls',
learning_rate = 0.0035,
max_depth=6,
n_estimators=3275,
max_features=2,
min_samples_leaf=2,
min_samples_split=2,
min_weight_fraction_leaf=0.08012)
 xgb_2 = XGBRegressor(
 learning_rate=0.005,
n_jobs=3,
n_estimators= 2386,
gamma= 0.0,
subsample= 0.90919,
colsample_bytree= 0.59049,
colsample_bylevel= 0.59404,
max_delta_step= 99,
max_depth=58,
min_child_weight= 85,
reg_lambda= 0.031165789070644215,
silent= True,
)
traindat["fecha_alta"] = traindat["fecha_alta"].astype("datetime64")
 testdat["fecha_alta"] = testdat["fecha_alta"].astype("datetime64")
 traindat["renta"] = np.log(traindat["renta"])
 testdat["renta"] = np.log(testdat["renta"])
 bins_dt = pd.date_range('1994-01-01', freq='16Y', periods=3)
 bins_str = bins_dt.astype(str ).values
labels = ['({}, {}]'.format(bins_str[i-1], bins_str[i])for i in range(1, len(bins_str)) ]
traindat['fecha_alta'] = pd.cut(traindat.fecha_alta.astype(np.int64)//10**9,
bins=bins_dt.astype(np.int64)//10**9,
labels=labels)
 testdat['fecha_alta'] = pd.cut(testdat.fecha_alta.astype(np.int64)//10**9,
bins=bins_dt.astype(np.int64)//10**9,
labels=labels)
 bins_renta = [0,np.percentile(traindat.renta, 25),np.percentile(traindat.renta, 75),25]
traindat['renta'] = pd.cut(traindat.renta,
bins=bins_renta)
 testdat['renta'] = pd.cut(testdat.renta,
bins=bins_renta)
 bins_age = [0,25,42,60,1000]
labels_age = ['young','middle','older','old']
traindat['age'] = pd.cut(traindat.age,
bins=bins_age,
labels=labels_age)
 testdat['age'] = pd.cut(testdat.age,
bins=bins_age,
labels=labels_age)
 bins_anti = [-1,220,300]
labels_anti = ['new','old']
traindat.antiguedad[traindat.antiguedad<0] = 0
traindat['antiguedad'] = pd.cut(traindat.antiguedad,
bins=bins_anti,
labels=labels_anti)
 testdat['antiguedad'] = pd.cut(testdat.antiguedad,
bins=bins_anti,
labels=labels_anti)
<filter>
stacked_1 = StackingAveragedModels(base_models=[xgb_1, lgb_1, grad_1, cat_1], meta_model=linear)
 stack_1 = stacked_1.fit(X_scale, y1)
 stacked_2 = StackingAveragedModels(base_models=[xgb_2, lgb_2, grad_2, cat_1], meta_model=linear)
 stack_2 = stacked_2.fit(X_scale, y2 )
traindat = traindat[traindat.fecha_dato.isin(['2015-05-28','2015-06-28','2016-05-28'])]<merge>
traindat = traindat.sort_values(['ncodpers','fecha_dato'],ascending=[True,True] ).reset_index(drop=True)
 print('sort completed')
 traindat['new'] = traindat.index
train_index = traindat.copy()
train_index['new'] += 1
merge_drop_cols = demographic_cols.copy()
merge_drop_cols.remove('ncodpers')
 traindat_use = pd.merge(traindat,train_index.drop(merge_drop_cols,1), on=['new','ncodpers'],how='left',suffixes=['','_previous'])
 print('merge completed')
 for i in product_col:
traindat_use[i] = traindat_use[i]-traindat_use[i+"_previous"]
traindat_use[i][traindat_use[i] < 0] = 0
traindat_use[product_col] = traindat_use[product_col].fillna(0)
 new_product_col = [i + "_previous" for i in product_col]
traindat_use[new_product_col] = traindat_use[new_product_col].fillna(0)
 del train_index 
<merge>
one = stack_1.predict(X_scaled_test)
 two = stack_2.predict(X_scaled_test)
 _id = test['id']
submit = pd.DataFrame({'id':_id,
'formation_energy_ev_natom':np.expm1(one ).flatten() ,
'bandgap_energy_ev':np.expm1(two ).flatten() } )
test_col = product_col + ['ncodpers']
testdat_use = pd.merge(testdat,traindat[traindat.fecha_dato=='2016-05-28'][test_col],on='ncodpers',how='left',suffixes=['','_previous'])
 testdat_use.rename(
 columns={i:j for i,j in zip(product_col,new_product_col)}, inplace=True 
)
 testdat_use[new_product_col] = testdat_use[new_product_col].fillna(0)
 del traindat, testdat
gc.collect()<define_variables>
submit.to_csv('submission.csv', index=False )