Kaggle competition: Nomad2018 Predicting Transparent Conductors
traindat_purchases = traindat_use[traindat_use.fecha_dato == '2015-06-28'][product_col].copy()
traindat_final = traindat_use[traindat_use.fecha_dato == '2015-06-28'][new_product_col].copy()
demog_col = ['sexo','age','fecha_alta','ind_nuevo','indrel','indresi','indfall','tipodom','ind_actividad_cliente']
traindat_demog_final = traindat_use[traindat_use.fecha_dato == '2015-06-28'][demog_col].copy()
sexo_map = {'V': 1,'H': 0}
age_map = {'old': 1,'young': 0}
fecha_alta_map = {'(1994-12-31, 2010-12-31]': 1,'(2010-12-31, 2026-12-31]': 0}
indresi_map = {'S': 1,'N': 0}
indfall_map = {'S': 1,'N': 0}
traindat_demog_final.loc[traindat_demog_final['age']=='older', 'age'] = 'old'
traindat_demog_final.loc[traindat_demog_final['age']=='middle', 'age'] = 'young'
traindat_demog_final.sexo = [sexo_map[item] for item in traindat_demog_final.sexo]
traindat_demog_final.age = [age_map[item] for item in traindat_demog_final.age]
traindat_demog_final.fecha_alta = [fecha_alta_map[item] for item in traindat_demog_final.fecha_alta]
traindat_demog_final.indresi = [indresi_map[item] for item in traindat_demog_final.indresi]
traindat_demog_final.indfall = [indfall_map[item] for item in traindat_demog_final.indfall]
new_product_col_aug = new_product_col + ['ncodpers']
testdat_final = testdat_use[new_product_col_aug].copy()
testdat_final_unique = testdat_final.drop('ncodpers',1 ).drop_duplicates().copy().reset_index(drop=True)
 demog_col_aug = demog_col + ['ncodpers']
testdat_demog_final = testdat_use[demog_col_aug].copy()
testdat_demog_final.loc[testdat_demog_final['age']=='older', 'age'] = 'old'
testdat_demog_final.loc[testdat_demog_final['age']=='middle', 'age'] = 'young'
testdat_demog_final.sexo = [sexo_map[item] for item in testdat_demog_final.sexo]
testdat_demog_final.age = [age_map[item] for item in testdat_demog_final.age]
testdat_demog_final.fecha_alta = [fecha_alta_map[item] for item in testdat_demog_final.fecha_alta]
testdat_demog_final.indresi = [indresi_map[item] for item in testdat_demog_final.indresi]
testdat_demog_final.indfall = [indfall_map[item] for item in testdat_demog_final.indfall]
testdat_demog_final_unique = testdat_demog_final.drop('ncodpers',1 ).drop_duplicates().copy().reset_index(drop=True)
<remove_duplicates>
path = '.. /input/'
train_df = pd.read_csv(path+"/train.csv")
 test_df = pd.read_csv(path+"/test.csv" )
traindat_index = np.random.rand(len(traindat_final)) < 0.8
traindat_train = traindat_final[traindat_index]
traindat_test = traindat_final[~traindat_index]
traindat_test_unique = traindat_test.drop_duplicates().copy().reset_index(drop=True)
 traindat_purchases_train = traindat_purchases[traindat_index]
traindat_purchases_test = traindat_purchases[~traindat_index]
traindat_ncodpers = traindat_use[traindat_use.fecha_dato == '2015-06-28'][traindat_index][['fecha_dato','ncodpers']]
traindat_test_ncodpers = traindat_use[traindat_use.fecha_dato == '2015-06-28'][~traindat_index][['fecha_dato','ncodpers']]
traindat_demog_train = traindat_demog_final[traindat_index]
traindat_demog_test = traindat_demog_final[~traindat_index]
traindat_demog_test_unique = traindat_demog_test.drop_duplicates().copy().reset_index(drop=True)
<compute_train_metric>
Targets_df=pd.DataFrame()
Targets_df["bandgap_energy_ev"]=train_df["bandgap_energy_ev"].copy()
Targets_df["formation_energy_ev_natom"]=train_df["formation_energy_ev_natom"].copy()
train_df=train_df.drop(["formation_energy_ev_natom","bandgap_energy_ev"],axis=1)
 train_id_df=pd.DataFrame()
train_id_df["id"]=train_df["id"].copy()
train_df=train_df.drop(["id"],axis=1)
 test_id_df=pd.DataFrame()
test_id_df["id"]=test_df["id"].copy()
test_df=test_df.drop(["id"],axis=1)
 combined_df = pd.concat([train_df, test_df], ignore_index=True)
 print("Total number of null values in the df","
")
 print(combined_df.isna().sum().sum() )
predict_product_col = [i + "_predict" for i in new_product_col]
def probability_calculation(dataset,training,training_purchases,used_columns,metric,test_remap,print_option=False):
n = dataset.shape[0]
for index, row in dataset.iterrows() :
if print_option == True:
print(str(index)+ '/' + str(n))
row_use = row.to_frame().T
row_history = row_use[used_columns]
distances = metrics.pairwise_distances(row_use,training,metric=metric)+ 1e-6
norm_distances = 1/distances
sim = pd.DataFrame(norm_distances.dot(training_purchases)/np.sum(norm_distances),columns = new_product_col)
 if(index == 0):
probabilities = sim
else:
probabilities = probabilities.append(sim)
 print("probabilities calculated")
 reindexed_output = probabilities.reset_index().drop('index',axis=1 ).copy()
indexed_unique_test = dataset.reset_index().drop('index',axis=1 ).copy()
output_unique = indexed_unique_test.join(reindexed_output,rsuffix='_predict')
 output_final = pd.merge(test_remap,output_unique,on=used_columns,how='left')
 output_final = output_final.drop(used_columns,1)
 output_final.columns = output_final.columns.str.replace("_predict", "")
 output_final.columns = output_final.columns.str.replace("_previous", "_predict")
 return output_final 
<compute_train_metric>
numerical_df=pd.DataFrame.copy(combined_df[['number_of_total_atoms', 'percent_atom_al',
'percent_atom_ga', 'percent_atom_in', 'lattice_vector_1_ang',
'lattice_vector_2_ang', 'lattice_vector_3_ang',
'lattice_angle_alpha_degree', 'lattice_angle_beta_degree',
'lattice_angle_gamma_degree']])
 one_hot_df=pd.DataFrame.copy(combined_df[["spacegroup"]])
 one_hot_df=pd.get_dummies(one_hot_df,prefix=["spacegroup"],
columns=["spacegroup"])
 features_df=pd.concat([numerical_df,one_hot_df],axis=1)
 ")
 skewed_feats= numerical_df.skew()
skewed_feats = skewed_feats[skewed_feats > 0.1]
skewed_feats = skewed_feats.index
unskewed_feats= numerical_df.skew()
unskewed_feats = unskewed_feats[unskewed_feats < 0.1]
unskewed_feats = unskewed_feats.index
transform_df=pd.DataFrame()
transform_df[unskewed_feats]=(numerical_df[unskewed_feats]
- numerical_df[unskewed_feats].mean())/(numerical_df[unskewed_feats].max() - numerical_df[unskewed_feats].min())
 transform_df[skewed_feats] = np.log1p(numerical_df[skewed_feats])
 ")
 features_transform_df=pd.concat([transform_df,one_hot_df],axis=1)
 features_df=pd.concat([numerical_df,one_hot_df],axis=1 )
probabilities_memory = probability_calculation(traindat_test_unique,traindat_train,traindat_purchases_train,new_product_col,'manhattan',traindat_test)
 probabilities_demog = probability_calculation(traindat_demog_test_unique,traindat_demog_train,traindat_purchases_train,demog_col,'manhattan',traindat_demog_test)
 probabilities_avg_90 = 0.9*probabilities_memory + 0.1*probabilities_demog
probabilities_avg_70 = 0.7*probabilities_memory + 0.3*probabilities_demog
probabilities_avg_50 = 0.5*probabilities_memory + 0.5*probabilities_demog<merge>
training_examples=features_df.iloc[0:2400].copy()
test_examples=features_df.iloc[2400:3000].copy()
training_examples_transform=features_transform_df.iloc[0:2400].copy()
test_examples_transform=features_transform_df.iloc[2400:3000].copy()
predict_col = [i + "_predict" for i in product_col]
predict_previous_col = predict_col + new_product_col
def purchase_nullifier(probabilities,purchase_history,print_option=False):
purchase_history = purchase_history.reset_index(drop=True)
 joined_data = purchase_history.join(probabilities)
 unique_data = joined_data.drop_duplicates().copy().reset_index(drop=True)
 n = unique_data.shape[0]
print("data joined")
 for index,row in unique_data.iterrows() :
if print_option == True:
print(str(index)+ "/" + str(n))
row = row.to_frame().T
row_purchases = row[new_product_col]
row_purchases.columns = row_purchases.columns.str.replace("_previous","")
 row_probabilities = row[predict_col]
row_probabilities.columns = row_probabilities.columns.str.replace("_predict","")
 prob_norm =(1-row_purchases ).multiply(row_probabilities,axis=0)
 if(index == 0):
output_norm = prob_norm
else:
output_norm = output_norm.append(prob_norm)
 print("nullification complete")
 output_index = output_norm.reset_index(drop=True)
 prob_predict = output_index.join(unique_data)
 scaled_predict = pd.merge(joined_data,prob_predict,how='left')
 output = scaled_predict[product_col]
output.columns = output.columns.str.replace("ult1","ult1_predict")
 return output 
<load_pretrained>
def rmsle_cv(model):
rmsle= np.sqrt(-cross_val_score(model, training_examples_transform, training_targets, scoring="neg_mean_squared_log_error", cv = 5))
return(rmsle )
nulled_probabilities_100 = purchase_nullifier(probabilities_memory,traindat_test)
 nulled_probabilities_90 = purchase_nullifier(probabilities_avg_90,traindat_test)
 nulled_probabilities_70 = purchase_nullifier(probabilities_avg_70,traindat_test)
 nulled_probabilities_50 = purchase_nullifier(probabilities_avg_50,traindat_test)
 nulled_probabilities_0 = purchase_nullifier(probabilities_demog,traindat_test )<count_unique_values>
training_targets=Targets_df["bandgap_energy_ev"].copy()
model_linear=LinearRegression().fit(training_examples_transform, training_targets)
 linear_BG_pred = model_linear.predict(test_examples_transform)
 BG_rmsle=rmsle_cv(model_linear ).mean()
print("Band gap RMSLE:")
 print(BG_rmsle,"
")
 training_targets=Targets_df["formation_energy_ev_natom"].copy()
model_linear=LinearRegression().fit(training_examples_transform, training_targets)
 linear_EF_pred = model_linear.predict(test_examples_transform)
 EF_rmsle=rmsle_cv(model_linear ).mean()
print("Formation Energy RMSLE:")
 print(EF_rmsle,"
")
 print("Expected combined RMSLE")
 combined_rmsle=(EF_rmsle+BG_rmsle)/2
print(combined_rmsle)
 Predictions_df=pd.DataFrame()
Predictions_df["id"]=test_id_df["id"].copy()
Predictions_df["formation_energy_ev_natom"]=linear_EF_pred
Predictions_df["bandgap_energy_ev"]=linear_BG_pred
Predictions_df.to_csv("Linear_Nomad.csv",index=False )
def probabilities_to_predictions(probabilities,ncodpers,print_option=False):
unique_probabilities = probabilities.drop_duplicates().copy().reset_index(drop=True)
 print(unique_probabilities.shape)
 n = unique_probabilities.shape[0]
for index, row in unique_probabilities.iterrows() :
if print_option == True:
print(str(index)+ '/' + str(n))
row_use = row.to_frame().T
arank = row_use.apply(np.argsort, axis=1)
 ranked_cols = row_use.columns.to_series() [arank.values[:,::-1][:,:7]]
new_frame = pd.DataFrame(ranked_cols)
 recoms = new_frame[0] + ' ' + new_frame[1] + ' ' + new_frame[2] + ' ' + new_frame[3] + ' ' + new_frame[4] + ' ' + new_frame[5] + ' ' + new_frame[6]
recoms_final = recoms.str.replace('_predict', '', regex=True)
 if(index == 0):
predictions = recoms_final
else:
predictions = predictions.append(recoms_final)
 mapped_predictions = predictions.to_frame().rename(columns={0:'added_products'} ).reset_index(drop=True)
 output_unique = mapped_predictions.join(unique_probabilities)
 output_final = pd.merge(probabilities,output_unique,on=predict_col,how='left')
 no_index_ncodpers = ncodpers.copy().reset_index(drop=True)
 output_ncodpers = no_index_ncodpers.join(output_final['added_products'] ).drop('fecha_dato',axis=1)
 return output_ncodpers<predict_on_test>
training_targets=np.log1p(Targets_df["bandgap_energy_ev"].copy())
 dtrain = xgb.DMatrix(training_examples, label = training_targets)
 dtest = xgb.DMatrix(test_examples)
 params = {"max_depth":2,
"eta":0.1,
'gamma':0,
'subsample':0.8,
'colsample_bytree':1,
'min_child_weight':10,
'reg_alpha':7e-3,
'reg_lambda':1,
}
model_xgb = xgb.cv(params, dtrain, num_boost_round=500,early_stopping_rounds=100)
 model_xgb.loc[30:,["test-rmse-mean", "train-rmse-mean"]].plot()
last=len(model_xgb.loc[:])-1
print(model_xgb.loc[last:,["test-rmse-mean"]] )
predictions_output_100 = probabilities_to_predictions(nulled_probabilities_100,traindat_test_ncodpers)
 predictions_output_90 = probabilities_to_predictions(nulled_probabilities_90,traindat_test_ncodpers)
 predictions_output_70 = probabilities_to_predictions(nulled_probabilities_70,traindat_test_ncodpers)
 predictions_output_50 = probabilities_to_predictions(nulled_probabilities_50,traindat_test_ncodpers)
 predictions_output_0 = probabilities_to_predictions(nulled_probabilities_0,traindat_test_ncodpers )<prepare_output>
model_xgb = xgb.XGBRegressor(n_estimators=last,max_depth=2, learning_rate=0.1,
gamma=0,subsample=0.8,colsample_bytree=1,min_child_weight=10,
reg_alpha=7e-3,reg_lambda=1)
 model_xgb.fit(training_examples, training_targets)
 xgb.plot_importance(model_xgb)
 xgb_BG_preds = np.expm1(model_xgb.predict(test_examples))
evaluation_col = product_col + ['added_products']
def evaluation_metric(predictions,reality,print_option=False):
reality = reality.reset_index(drop=True)
 reality['added_products'] = predictions['added_products']
data_unique = reality.drop_duplicates().copy().reset_index(drop=True)
 predictions_unique = data_unique['added_products'].to_frame()
reality_unique = data_unique.drop('added_products',1)
 n = predictions_unique.shape[0]
for index, row in predictions_unique.iterrows() :
if print_option == True:
print(str(index)+ '/' + str(n))
prediction_use = row.to_frame().T['added_products'].str.split(' ',expand=True ).T
prediction_use = prediction_use.rename(columns={list(prediction_use)[0]:'predict_products'})
 reality_use = reality_unique.iloc[index].to_frame()
reality_use = reality_use.rename(columns={list(reality_use)[0]:'added_products'})
 reality_use['product_name'] = reality_use.index
reality_use = reality_use[reality_use.added_products==1]
reality_use['ind'] = 1
if reality_use.empty:
P = [0]
else:
P = [precision_at_k(prediction_use,reality_use)]
if index == 0:
eval_sum = P
else:
eval_sum.extend(P)
 print('precisions calculated')
 data_unique['precision'] = eval_sum
reality_final = pd.merge(reality,data_unique,on=evaluation_col,how='left')
 U = predictions.shape[0]
output = sum(reality_final.precision)/U
return output
def precision_at_k(prediction,reality):
summand = min(prediction.shape[0],7)
 sum_prec = 0
for k in range(summand):
top_k_predictions = prediction.head(k+1)
 add_vs_pred = pd.merge(reality,top_k_predictions,left_on='product_name',right_on='predict_products',how='inner')
 sum_prec = sum_prec + sum(add_vs_pred.ind)/top_k_predictions.shape[0]
denom = min(reality.shape[0],7)
 output = sum_prec/denom
return output 
<compute_train_metric>
training_targets=np.log1p(Targets_df["formation_energy_ev_natom"].copy())
 dtrain = xgb.DMatrix(training_examples, label = training_targets)
 dtest = xgb.DMatrix(test_examples)
 params = {"max_depth":4,
"eta":0.08,
'gamma':0,
'subsample':1,
'colsample_bytree':0.4,
'min_child_weight':3,
'reg_alpha':0,
'reg_lambda':0,
}
model_xgb = xgb.cv(params, dtrain, num_boost_round=500,early_stopping_rounds=100)
 model_xgb.loc[30:,["test-rmse-mean", "train-rmse-mean"]].plot()
last=len(model_xgb.loc[:])-1
print(model_xgb.loc[last:,["test-rmse-mean"]] )
evaluation_100 = evaluation_metric(predictions_output_100,traindat_purchases_test)
 evaluation_90 = evaluation_metric(predictions_output_90,traindat_purchases_test)
 evaluation_70 = evaluation_metric(predictions_output_70,traindat_purchases_test)
 evaluation_50 = evaluation_metric(predictions_output_50,traindat_purchases_test)
 evaluation_0 = evaluation_metric(predictions_output_0,traindat_purchases_test )<categorify>
model_xgb = xgb.XGBRegressor(n_estimators=last,max_depth=4, learning_rate=0.08,
gamma= 0, colsample=1,colsample_bytree=0.4,min_child_weight=3,
reg_lambda=0)
 model_xgb.fit(training_examples, training_targets)
 xgb.plot_importance(model_xgb)
 xgb_EF_preds = np.expm1(model_xgb.predict(test_examples))
print("all memory: " + str(evaluation_100)+ '
' +
"90% memory: " + str(evaluation_90)+ '
' +
"70% memory: " + str(evaluation_70)+ '
' +
"50% memory: " + str(evaluation_50)+ '
' +
"all demographics: " + str(evaluation_0))<save_to_csv>
Predictions_df=pd.DataFrame()
Predictions_df["id"]=test_id_df["id"].copy()
Predictions_df["formation_energy_ev_natom"]=xgb_EF_preds
Predictions_df["bandgap_energy_ev"]=xgb_BG_preds
Predictions_df.to_csv("XGB_Nomad.csv",index=False )
<define_variables><EOS> 
Predictions_df=pd.DataFrame()
Predictions_df["id"]=test_id_df["id"].copy()
stacked_EF_preds=0.9*xgb_EF_preds+0.1*linear_EF_pred
Predictions_df["formation_energy_ev_natom"]=stacked_EF_preds
stacked_BG_preds=0.9*xgb_BG_preds+0.1*linear_BG_pred
Predictions_df["bandgap_energy_ev"]=stacked_BG_preds
Predictions_df.to_csv("Stacked_Nomad.csv",index=False )