Kaggle competition: Nomad2018 Predicting Transparent Conductors
<SOS> metric: MCRMSLE Kaggle data source: nomad2018-predicting-transparent-conductors<feature_engineering>
%matplotlib inline
for pro in product_col[2:]:
df_train[pro+'_past'] = df_train[pro+'_past']*(1-df_train[pro+'_5'])
 df_test[pro+'_past'] = df_test[pro+'_past']*(1-df_test[pro+'_5'] )<feature_engineering>
df_train = pd.read_csv('.. /input/train.csv')
 df_train['dataset'] = 'train'
df_test = pd.read_csv('.. /input/test.csv')
 df_test['dataset'] = 'test'
test_len = len(df_test)
 df = pd.concat([df_train, df_test], axis=0, ignore_index=True, sort=False)
 df_len = len(df )
for col in product_col[2:]:
for month in range(2,6):
df_train[col+'_'+str(month)+'_diff'] = df_train[col+'_'+str(month)] - df_train[col+'_'+str(month-1)]
df_test[col+'_'+str(month)+'_diff'] = df_test[col+'_'+str(month)] - df_test[col+'_'+str(month-1)]
df_train[col+'_'+str(month)+'_add'] = df_train[col+'_'+str(month)+'_diff'].apply(lambda x: max(x,0))
df_test[col+'_'+str(month)+'_add'] = df_test[col+'_'+str(month)+'_diff'].apply(lambda x: max(x,0))<define_variables>
def get_xyz(filename):
row = []
xyz = []
lattice = []
with open(filename)as f:
for line in f.readlines() :
row = line.split()
if row[0] == 'atom':
xyz.append(( np.array(row[1:4], dtype=np.float), row[4]))
elif row[0] == 'lattice_vector':
lattice.append(np.array(row[1:4], dtype=np.float))
return xyz, lattice
product_col_5_diff = [col for col in df_train.columns if '5_diff' in col]
product_col_4_diff = [col for col in df_train.columns if '4_diff' in col]
product_col_3_diff = [col for col in df_train.columns if '3_diff' in col]
product_col_2_diff = [col for col in df_train.columns if '2_diff' in col]
product_col_5_add = [col for col in df_train.columns if '5_add' in col]
product_col_4_add = [col for col in df_train.columns if '4_add' in col]
product_col_3_add = [col for col in df_train.columns if '3_add' in col]
product_col_2_add = [col for col in df_train.columns if '2_add' in col]
product_col_all_diff = [col for col in df_train.columns if '_diff' in col]
product_col_all_add = [col for col in df_train.columns if '_add' in col]<feature_engineering>
def get_sine_matrix(xyz, lattice):
n_atom = len(xyz)
 distance_matrix = np.ones(( n_atom, n_atom))
A = np.transpose(lattice)
 B = LA.inv(A)
 for i in range(n_atom):
for j in range(i):
r_ij = np.dot(B, xyz[i][0] - xyz[j][0])
 sin_sq_r =(np.sin(np.pi * r_ij)) **2
distance = LA.norm(np.dot(A, sin_sq_r))
distance_matrix[i, j], distance_matrix[j, i] = distance, distance
labels = np.transpose(xyz)[1]
labels = labels.reshape(-1,1)
 for at, charge in zip(['O', 'Al', 'Ga', 'In'], [8, 13, 31, 49]):
labels = np.where(labels==at, charge, labels)
 charge_matrix = np.dot(labels, np.transpose(labels)).astype(np.float)
 charge_matrix -= np.diag(np.diag(charge_matrix))
charge_matrix += np.diag(0.5 * labels**2.4 ).astype(float)
 sine_matrix = charge_matrix / distance_matrix
return sine_matrix
df_train['tot5_add'] = df_train[product_col_5_add].sum(axis=1)
 df_test['tot5_add'] = df_test[product_col_5_add].sum(axis=1)
 df_train['tot4_add'] = df_train[product_col_4_add].sum(axis=1)
 df_test['tot4_add'] = df_test[product_col_4_add].sum(axis=1)
 df_train['tot3_add'] = df_train[product_col_3_add].sum(axis=1)
 df_test['tot3_add'] = df_test[product_col_3_add].sum(axis=1)
 df_train['tot2_add'] = df_train[product_col_2_add].sum(axis=1)
 df_test['tot2_add'] = df_test[product_col_2_add].sum(axis=1 )<prepare_x_and_y>
def get_eigenspectrum(matrix):
spectrum = LA.eigvalsh(matrix)
 spectrum = np.sort(spectrum)[::-1]
return spectrum
cols = list(df_train.drop(['target','ncodpers']+product_col_all_diff+product_col_all_add, 1 ).columns.values)
 id_preds = defaultdict(list)
 ids = df_test['ncodpers'].values
y_train = pd.get_dummies(df_train['target'].astype(int))
x_train = df_train[cols]<choose_model_class>
spectrum_list = []
for index in range(df_len):
dataset_label = df.dataset.values[index]
row_id = df.id.values[index]
filename = ".. /input/{}/{}/geometry.xyz".format(dataset_label, row_id)
 xyz, lattice = get_xyz(filename)
 sine_matrix = get_sine_matrix(xyz, lattice)
 spectrum = get_eigenspectrum(sine_matrix)
 spectrum_list.append(spectrum )
model = Sequential()
model.add(Dense(150, input_dim=len(cols), activation='relu'))
model.add(Dense(22, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adagrad', metrics=['categorical_accuracy'])
 model.fit(x_train.as_matrix() , y_train.as_matrix() , validation_split=0.2, nb_epoch=150, batch_size=10)
 x_test = df_test[cols]
x_test = x_test.fillna(0)
 p_test = model.predict(x_test.as_matrix())
 for id, p in zip(ids, p_test):
id_preds[id] = [0,0] + list(p )<define_variables>
spectrum_df = pd.DataFrame(spectrum_list ).astype(np.float)
 spectrum_df = spectrum_df.fillna(0 )
model.inputs<find_best_params>
ss = StandardScaler()
spectrum_std_df = pd.DataFrame(ss.fit_transform(spectrum_df.values))
pca = PCA(n_components=80)
 spectrum_pca_df = pd.DataFrame(pca.fit_transform(spectrum_std_df.values))
spectrum_pca_df.head()
model.outputs<prepare_output>
df.number_of_total_atoms = df.number_of_total_atoms.astype('int')
 df['group_natoms'] = df.spacegroup.astype('str')+ '_' + df.number_of_total_atoms.astype('str' )
fraction = 1
id_preds_combined = {}
for uid, p in id_preds.items() :
id_preds_combined[uid] = fraction*np.asarray(id_preds[uid])
 id_preds = id_preds_combined<define_variables>
df = df.join(pd.get_dummies(df.group_natoms))
df.drop(['group_natoms'], axis=1, inplace=True )
usecols = ['ncodpers', 'ind_ahor_fin_ult1', 'ind_aval_fin_ult1', 'ind_cco_fin_ult1',
'ind_cder_fin_ult1', 'ind_cno_fin_ult1', 'ind_ctju_fin_ult1',
'ind_ctma_fin_ult1', 'ind_ctop_fin_ult1', 'ind_ctpp_fin_ult1',
'ind_deco_fin_ult1', 'ind_deme_fin_ult1', 'ind_dela_fin_ult1',
'ind_ecue_fin_ult1', 'ind_fond_fin_ult1', 'ind_hip_fin_ult1',
'ind_plan_fin_ult1', 'ind_pres_fin_ult1', 'ind_reca_fin_ult1',
'ind_tjcr_fin_ult1', 'ind_valo_fin_ult1', 'ind_viv_fin_ult1',
'ind_nomina_ult1', 'ind_nom_pens_ult1', 'ind_recibo_ult1']<load_from_csv>
df.drop(['dataset'], axis=1, inplace=True)
 df.drop(['id', 'spacegroup', 'number_of_total_atoms'], axis=1, inplace=True )
df_recent = pd.read_csv('.. /input/train-ver2/train_ver2.csv',usecols=usecols)
 df_recent=df_recent[df_recent['ncodpers'].isin(customer)]
df_recent.fillna(0, inplace=True )<load_from_csv>
df_new = pd.concat([spectrum_pca_df, df], axis=1)
 df_new.head()
sample = pd.read_csv('.. /input/sample-submission/sample_submission.csv' )<define_variables>
df_new.drop([395, 126, 1215, 1886, 2075, 353, 308, 2154, 531, 1379, 2319, 2337, 2370, 2333], axis=0, inplace=True )
already_active = {}
for row in df_recent.values:
row = list(row)
 id = row.pop(0)
 active = [c[0] for c in zip(tuple(product_col), row)if c[1] > 0]
already_active[id] = active
train_preds = {}
for id, p in id_preds.items() :
preds = [i[0] for i in sorted([i for i in zip(tuple(product_col), p)if i[0] not in already_active[id]],
key=lambda i:i [1],
reverse=True)[:7]]
train_preds[id] = preds
test_preds = []
for row in sample.values:
id = row[0]
p = train_preds[id]
test_preds.append(' '.join(p)) 
<save_model>
df_len = len(df_new)
 train_len = df_len - test_len
X_train = df_new.drop(['formation_energy_ev_natom', 'bandgap_energy_ev'], axis=1)[:train_len].values
X_test = df_new.drop(['formation_energy_ev_natom', 'bandgap_energy_ev'], axis=1)[train_len:].values
y_formation = df_new['formation_energy_ev_natom'][:train_len].values
y_bandgap = df_new['bandgap_energy_ev'][:train_len].values
model.save('kerass.model' )<save_to_csv>
xgb_formation = xgb.XGBRegressor()
parameters = {
'max_depth': [2, 3, 4],
'n_estimators' : [100, 200, 300],
}
cv_formation = GridSearchCV(xgb_formation, param_grid=parameters, cv=4, verbose=1)
 cv_formation.fit(X_train, y_formation )
sample['added_products'] = test_preds
sample.to_csv('Keras1.csv', index=False)
 print(datetime.datetime.now() -start_time )<define_variables>
cv_formation.best_params_
start_time = datetime.datetime.now()
demographic_cols = ['fecha_dato',
'ncodpers','ind_empleado','pais_residencia','sexo','age','fecha_alta','ind_nuevo','antiguedad','indrel',
'indrel_1mes','tiprel_1mes','indresi','indext','conyuemp','canal_entrada','indfall',
'tipodom','cod_prov','ind_actividad_cliente','renta','segmento']
notuse = ["ult_fec_cli_1t","nomprov"]
product_col = [
'ind_ahor_fin_ult1','ind_aval_fin_ult1','ind_cco_fin_ult1','ind_cder_fin_ult1','ind_cno_fin_ult1','ind_ctju_fin_ult1',
'ind_ctma_fin_ult1','ind_ctop_fin_ult1','ind_ctpp_fin_ult1','ind_deco_fin_ult1','ind_deme_fin_ult1',
'ind_dela_fin_ult1','ind_ecue_fin_ult1','ind_fond_fin_ult1','ind_hip_fin_ult1','ind_plan_fin_ult1',
'ind_pres_fin_ult1','ind_reca_fin_ult1','ind_tjcr_fin_ult1','ind_valo_fin_ult1','ind_viv_fin_ult1','ind_nomina_ult1',
'ind_nom_pens_ult1','ind_recibo_ult1']
train_cols = demographic_cols + product_col<load_from_csv>
xgb_bandgap = xgb.XGBRegressor()
parameters = {
'max_depth': [2, 3, 4],
'n_estimators' : [100, 200, 300],
}
cv_bandgap = GridSearchCV(xgb_bandgap, param_grid=parameters, cv=4, verbose=1)
 cv_bandgap.fit(X_train, y_bandgap )
df_train = pd.read_csv('.. /input/juneextra/juneExtra.csv', usecols=train_cols )<load_from_csv>
cv_bandgap.best_params_
df_test = pd.read_csv('.. /input/santander-product-recommendation/test_ver2.csv', usecols = demographic_cols )<filter>
formation_pred = cv_formation.predict(X_test)
 bandgap_pred = cv_bandgap.predict(X_test )
df_train = df_train[df_train['ind_nuevo'] == 0]
df_train = df_train[df_train['antiguedad'] != -999999]
df_train = df_train[df_train['indrel'] == 1]
df_train = df_train[df_train['indresi'] == 'S']
df_train = df_train[df_train['indfall'] == 'N']
df_train = df_train[df_train['tipodom'] == 1]
df_train = df_train[df_train['ind_empleado'] == 'N']
df_train = df_train[df_train['pais_residencia'] == 'ES']
df_train = df_train[df_train['indrel_1mes'] == 1]
df_train = df_train[df_train['tiprel_1mes'] ==('A' or 'I')]
df_train = df_train[df_train['indext'] == 'N']<count_missing_values>
submission = pd.DataFrame(np.arange(1, test_len + 1), columns=['id'])
 submission['formation_energy_ev_natom'] = formation_pred
submission['bandgap_energy_ev'] = bandgap_pred
submission.shape
df_train.isnull().sum()<count_values>
submission.to_csv('submission.csv', index=False )
<drop_column><EOS> 
submission.to_csv('submission.csv', index=False )