Kaggle competition: Nomad2018 Predicting Transparent Conductors
df_test.renta.fillna(df_test["renta"].median() , inplace=True )<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
replace_mapping = {
"sexo": {"V":0,'H':1},
}
df_train.replace(replace_mapping, inplace= True )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
dummy_col = ['sexo','canal_entrada','cod_prov','segmento']<count_values>
spacegroups = train.spacegroup.unique()
spacegroups.sort()
print(spacegroups)
 n_spacegroups = len(spacegroups )
limit = int(0.05 * len(df_train.index))
for col in dummy_col:
if len(df_train[col].unique())> 6:
trainlist = df_train[col].value_counts()
print(trainlist)
 use_col = []
for i,item in enumerate(trainlist):
if item > limit:
use_col.append(df_train[col].value_counts().index[i])
 for item in df_train[col].unique() :
if item not in use_col:
row_index = df_train[col] == item
df_train.loc[row_index,col] = np.nan
for item in df_test[col].unique() :
if item not in use_col:
row_index = df_test[col] == item
df_test.loc[row_index,col] = np.nan<categorify>
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler, StandardScaler
df_train = pd.get_dummies(df_train, prefix=dummy_col, columns = dummy_col)
 df_test = pd.get_dummies(df_test, prefix=dummy_col, columns = dummy_col )<categorify>
t1 = 'formation_energy_ev_natom'
t2 = 'bandgap_energy_ev'
transform_columns = ['number_of_total_atoms', 'percent_atom_al', 'percent_atom_ga', 'percent_atom_in', 'lattice_vector_1_ang', 'lattice_vector_2_ang', 'lattice_vector_3_ang', 'lattice_angle_alpha_degree', 'lattice_angle_beta_degree', 'lattice_angle_gamma_degree']
feature_columns = ['spacegroup'] + transform_columns
df_train["age"] = pd.to_numeric(df_train["age"], errors="coerce")
 df_test["age"] = pd.to_numeric(df_test["age"], errors="coerce")
 age_group = [[0,19],[19,26],[26,36],[36,41],[41,47],[47,55],[55,60],[60,70],[70,80],[80,170]]
def create_age_group(df):
df['age_group'] = np.nan
for i,age in enumerate(age_group):
row_index =(df['age'] >= age[0])&(df['age'] < age[1])
 df.loc[row_index,'age_group'] = i
create_age_group(df_train)
 create_age_group(df_test)
 df_train.drop('age', axis=1, inplace = True)
 df_test.drop('age', axis=1, inplace = True )<train_model>
import matplotlib.pyplot as plt
import matplotlib.cm as cm
models = {}
model_preds = {}
id_preds = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = LogisticRegression()
clf.fit(x_train, y_train)
 x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds[id].append(p)
<load_from_csv>
all = pd.concat([train[feature_columns], test])
 scaler = MinMaxScaler()
scaler.fit(all[transform_columns])
 train[transform_columns] = scaler.transform(train[transform_columns])
 test[transform_columns] = scaler.transform(test[transform_columns] )
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col)
 df_recent = df_recent.drop_duplicates(['ncodpers'], keep='last' )<load_from_csv>
random_seed = 314
X_train, X_validation = train_test_split(train, test_size=0.2, random_state=random_seed)
 y_train = np.log1p(X_train[[t1, t2]])
 X_train = X_train.drop(['id', t1, t2], axis=1)
 y_validation = np.log1p(X_validation[[t1, t2]])
 X_validation = X_validation.drop(['id', t1, t2], axis=1)
 print(X_train.shape, y_train.shape)
 print(X_validation.shape, y_validation.shape )
df_recent = pd.read_csv('.. /input/santander-product-recommendation/train_ver2.csv',usecols=['ncodpers'] + product_col)
<load_from_csv>
import tensorflow as tf
sample = pd.read_csv('.. /input/santander-product-recommendation/sample_submission.csv' )<define_variables>
tf.set_random_seed(1)
 np.random.seed(1)
 layers = [16] * 8
activation = tf.tanh
kernel_regularizer_l2 = None
already_active = {}
for row in df_recent.values:
row = list(row)
 id = row.pop(0)
 active = [c[0] for c in zip(tuple(product_col), row)if c[1] > 0]
already_active[id] = active
train_preds = {}
for id, p in id_preds.items() :
preds = [i[0] for i in sorted([i for i in zip(tuple(product_col), p)if i[0] not in already_active[id]],
key=lambda i:i [1],
reverse=True)[:7]]
train_preds[id] = preds
test_preds = []
for row in sample.values:
id = row[0]
p = train_preds[id]
test_preds.append(' '.join(p)) 
<prepare_output>
tf.reset_default_graph()
tf_is_training = tf.placeholder(tf.bool, None)
 tf_x = tf.placeholder(tf.float32,(None, X_train.shape[1]), name='tf_x')
 tf_y = tf.placeholder(tf.float32,(None, 2), name='tf_y' )
sample['added_products'] = test_preds 
<save_to_csv>
l = tf_x
for i, n in enumerate(layers):
l = tf.layers.dense(l, n, activation=activation, kernel_regularizer=kernel_regularizer_l2, name='layer%s' %(i + 1))
output = tf.layers.dense(l, 2, name='output' )
sample.to_csv('LR_sub.csv', index=False)
 print(datetime.datetime.now() -start_time)
<define_variables>
loss_y = tf.sqrt(tf.reduce_mean(tf.squared_difference(tf_y, output), 0), name='tf_y_loss')
 loss_total = tf.reduce_mean(loss_y, name='tf_total_loss')
 rl = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
 if len(rl)> 0:
loss_total += tf.add_n(rl)
 optimizer = tf.train.AdamOptimizer(learning_rate=0.002)
 train_op = optimizer.minimize(loss_total )
start_time = datetime.datetime.now()
demographic_cols = ['ncodpers','fecha_alta','ind_empleado','pais_residencia','sexo','age','ind_nuevo','antiguedad','indrel',
'indrel_1mes','tiprel_1mes','indresi','indext','conyuemp','canal_entrada','indfall',
'tipodom','cod_prov','ind_actividad_cliente','renta','segmento']
notuse = ["ult_fec_cli_1t","nomprov",'fecha_dato']
product_col = [
'ind_ahor_fin_ult1','ind_aval_fin_ult1','ind_cco_fin_ult1','ind_cder_fin_ult1','ind_cno_fin_ult1','ind_ctju_fin_ult1',
'ind_ctma_fin_ult1','ind_ctop_fin_ult1','ind_ctpp_fin_ult1','ind_deco_fin_ult1','ind_deme_fin_ult1',
'ind_dela_fin_ult1','ind_ecue_fin_ult1','ind_fond_fin_ult1','ind_hip_fin_ult1','ind_plan_fin_ult1',
'ind_pres_fin_ult1','ind_reca_fin_ult1','ind_tjcr_fin_ult1','ind_valo_fin_ult1','ind_viv_fin_ult1','ind_nomina_ult1',
'ind_nom_pens_ult1','ind_recibo_ult1']<load_from_csv>
sess = tf.Session()
sess.run(tf.global_variables_initializer() )
df_train = pd.read_csv('.. /input/datamulticlass-6-withpast2/DataMulticlass_6_withpast2.csv' )<load_from_csv>
loss_data = []
for step in range(500):
_, lt, lt_y = sess.run([train_op, loss_total, loss_y], {tf_x: X_train, tf_y: y_train, tf_is_training: True})
 lv, lv_y = sess.run([loss_total, loss_y], {tf_x: X_validation, tf_y: y_validation, tf_is_training: False})
 loss_data.append([step, lt, lt_y[0], lt_y[1], lv, lv_y[0], lv_y[1]])
 loss_data = np.array(loss_data)
 print(loss_data[-1][1:] )
df_test = pd.read_csv('.. /input/testset-withpast3/TestSet_withpast3.csv' )<set_options>
loss, pred_y = sess.run([loss_total, output], {tf_x: X_validation, tf_y: y_validation, tf_is_training: False})
 print('loss:', loss )
pd.set_option('display.max_columns', None )<filter>
pd_y_pred = pd.DataFrame()
pd_y_pred[t1] = pred_y[:, 0]
pd_y_pred[t2] = pred_y[:, 1]
pd_y_pred['index'] = y_validation.index
pd_y_pred.set_index('index', inplace=True )
def filter_data(df):
df = df[df['ind_nuevo'] == 0]
df = df[df['antiguedad'] != -999999]
df = df[df['indrel'] == 1]
df = df[df['indresi'] == 'S']
df = df[df['indfall'] == 'N']
df = df[df['tipodom'] == 1]
df = df[df['ind_empleado'] == 'N']
df = df[df['pais_residencia'] == 'ES']
df = df[df['indrel_1mes'] == 1]
df = df[df['tiprel_1mes'] ==('A' or 'I')]
df = df[df['indext'] == 'N']<drop_column>
for i, g in enumerate(spacegroups):
labels = y_validation.loc[X_validation[X_validation['spacegroup'] == g].index]
predictions = pd_y_pred.loc[X_validation[X_validation['spacegroup'] == g].index]
loss = np.sqrt(mean_squared_error(labels, predictions, multioutput='raw_values'))
print('spacegroup=%d' % g, *loss, np.mean(loss))
drop_column = ['ind_nuevo','indrel','indresi','indfall','tipodom','ind_empleado','pais_residencia','indrel_1mes','indext','conyuemp','fecha_alta','tiprel_1mes']
df_train.drop(drop_column, axis=1, inplace = True)
 df_test.drop(drop_column, axis=1, inplace = True )<feature_engineering>
sample = pd.read_csv('.. /input/sample_submission.csv')
 sample.head()
df_test["renta"] = pd.to_numeric(df_test["renta"], errors="coerce")
 unique_prov = df_test[df_test.cod_prov.notnull() ].cod_prov.unique()
grouped = df_test.groupby("cod_prov")["renta"].median()
def impute_renta(df):
df["renta"] = pd.to_numeric(df["renta"], errors="coerce")
 for cod in unique_prov:
df.loc[df['cod_prov']==cod,['renta']] = df.loc[df['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
df.renta.fillna(df_test["renta"].median() , inplace=True)
 impute_renta(df_train)
 impute_renta(df_test )<drop_column>
y_train = np.log1p(train[[t1, t2]])
 X_train = train.drop(['id', t1, t2], axis=1)
 sess = tf.Session()
sess.run(tf.global_variables_initializer())
 loss_data = []
for step in range(2000):
_, l, l_y = sess.run([train_op, loss_total, loss_y], {tf_x: X_train, tf_y: y_train, tf_is_training: True})
 loss_data.append([step, l, l_y[0], l_y[1]])
 print(loss_data[-1][1:])
 loss_data = np.array(loss_data )
<define_variables><EOS> 
X_test = test.drop(['id'], axis=1)
 pred_y = sess.run(output, {tf_x: X_test, tf_is_training: False})
 pred_y = np.expm1(pred_y)
 pred_y[pred_y[:, 0] < 0, 0] = 0
pred_y[pred_y[:, 1] < 0, 1] = 0
subm = pd.DataFrame()
subm['id'] = sample['id']
subm[t1] = pred_y[:, 0]
subm[t2] = pred_y[:, 1]
subm.to_csv("subm.csv", index=False)
 subm.head()