Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
%matplotlib inline
print(os.path.dirname(os.getcwd())+':', os.listdir(os.path.dirname(os.getcwd())));
print(os.getcwd() +':', os.listdir(os.getcwd())) ;
def modify_df(df):
df['cat4'] = df['cat4'].apply(lambda x: x if x == 'B' else 'Z')
 df['cat5'] = df['cat5'].apply(lambda x: x if x in ['B', 'D'] else 'Z')
 df['cat6'] = df['cat6'].apply(lambda x: x if x == 'A' else 'Z')
 df['cat7'] = df['cat7'].apply(lambda x: x if x in ['E', 'D'] else 'Z')
 df['cat8'] = df['cat8'].apply(lambda x: x if x in ['E', 'C', 'G', 'A'] else 'Z')
 df['cont001'] = df['cont8'] * df['cont0']
df['cont002'] = df['cont9'] * df['cont0']
df['cont003'] = df['cont9'] * df['cont5']
df['cont004'] = df['cont8'] * df['cont5']
df['cont005'] = df['cont2'] * df['cont4']
df['cont006'] = df['cont1'] * df['cont3']
df['cont007'] = df['cont13'] * df['cont1']
return df
mod_train_df = modify_df(train_df.copy())
 mod_test_df = modify_df(test_df.copy() )<categorify>
if os.path.isfile('.. /input/train.csv'):
data_df = pd.read_csv('.. /input/train.csv')
 print('train.csv loaded: data_df({0[0]},{0[1]})'.format(data_df.shape))
elif os.path.isfile('data/train.csv'):
data_df = pd.read_csv('data/train.csv')
 print('train.csv loaded: data_df({0[0]},{0[1]})'.format(data_df.shape))
else:
print('Error: train.csv not found')
 print('')
 print(data_df.isnull().any().describe())
 print('')
 print('distinct labels ', data_df['label'].unique())
 print('')
 print(data_df['label'].value_counts() )
for feature in categorical_columns:
le = LabelEncoder()
le.fit(train_df[feature])
 train_df[feature] = le.transform(train_df[feature])
 test_df[feature] = le.transform(test_df[feature])
 for feature in categorical_columns:
le = LabelEncoder()
le.fit(mod_train_df[feature])
 mod_train_df[feature] = le.transform(mod_train_df[feature])
 mod_test_df[feature] = le.transform(mod_test_df[feature] )<split>
def normalize_data(data):
data = data / data.max()
return data
def dense_to_one_hot(labels_dense, num_classes):
num_labels = labels_dense.shape[0]
index_offset = np.arange(num_labels)* num_classes
labels_one_hot = np.zeros(( num_labels, num_classes))
labels_one_hot.flat[index_offset + labels_dense.ravel() ] = 1
return labels_one_hot
def one_hot_to_dense(labels_one_hot):
return np.argmax(labels_one_hot,1)
 def accuracy_from_dense_labels(y_target, y_pred):
y_target = y_target.reshape(-1,)
 y_pred = y_pred.reshape(-1,)
 return np.mean(y_target == y_pred)
 def accuracy_from_one_hot_labels(y_target, y_pred):
y_target = one_hot_to_dense(y_target ).reshape(-1,)
 y_pred = one_hot_to_dense(y_pred ).reshape(-1,)
 return np.mean(y_target == y_pred)
 x_train_valid = data_df.iloc[:,1:].values.reshape(-1,28,28,1)
 x_train_valid = x_train_valid.astype(np.float)
 x_train_valid = normalize_data(x_train_valid)
 image_width = image_height = 28
image_size = 784
y_train_valid_labels = data_df.iloc[:,0].values
labels_count = np.unique(y_train_valid_labels ).shape[0];
plt.figure(figsize=(15,9))
for i in range(50):
plt.subplot(5,10,1+i)
 plt.title(y_train_valid_labels[i])
 plt.imshow(x_train_valid[i].reshape(28,28), cmap=cm.binary)
 y_train_valid = dense_to_one_hot(y_train_valid_labels, labels_count ).astype(np.uint8)
 y_valid_pred = {}
y_train_pred = {}
y_test_pred = {}
train_loss, valid_loss = {}, {}
train_acc, valid_acc = {}, {}
print('x_train_valid.shape = ', x_train_valid.shape)
 print('y_train_valid_labels.shape = ', y_train_valid_labels.shape)
 print('image_size = ', image_size)
 print('image_width = ', image_width)
 print('image_height = ', image_height)
 print('labels_count = ', labels_count)
x = train_df[feature_cols]
y = train_df['target']
feature_cols_mod = mod_train_df.drop(['id', 'target'], axis=1 ).columns
xmod, ymod = mod_train_df[feature_cols_mod], mod_train_df['target']
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=42 )<train_model>
def generate_images(imgs):
image_generator = keras.preprocessing.image.ImageDataGenerator(
 rotation_range = 10, width_shift_range = 0.1 , height_shift_range = 0.1,
zoom_range = 0.1)
 imgs = image_generator.flow(imgs.copy() , np.zeros(len(imgs)) ,
batch_size=len(imgs), shuffle = False ).next()
return imgs[0]
fig,axs = plt.subplots(5,10, figsize=(15,9))
for i in range(5):
n = np.random.randint(0,x_train_valid.shape[0]-2)
 axs[i,0].imshow(x_train_valid[n:n+1].reshape(28,28),cmap=cm.binary)
 axs[i,1].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,2].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,3].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,4].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,5].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,6].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,7].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,8].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,9].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary )
clf = XGBRegressor(random_state=42, tree_method='gpu_hist')
 clf.fit(x_train, y_train )<compute_train_metric>
logreg = sklearn.linear_model.LogisticRegression(verbose=0, solver='lbfgs',
multi_class='multinomial')
 decision_tree = sklearn.tree.DecisionTreeClassifier()
extra_trees = sklearn.ensemble.ExtraTreesClassifier(verbose=0)
 gradient_boost = sklearn.ensemble.GradientBoostingClassifier(verbose=0)
 random_forest = sklearn.ensemble.RandomForestClassifier(verbose=0)
 gaussianNB = sklearn.naive_bayes.GaussianNB()
base_models = {'logreg': logreg, 'extra_trees': extra_trees,
'gradient_boost': gradient_boost, 'random_forest': random_forest,
'decision_tree': decision_tree, 'gaussianNB': gaussianNB}
take_models = ['logreg','random_forest','extra_trees']
for mn in take_models:
train_acc[mn] = []
valid_acc[mn] = []
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True, random_state=123)
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
start = datetime.datetime.now() ;
x_train = x_train_valid[train_index].reshape(-1,784)
 y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index].reshape(-1,784)
 y_valid = y_train_valid[valid_index]
for mn in take_models:
model = sklearn.base.clone(base_models[mn])
 model.fit(x_train, one_hot_to_dense(y_train))
y_train_pred[mn] = model.predict_proba(x_train)
 y_valid_pred[mn] = model.predict_proba(x_valid)
 train_acc[mn].append(accuracy_from_one_hot_labels(y_train_pred[mn], y_train))
valid_acc[mn].append(accuracy_from_one_hot_labels(y_valid_pred[mn], y_valid))
print(i,': '+mn+' train/valid accuracy = %.3f/%.3f'%(train_acc[mn][-1],
valid_acc[mn][-1]))
if False:
break;
print(mn+': averaged train/valid accuracy = %.3f/%.3f'%(np.mean(train_acc[mn]),
np.mean(valid_acc[mn])))
predictions = clf.predict(x_test)
 score_rmse = math.sqrt(mean_squared_error(y_test, predictions))
print(Fore.GREEN + 'Base XGBoost RMSE: {}'.format(score_rmse))<predict_on_test>
class nn_class:
def __init__(self, nn_name = 'nn_1'):
self.s_f_conv1 = 3;
self.n_f_conv1 = 36;
self.s_f_conv2 = 3;
self.n_f_conv2 = 36;
self.s_f_conv3 = 3;
self.n_f_conv3 = 36;
self.n_n_fc1 = 576;
self.mb_size = 50
self.keep_prob = 0.33
self.learn_rate_array = [10*1e-4, 7.5*1e-4, 5*1e-4, 2.5*1e-4, 1*1e-4, 1*1e-4,
1*1e-4,0.75*1e-4, 0.5*1e-4, 0.25*1e-4, 0.1*1e-4,
0.1*1e-4, 0.075*1e-4,0.050*1e-4, 0.025*1e-4, 0.01*1e-4,
0.0075*1e-4, 0.0050*1e-4,0.0025*1e-4,0.001*1e-4]
self.learn_rate_step_size = 3
self.learn_rate = self.learn_rate_array[0]
self.learn_rate_pos = 0
self.index_in_epoch = 0
self.current_epoch = 0
self.log_step = 0.2
self.n_log_step = 0
self.use_tb_summary = False
self.use_tf_saver = False
self.nn_name = nn_name
self.perm_array = np.array([])
 def next_mini_batch(self):
start = self.index_in_epoch
self.index_in_epoch += self.mb_size
self.current_epoch += self.mb_size/len(self.x_train)
 if not len(self.perm_array)== len(self.x_train):
self.perm_array = np.arange(len(self.x_train))
if start == 0:
np.random.shuffle(self.perm_array)
 if self.index_in_epoch > self.x_train.shape[0]:
np.random.shuffle(self.perm_array)
 start = 0
self.index_in_epoch = self.mb_size
if self.train_on_augmented_data:
self.x_train_aug = normalize_data(self.generate_images(self.x_train))
self.y_train_aug = self.y_train
end = self.index_in_epoch
if self.train_on_augmented_data:
x_tr = self.x_train_aug[self.perm_array[start:end]]
y_tr = self.y_train_aug[self.perm_array[start:end]]
else:
x_tr = self.x_train[self.perm_array[start:end]]
y_tr = self.y_train[self.perm_array[start:end]]
return x_tr, y_tr
def generate_images(self, imgs):
print('generate new set of images')
 image_generator = keras.preprocessing.image.ImageDataGenerator(
 rotation_range = 10, width_shift_range = 0.1 , height_shift_range = 0.1,
zoom_range = 0.1)
 imgs = image_generator.flow(imgs.copy() , np.zeros(len(imgs)) ,
batch_size=len(imgs), shuffle = False ).next()
return imgs[0]
def weight_variable(self, shape, name = None):
initial = tf.truncated_normal(shape, stddev=0.1)
 return tf.Variable(initial, name = name)
 def bias_variable(self, shape, name = None):
initial = tf.constant(0.1, shape=shape)
 return tf.Variable(initial, name = name)
 def conv2d(self, x, W, name = None):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME', name = name)
 def max_pool_2x2(self, x, name = None):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding='SAME', name = name)
 def summary_variable(self, var, var_name):
with tf.name_scope(var_name):
mean = tf.reduce_mean(var)
 stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
 tf.summary.scalar('mean', mean)
 tf.summary.scalar('stddev', stddev)
 tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
 def create_graph(self):
tf.reset_default_graph()
self.x_data_tf = tf.placeholder(dtype=tf.float32, shape=[None,28,28,1],
name='x_data_tf')
 self.y_data_tf = tf.placeholder(dtype=tf.float32, shape=[None,10], name='y_data_tf')
 self.W_conv1_tf = self.weight_variable([self.s_f_conv1, self.s_f_conv1, 1,
self.n_f_conv1],
name = 'W_conv1_tf')
 self.b_conv1_tf = self.bias_variable([self.n_f_conv1], name = 'b_conv1_tf')
 self.h_conv1_tf = tf.nn.relu(self.conv2d(self.x_data_tf,
self.W_conv1_tf)+ self.b_conv1_tf,
name = 'h_conv1_tf')
 self.h_pool1_tf = self.max_pool_2x2(self.h_conv1_tf,
name = 'h_pool1_tf')
 self.W_conv2_tf = self.weight_variable([self.s_f_conv2, self.s_f_conv2,
self.n_f_conv1, self.n_f_conv2],
name = 'W_conv2_tf')
 self.b_conv2_tf = self.bias_variable([self.n_f_conv2], name = 'b_conv2_tf')
 self.h_conv2_tf = tf.nn.relu(self.conv2d(self.h_pool1_tf,
self.W_conv2_tf)+ self.b_conv2_tf,
name ='h_conv2_tf')
 self.h_pool2_tf = self.max_pool_2x2(self.h_conv2_tf, name = 'h_pool2_tf')
 self.W_conv3_tf = self.weight_variable([self.s_f_conv3, self.s_f_conv3,
self.n_f_conv2, self.n_f_conv3],
name = 'W_conv3_tf')
 self.b_conv3_tf = self.bias_variable([self.n_f_conv3], name = 'b_conv3_tf')
 self.h_conv3_tf = tf.nn.relu(self.conv2d(self.h_pool2_tf,
self.W_conv3_tf)+ self.b_conv3_tf,
name = 'h_conv3_tf')
 self.h_pool3_tf = self.max_pool_2x2(self.h_conv3_tf,
name = 'h_pool3_tf')
 self.W_fc1_tf = self.weight_variable([4*4*self.n_f_conv3,self.n_n_fc1],
name = 'W_fc1_tf')
 self.b_fc1_tf = self.bias_variable([self.n_n_fc1], name = 'b_fc1_tf')
 self.h_pool3_flat_tf = tf.reshape(self.h_pool3_tf, [-1,4*4*self.n_f_conv3],
name = 'h_pool3_flat_tf')
 self.h_fc1_tf = tf.nn.relu(tf.matmul(self.h_pool3_flat_tf,
self.W_fc1_tf)+ self.b_fc1_tf,
name = 'h_fc1_tf')
 self.keep_prob_tf = tf.placeholder(dtype=tf.float32, name = 'keep_prob_tf')
 self.h_fc1_drop_tf = tf.nn.dropout(self.h_fc1_tf, self.keep_prob_tf,
name = 'h_fc1_drop_tf')
 self.W_fc2_tf = self.weight_variable([self.n_n_fc1, 10], name = 'W_fc2_tf')
 self.b_fc2_tf = self.bias_variable([10], name = 'b_fc2_tf')
 self.z_pred_tf = tf.add(tf.matmul(self.h_fc1_drop_tf, self.W_fc2_tf),
self.b_fc2_tf, name = 'z_pred_tf')
 self.cross_entropy_tf = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
 labels=self.y_data_tf, logits=self.z_pred_tf), name = 'cross_entropy_tf')
 self.learn_rate_tf = tf.placeholder(dtype=tf.float32, name="learn_rate_tf")
 self.train_step_tf = tf.train.AdamOptimizer(self.learn_rate_tf ).minimize(
 self.cross_entropy_tf, name = 'train_step_tf')
 self.y_pred_proba_tf = tf.nn.softmax(self.z_pred_tf, name='y_pred_proba_tf')
 self.y_pred_correct_tf = tf.equal(tf.argmax(self.y_pred_proba_tf, 1),
tf.argmax(self.y_data_tf, 1),
name = 'y_pred_correct_tf')
 self.accuracy_tf = tf.reduce_mean(tf.cast(self.y_pred_correct_tf, dtype=tf.float32),
name = 'accuracy_tf')
 self.train_loss_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='train_loss_tf', validate_shape = False)
 self.valid_loss_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='valid_loss_tf', validate_shape = False)
 self.train_acc_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='train_acc_tf', validate_shape = False)
 self.valid_acc_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='valid_acc_tf', validate_shape = False)
 num_weights =(self.s_f_conv1**2*self.n_f_conv1
+ self.s_f_conv2**2*self.n_f_conv1*self.n_f_conv2
+ self.s_f_conv3**2*self.n_f_conv2*self.n_f_conv3
+ 4*4*self.n_f_conv3*self.n_n_fc1 + self.n_n_fc1*10)
 num_biases = self.n_f_conv1 + self.n_f_conv2 + self.n_f_conv3 + self.n_n_fc1
print('num_weights =', num_weights)
 print('num_biases =', num_biases)
 return None
def attach_summary(self, sess):
self.use_tb_summary = True
self.summary_variable(self.W_conv1_tf, 'W_conv1_tf')
 self.summary_variable(self.b_conv1_tf, 'b_conv1_tf')
 self.summary_variable(self.W_conv2_tf, 'W_conv2_tf')
 self.summary_variable(self.b_conv2_tf, 'b_conv2_tf')
 self.summary_variable(self.W_conv3_tf, 'W_conv3_tf')
 self.summary_variable(self.b_conv3_tf, 'b_conv3_tf')
 self.summary_variable(self.W_fc1_tf, 'W_fc1_tf')
 self.summary_variable(self.b_fc1_tf, 'b_fc1_tf')
 self.summary_variable(self.W_fc2_tf, 'W_fc2_tf')
 self.summary_variable(self.b_fc2_tf, 'b_fc2_tf')
 tf.summary.scalar('cross_entropy_tf', self.cross_entropy_tf)
 tf.summary.scalar('accuracy_tf', self.accuracy_tf)
 self.merged = tf.summary.merge_all()
timestamp = datetime.datetime.now().strftime('%d-%m-%Y_%H-%M-%S')
 filepath = os.path.join(os.getcwd() , 'logs',(self.nn_name+'_'+timestamp))
self.train_writer = tf.summary.FileWriter(os.path.join(filepath,'train'), sess.graph)
 self.valid_writer = tf.summary.FileWriter(os.path.join(filepath,'valid'), sess.graph)
 def attach_saver(self):
self.use_tf_saver = True
self.saver_tf = tf.train.Saver()
def train_graph(self, sess, x_train, y_train, x_valid, y_valid, n_epoch = 1,
train_on_augmented_data = False):
self.train_on_augmented_data = train_on_augmented_data
self.x_train = x_train
self.y_train = y_train
self.x_valid = x_valid
self.y_valid = y_valid
if self.train_on_augmented_data:
print('generate new set of images')
 self.x_train_aug = normalize_data(self.generate_images(self.x_train))
self.y_train_aug = self.y_train
mb_per_epoch = self.x_train.shape[0]/self.mb_size
train_loss, train_acc, valid_loss, valid_acc = [],[],[],[]
start = datetime.datetime.now() ;
print(datetime.datetime.now().strftime('%d-%m-%Y %H:%M:%S'),': start training')
 print('learnrate = ',self.learn_rate,', n_epoch = ', n_epoch,
', mb_size = ', self.mb_size)
 for i in range(int(n_epoch*mb_per_epoch)+1):
self.learn_rate_pos = int(self.current_epoch // self.learn_rate_step_size)
 if not self.learn_rate == self.learn_rate_array[self.learn_rate_pos]:
self.learn_rate = self.learn_rate_array[self.learn_rate_pos]
print(datetime.datetime.now() -start,': set learn rate to %.6f'%self.learn_rate)
 x_batch, y_batch = self.next_mini_batch()
sess.run(self.train_step_tf, feed_dict={self.x_data_tf: x_batch,
self.y_data_tf: y_batch,
self.keep_prob_tf: self.keep_prob,
self.learn_rate_tf: self.learn_rate})
 if i%int(self.log_step*mb_per_epoch)== 0 or i == int(n_epoch*mb_per_epoch):
self.n_log_step += 1
feed_dict_train = {
self.x_data_tf: self.x_train[self.perm_array[:len(self.x_valid)]],
self.y_data_tf: self.y_train[self.perm_array[:len(self.y_valid)]],
self.keep_prob_tf: 1.0}
feed_dict_valid = {self.x_data_tf: self.x_valid,
self.y_data_tf: self.y_valid,
self.keep_prob_tf: 1.0}
if self.use_tb_summary:
train_summary = sess.run(self.merged, feed_dict = feed_dict_train)
 valid_summary = sess.run(self.merged, feed_dict = feed_dict_valid)
 self.train_writer.add_summary(train_summary, self.n_log_step)
 self.valid_writer.add_summary(valid_summary, self.n_log_step)
 train_loss.append(sess.run(self.cross_entropy_tf,
feed_dict = feed_dict_train))
train_acc.append(self.accuracy_tf.eval(session = sess,
feed_dict = feed_dict_train))
valid_loss.append(sess.run(self.cross_entropy_tf,
feed_dict = feed_dict_valid))
valid_acc.append(self.accuracy_tf.eval(session = sess,
feed_dict = feed_dict_valid))
print('%.2f epoch: train/val loss = %.4f/%.4f, train/val acc = %.4f/%.4f'%(
 self.current_epoch, train_loss[-1], valid_loss[-1],
train_acc[-1], valid_acc[-1]))
tl_c = np.concatenate([self.train_loss_tf.eval(session=sess), train_loss], axis = 0)
 vl_c = np.concatenate([self.valid_loss_tf.eval(session=sess), valid_loss], axis = 0)
 ta_c = np.concatenate([self.train_acc_tf.eval(session=sess), train_acc], axis = 0)
 va_c = np.concatenate([self.valid_acc_tf.eval(session=sess), valid_acc], axis = 0)
 sess.run(tf.assign(self.train_loss_tf, tl_c, validate_shape = False))
sess.run(tf.assign(self.valid_loss_tf, vl_c , validate_shape = False))
sess.run(tf.assign(self.train_acc_tf, ta_c , validate_shape = False))
sess.run(tf.assign(self.valid_acc_tf, va_c , validate_shape = False))
print('running time for training: ', datetime.datetime.now() - start)
 return None
def save_model(self, sess):
if self.use_tf_saver:
filepath = os.path.join(os.getcwd() , self.nn_name)
 self.saver_tf.save(sess, filepath)
 if self.use_tb_summary:
self.train_writer.close()
self.valid_writer.close()
return None
def forward(self, sess, x_data):
y_pred_proba = self.y_pred_proba_tf.eval(session = sess,
feed_dict = {self.x_data_tf: x_data,
self.keep_prob_tf: 1.0})
 return y_pred_proba
def load_tensors(self, graph):
self.x_data_tf = graph.get_tensor_by_name("x_data_tf:0")
 self.y_data_tf = graph.get_tensor_by_name("y_data_tf:0")
 self.W_conv1_tf = graph.get_tensor_by_name("W_conv1_tf:0")
 self.W_conv2_tf = graph.get_tensor_by_name("W_conv2_tf:0")
 self.W_conv3_tf = graph.get_tensor_by_name("W_conv3_tf:0")
 self.W_fc1_tf = graph.get_tensor_by_name("W_fc1_tf:0")
 self.W_fc2_tf = graph.get_tensor_by_name("W_fc2_tf:0")
 self.b_conv1_tf = graph.get_tensor_by_name("b_conv1_tf:0")
 self.b_conv2_tf = graph.get_tensor_by_name("b_conv2_tf:0")
 self.b_conv3_tf = graph.get_tensor_by_name("b_conv3_tf:0")
 self.b_fc1_tf = graph.get_tensor_by_name("b_fc1_tf:0")
 self.b_fc2_tf = graph.get_tensor_by_name("b_fc2_tf:0")
 self.h_conv1_tf = graph.get_tensor_by_name('h_conv1_tf:0')
 self.h_pool1_tf = graph.get_tensor_by_name('h_pool1_tf:0')
 self.h_conv2_tf = graph.get_tensor_by_name('h_conv2_tf:0')
 self.h_pool2_tf = graph.get_tensor_by_name('h_pool2_tf:0')
 self.h_conv3_tf = graph.get_tensor_by_name('h_conv3_tf:0')
 self.h_pool3_tf = graph.get_tensor_by_name('h_pool3_tf:0')
 self.h_fc1_tf = graph.get_tensor_by_name('h_fc1_tf:0')
 self.z_pred_tf = graph.get_tensor_by_name('z_pred_tf:0')
 self.learn_rate_tf = graph.get_tensor_by_name("learn_rate_tf:0")
 self.keep_prob_tf = graph.get_tensor_by_name("keep_prob_tf:0")
 self.cross_entropy_tf = graph.get_tensor_by_name('cross_entropy_tf:0')
 self.train_step_tf = graph.get_operation_by_name('train_step_tf')
 self.z_pred_tf = graph.get_tensor_by_name('z_pred_tf:0')
 self.y_pred_proba_tf = graph.get_tensor_by_name("y_pred_proba_tf:0")
 self.y_pred_correct_tf = graph.get_tensor_by_name('y_pred_correct_tf:0')
 self.accuracy_tf = graph.get_tensor_by_name('accuracy_tf:0')
 self.train_loss_tf = graph.get_tensor_by_name("train_loss_tf:0")
 self.train_acc_tf = graph.get_tensor_by_name("train_acc_tf:0")
 self.valid_loss_tf = graph.get_tensor_by_name("valid_loss_tf:0")
 self.valid_acc_tf = graph.get_tensor_by_name("valid_acc_tf:0")
 return None
def get_loss(self, sess):
train_loss = self.train_loss_tf.eval(session = sess)
 valid_loss = self.valid_loss_tf.eval(session = sess)
 return train_loss, valid_loss
def get_accuracy(self, sess):
train_acc = self.train_acc_tf.eval(session = sess)
 valid_acc = self.valid_acc_tf.eval(session = sess)
 return train_acc, valid_acc
def get_weights(self, sess):
W_conv1 = self.W_conv1_tf.eval(session = sess)
 W_conv2 = self.W_conv2_tf.eval(session = sess)
 W_conv3 = self.W_conv3_tf.eval(session = sess)
 W_fc1_tf = self.W_fc1_tf.eval(session = sess)
 W_fc2_tf = self.W_fc2_tf.eval(session = sess)
 return W_conv1, W_conv2, W_conv3, W_fc1_tf, W_fc2_tf
def get_biases(self, sess):
b_conv1 = self.b_conv1_tf.eval(session = sess)
 b_conv2 = self.b_conv2_tf.eval(session = sess)
 b_conv3 = self.b_conv3_tf.eval(session = sess)
 b_fc1_tf = self.b_fc1_tf.eval(session = sess)
 b_fc2_tf = self.b_fc2_tf.eval(session = sess)
 return b_conv1, b_conv2, b_conv3, b_fc1_tf, b_fc2_tf
def load_session_from_file(self, filename):
tf.reset_default_graph()
filepath = os.path.join(os.getcwd() , filename + '.meta')
 saver = tf.train.import_meta_graph(filepath)
 print(filepath)
 sess = tf.Session()
saver.restore(sess, mn)
 graph = tf.get_default_graph()
self.load_tensors(graph)
 return sess
def get_activations(self, sess, x_data):
feed_dict = {self.x_data_tf: x_data, self.keep_prob_tf: 1.0}
h_conv1 = self.h_conv1_tf.eval(session = sess, feed_dict = feed_dict)
 h_pool1 = self.h_pool1_tf.eval(session = sess, feed_dict = feed_dict)
 h_conv2 = self.h_conv2_tf.eval(session = sess, feed_dict = feed_dict)
 h_pool2 = self.h_pool2_tf.eval(session = sess, feed_dict = feed_dict)
 h_conv3 = self.h_conv3_tf.eval(session = sess, feed_dict = feed_dict)
 h_pool3 = self.h_pool3_tf.eval(session = sess, feed_dict = feed_dict)
 h_fc1 = self.h_fc1_tf.eval(session = sess, feed_dict = feed_dict)
 h_fc2 = self.z_pred_tf.eval(session = sess, feed_dict = feed_dict)
 return h_conv1,h_pool1,h_conv2,h_pool2,h_conv3,h_pool3,h_fc1,h_fc2
sub_xgb_base = clf.predict(test_df[feature_cols] )<train_model>
nn_name = ['tmp']
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True, random_state=123)
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
start = datetime.datetime.now() ;
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index]
nn_graph = nn_class(nn_name = nn_name[i])
 nn_graph.create_graph()
nn_graph.attach_saver()
with tf.Session() as sess:
nn_graph.attach_summary(sess)
 sess.run(tf.global_variables_initializer())
 nn_graph.train_graph(sess, x_train, y_train, x_valid, y_valid, n_epoch = 1.0)
 nn_graph.train_graph(sess, x_train, y_train, x_valid, y_valid, n_epoch = 14.0,
train_on_augmented_data = True)
 nn_graph.save_model(sess)
 if True:
break;
print('total running time for training: ', datetime.datetime.now() - start)
clf = LGBMRegressor(random_state=42, device='gpu')
 clf.fit(x_train, y_train )<compute_train_metric>
if False:
!tensorboard --logdir=./logs
predictions = clf.predict(x_test)
 score_rmse = math.sqrt(mean_squared_error(y_test, predictions))
print(Fore.GREEN + 'Base LGBM RMSE: {}'.format(score_rmse))<define_variables>
mn = nn_name[0]
nn_graph = nn_class()
sess = nn_graph.load_session_from_file(mn)
 W_conv1, W_conv2, W_conv3, _, _ = nn_graph.get_weights(sess)
 sess.close()
print('W_conv1: min = ' + str(np.min(W_conv1)) + ' max = ' + str(np.max(W_conv1))
+ ' mean = ' + str(np.mean(W_conv1)) + ' std = ' + str(np.std(W_conv1)))
 print('W_conv2: min = ' + str(np.min(W_conv2)) + ' max = ' + str(np.max(W_conv2))
+ ' mean = ' + str(np.mean(W_conv2)) + ' std = ' + str(np.std(W_conv2)))
 print('W_conv3: min = ' + str(np.min(W_conv3)) + ' max = ' + str(np.max(W_conv3))
+ ' mean = ' + str(np.mean(W_conv3)) + ' std = ' + str(np.std(W_conv3)))
 s_f_conv1 = nn_graph.s_f_conv1
s_f_conv2 = nn_graph.s_f_conv2
s_f_conv3 = nn_graph.s_f_conv3
W_conv1 = np.reshape(W_conv1,(s_f_conv1,s_f_conv1,1,6,6))
W_conv1 = np.transpose(W_conv1,(3,0,4,1,2))
W_conv1 = np.reshape(W_conv1,(s_f_conv1*6,s_f_conv1*6,1))
W_conv2 = np.reshape(W_conv2,(s_f_conv2,s_f_conv2,6,6,36))
W_conv2 = np.transpose(W_conv2,(2,0,3,1,4))
W_conv2 = np.reshape(W_conv2,(6*s_f_conv2,6*s_f_conv2,6,6))
W_conv2 = np.transpose(W_conv2,(2,0,3,1))
W_conv2 = np.reshape(W_conv2,(6*6*s_f_conv2,6*6*s_f_conv2))
W_conv3 = np.reshape(W_conv3,(s_f_conv3,s_f_conv3,6,6,36))
W_conv3 = np.transpose(W_conv3,(2,0,3,1,4))
W_conv3 = np.reshape(W_conv3,(6*s_f_conv3,6*s_f_conv3,6,6))
W_conv3 = np.transpose(W_conv3,(2,0,3,1))
W_conv3 = np.reshape(W_conv3,(6*6*s_f_conv3,6*6*s_f_conv3))
plt.figure(figsize=(15,5))
plt.subplot(1,3,1)
 plt.gca().set_xticks(np.arange(-0.5, s_f_conv1*6, s_f_conv1), minor = False);
plt.gca().set_yticks(np.arange(-0.5, s_f_conv1*6, s_f_conv1), minor = False);
plt.grid(which = 'minor', color='b', linestyle='-', linewidth=1)
 plt.title('W_conv1 ' + str(W_conv1.shape))
plt.colorbar(plt.imshow(W_conv1[:,:,0], cmap=cm.binary)) ;
plt.subplot(1,3,2)
 plt.gca().set_xticks(np.arange(-0.5, 6*6*s_f_conv2, 6*s_f_conv2), minor = False);
plt.gca().set_yticks(np.arange(-0.5, 6*6*s_f_conv2, 6*s_f_conv2), minor = False);
plt.grid(which = 'minor', color='b', linestyle='-', linewidth=1)
 plt.title('W_conv2 ' + str(W_conv2.shape))
plt.colorbar(plt.imshow(W_conv2[:,:], cmap=cm.binary)) ;
plt.subplot(1,3,3)
 plt.gca().set_xticks(np.arange(-0.5, 6*6*s_f_conv3, 6*s_f_conv3), minor = False);
plt.gca().set_yticks(np.arange(-0.5, 6*6*s_f_conv3, 6*s_f_conv3), minor = False);
plt.grid(which = 'minor', color='b', linestyle='-', linewidth=1)
 plt.title('W_conv3 ' + str(W_conv3.shape))
plt.colorbar(plt.imshow(W_conv3[:,:], cmap=cm.binary)) ;
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape<train_model>
mn = nn_name[0]
nn_graph = nn_class()
sess = nn_graph.load_session_from_file(mn)
 y_valid_pred[mn] = nn_graph.forward(sess, x_valid)
 sess.close()
y_valid_pred_label = one_hot_to_dense(y_valid_pred[mn])
 y_valid_label = one_hot_to_dense(y_valid)
 y_val_false_index = []
for i in range(y_valid_label.shape[0]):
if y_valid_pred_label[i] != y_valid_label[i]:
y_val_false_index.append(i)
 print('
plt.figure(figsize=(10,15))
for j in range(0,5):
for i in range(0,10):
if j*10+i<len(y_val_false_index):
plt.subplot(10,10,j*10+i+1)
 plt.title('%d/%d'%(y_valid_label[y_val_false_index[j*10+i]],
y_valid_pred_label[y_val_false_index[j*10+i]]))
plt.imshow(x_valid[y_val_false_index[j*10+i]].reshape(28,28),cmap=cm.binary )
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(x, y))):
tmp_train_df, tmp_val_df = x.iloc[train_ind][feature_cols], x.iloc[val_ind][feature_cols]
train_target, val_target = y[train_ind], y[val_ind]
model = LGBMRegressor(random_state=42, device='gpu')
 model.fit(tmp_train_df, train_target, eval_set=[(tmp_val_df, val_target)], verbose=False)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(test_df[feature_cols])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))<save_to_csv>
if os.path.isfile('.. /input/test.csv'):
test_df = pd.read_csv('.. /input/test.csv')
 print('test.csv loaded: test_df{0}'.format(test_df.shape))
elif os.path.isfile('data/test.csv'):
test_df = pd.read_csv('data/test.csv')
 print('test.csv loaded: test_df{0}'.format(test_df.shape))
else:
print('Error: test.csv not found')
 x_test = test_df.iloc[:,0:].values.reshape(-1,28,28,1)
 x_test = x_test.astype(np.float)
 x_test = normalize_data(x_test)
 print('x_test.shape = ', x_test.shape)
 y_test_pred = {}
y_test_pred_labels = {}
sub_df['target'] = test_preds
sub_df.to_csv('submission_lgbm_cv.csv', index=False)
 sub_df.head()
sub_lgbm_cv = test_preds<init_hyperparams>
if False:
take_models = ['nn0','nn1','nn2','nn3','nn4','nn5','nn6','nn7','nn8','nn9']
kfold = sklearn.model_selection.KFold(len(take_models), shuffle=True, random_state = 123)
 x_train_meta = np.array([] ).reshape(-1,10)
 y_train_meta = np.array([] ).reshape(-1,10)
 x_test_meta = np.zeros(( x_test.shape[0], 10))
print('Out-of-folds predictions:')
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index]
mn = take_models[i]
nn_graph = nn_class()
sess = nn_graph.load_session_from_file(mn)
 y_train_pred[mn] = nn_graph.forward(sess, x_train[:len(x_valid)])
 y_valid_pred[mn] = nn_graph.forward(sess, x_valid)
 y_test_pred[mn] = nn_graph.forward(sess, x_test)
 sess.close()
x_train_meta = np.concatenate([x_train_meta, y_valid_pred[mn]])
 y_train_meta = np.concatenate([y_train_meta, y_valid])
 x_test_meta += y_test_pred[mn]
print(take_models[i],': train/valid accuracy = %.4f/%.4f'%(
 accuracy_from_one_hot_labels(y_train_pred[mn], y_train[:len(x_valid)]),
accuracy_from_one_hot_labels(y_valid_pred[mn], y_valid)))
 if False:
break;
x_test_meta = x_test_meta/(i+1)
 y_test_pred['stacked_models'] = x_test_meta
print('')
 print('Stacked models: valid accuracy = %.4f'%accuracy_from_one_hot_labels(x_train_meta,
y_train_meta))
xgb_params = {
'booster':'gbtree',
'n_estimators':20000,
'max_depth':5,
'eta':0.008,
'gamma':3.5,
'objective':'reg:squarederror',
'verbosity':0,
'subsample':0.75,
'colsample_bytree':0.35,
'reg_lambda':0.23,
'reg_alpha':0.52,
'scale_pos_weight':1,
'objective':'reg:squarederror',
'eval_metric':'rmse',
'seed': 42,
'tree_method':'gpu_hist',
'gpu_id':0
}<train_model>
if False:
logreg = sklearn.linear_model.LogisticRegression(verbose=0, solver='lbfgs',
multi_class='multinomial')
 take_meta_model = 'logreg'
model = sklearn.base.clone(base_models[take_meta_model])
 model.fit(x_train_meta, one_hot_to_dense(y_train_meta))
y_train_pred['meta_model'] = model.predict_proba(x_train_meta)
 y_test_pred['meta_model'] = model.predict_proba(x_test_meta)
 print('Meta model: train accuracy = %.4f'%accuracy_from_one_hot_labels(x_train_meta,
y_train_pred['meta_model']))
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
NUM_FOLDS = 5
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=42)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(x, y))):
tmp_train_df, tmp_val_df = x.iloc[train_ind][feature_cols], x.iloc[val_ind][feature_cols]
train_target, val_target = y[train_ind], y[val_ind]
model = XGBRegressor(**xgb_params)
 model.fit(
 tmp_train_df,
train_target,
eval_set=[(tmp_val_df, val_target)],
early_stopping_rounds = 50,
verbose = False 
)
 temp_oof = model.predict(tmp_val_df)
 temp_test = model.predict(test_df[feature_cols])
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))<save_to_csv>
if True:
mn = nn_name[0]
nn_graph = nn_class()
sess = nn_graph.load_session_from_file(mn)
 y_test_pred = {}
y_test_pred_labels = {}
kfold = sklearn.model_selection.KFold(40, shuffle=False)
 for i,(train_index, valid_index)in enumerate(kfold.split(x_test)) :
if i==0:
y_test_pred[mn] = nn_graph.forward(sess, x_test[valid_index])
 else:
y_test_pred[mn] = np.concatenate([y_test_pred[mn],
nn_graph.forward(sess, x_test[valid_index])])
 sess.close()
<init_hyperparams><EOS> 
mn = nn_name[0]
y_test_pred_labels[mn] = one_hot_to_dense(y_test_pred[mn])
 print(mn+': y_test_pred_labels[mn].shape = ', y_test_pred_labels[mn].shape)
 unique, counts = np.unique(y_test_pred_labels[mn], return_counts=True)
 print(dict(zip(unique, counts)))
 np.savetxt('submission.csv',
np.c_[range(1,len(x_test)+1), y_test_pred_labels[mn]],
delimiter=',',
header = 'ImageId,Label',
comments = '',
fmt='%d')
 print('submission.csv completed' )