Kaggle competition: Dogs vs. Cats Redux: Kernels Edition
<SOS> metric: LogLoss Kaggle data source: dogs-vs.-cats-redux:-kernels-edition<prepare_x_and_y>
%%time
!pip install --use-feature=2020-resolver https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds/xgboost-1.3.0_SNAPSHOT%2B00b0ad1293b4fa74d6aca5da4e9ab7a9d16777f0-py3-none-manylinux2010_x86_64.whl
lossArr = np.array([])
 accArr= np.array([])
 val_lossArr = np.array([])
 val_accArr= np.array([])
 epochArr= np.array([])
 pred_500X300_kp90 = pd.DataFrame()
pred_500X300_kp90['Id']= train_df['Id']
learning_rate = 0.001
epochs = 5000
batch_size = 80
tf.reset_default_graph()
X = tf.placeholder(dtype=tf.float32, shape = [None, 403])
 Y = tf.placeholder(dtype=tf.float32, shape = [None, 1])
 keep_prob = tf.placeholder(tf.float32)
 is_training = tf.placeholder(tf.bool)
 n_samples = y_trainRanked.shape[0]
W1 = tf.Variable(tf.ones([403, 500]), name='W1',dtype=tf.float32)
 b1 = tf.Variable(tf.zeros([500]), name='b1',dtype=tf.float32)
 W2 = tf.Variable(tf.ones([500, 300]), name='W2',dtype=tf.float32)
 b2 = tf.Variable(tf.zeros([300]), name='b2',dtype=tf.float32)
 W3 = tf.Variable(tf.ones([300, 1]), name='W3',dtype=tf.float32)
 b3 = tf.Variable(tf.zeros([1]), name='b3',dtype=tf.float32)
 hidden_out = tf.add(tf.matmul(X, W1), b1)
 hidden_out = bn_layer_top(hidden_out,'hidden_out_bn', is_training=is_training)
 hidden_out = tf.nn.relu(hidden_out)
 drop_out = tf.nn.dropout(hidden_out, keep_prob)
 hidden_out2 = tf.add(tf.matmul(drop_out, W2), b2)
 hidden_out2 = bn_layer_top(hidden_out2,'hidden_out_bn2', is_training=is_training)
 hidden_out2 = tf.nn.relu(hidden_out2)
 drop_out2 = tf.nn.dropout(hidden_out2, keep_prob)
 pred = tf.add(tf.matmul(drop_out2, W3), b3)
 deltas = tf.abs(pred - Y)
 loss = tf.reduce_sum(deltas)/batch_size
optimiser = tf.train.AdamOptimizer(learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 total_batch = int(len(X_trainRanked)/ batch_size)
 val_total_batch = int(len(X_valRanked)/ batch_size)
 for epoch in range(epochs):
avg_cost = 0
avg_acc = 0
avg_val_cost = 0
avg_val_acc = 0
for i in range(total_batch):
batch_x, batch_y = next_batch(num=batch_size,data=X_trainRanked,labels=y_trainRanked)
 _, c = sess.run([optimiser, loss],
feed_dict={X: batch_x, Y: batch_y, keep_prob : 0.90, is_training: True},)
 train_pred=sess.run(pred, feed_dict={X: batch_x,keep_prob : 1.0, is_training: False})
 c= rmse(batch_y,train_pred)
 a= r2_score(batch_y, train_pred,multioutput='variance_weighted')
 avg_cost += c / total_batch
avg_acc += a / total_batch
for i in range(val_total_batch):
val_batch_x, val_batch_y = next_batch(num=batch_size,data=X_valRanked,labels=y_valRanked)
 val_pred=sess.run(pred, feed_dict={X: val_batch_x,keep_prob : 1.0, is_training: False})
 val_c= rmse(val_batch_y,val_pred)
 val_a= r2_score(val_batch_y, val_pred,multioutput='variance_weighted')
 avg_val_cost += val_c / val_total_batch
avg_val_acc += val_a / val_total_batch
epoch=epoch + 1
lossArr = np.append(lossArr, [avg_cost])
 accArr= np.append(accArr, [avg_acc])
 val_lossArr = np.append(val_lossArr, [avg_val_cost])
 val_accArr= np.append(val_accArr, [avg_val_acc])
 epochArr= np.append(epochArr, [epoch])
 pred_500X300_kp90['SalePrice'] = sess.run(pred, feed_dict={X: train_df_munged,keep_prob : 1.0, is_training: False})
 def lcurves(ylim_loss_start=0.000,ylim_loss_end=0.200,ylim_acc_start=0.8000,ylim_acc_end=1.0000):
xnew = np.linspace(epochArr.min() ,epochArr.max() ,50)
 spl = make_interp_spline(epochArr, lossArr, k=3)
 val_spl = make_interp_spline(epochArr, val_lossArr, k=3)
 power_smooth = spl(xnew)
 val_power_smooth = val_spl(xnew)
 plt.ylim(ylim_loss_start,ylim_loss_end)
 plt.plot(xnew,power_smooth, 'b', label='Training loss')
 plt.plot(xnew,val_power_smooth, 'r', label='Validation loss')
 plt.title('Training and Validation loss')
 plt.legend()
plt.show()
spl = make_interp_spline(epochArr, accArr, k=3)
 val_spl = make_interp_spline(epochArr, val_accArr, k=3)
 power_smooth = spl(xnew)
 val_power_smooth = val_spl(xnew)
 plt.ylim(ylim_acc_start,ylim_acc_end)
 plt.plot(xnew,power_smooth, 'b', label='Training accuracy')
 plt.plot(xnew,val_power_smooth, 'r', label='Validation accuracy')
 plt.title('Training and Validation accuracy')
 plt.legend()
plt.show()
lcurves() 
<prepare_x_and_y>
xgb.__version__
lossArr = np.array([])
 accArr= np.array([])
 val_lossArr = np.array([])
 val_accArr= np.array([])
 epochArr= np.array([])
 pred_200X200_kp90 = pd.DataFrame()
pred_200X200_kp90['Id']= train_df['Id']
learning_rate = 0.001
epochs = 4
batch_size = 80
tf.reset_default_graph()
X = tf.placeholder(dtype=tf.float32, shape = [None, 403])
 Y = tf.placeholder(dtype=tf.float32, shape = [None, 1])
 keep_prob = tf.placeholder(tf.float32)
 is_training = tf.placeholder(tf.bool)
 n_samples = y_trainRanked.shape[0]
W1 = tf.Variable(tf.ones([403, 200]), name='W1',dtype=tf.float32)
 b1 = tf.Variable(tf.zeros([200]), name='b1',dtype=tf.float32)
 W2 = tf.Variable(tf.ones([200, 200]), name='W2',dtype=tf.float32)
 b2 = tf.Variable(tf.zeros([200]), name='b2',dtype=tf.float32)
 W3 = tf.Variable(tf.ones([200, 1]), name='W3',dtype=tf.float32)
 b3 = tf.Variable(tf.zeros([1]), name='b3',dtype=tf.float32)
 hidden_out = tf.add(tf.matmul(X, W1), b1)
 hidden_out = bn_layer_top(hidden_out,'hidden_out_bn', is_training=is_training)
 hidden_out = tf.nn.relu(hidden_out)
 drop_out = tf.nn.dropout(hidden_out, keep_prob)
 hidden_out2 = tf.add(tf.matmul(drop_out, W2), b2)
 hidden_out2 = bn_layer_top(hidden_out2,'hidden_out_bn2', is_training=is_training)
 hidden_out2 = tf.nn.relu(hidden_out2)
 drop_out2 = tf.nn.dropout(hidden_out2, keep_prob)
 pred = tf.add(tf.matmul(drop_out2, W3), b3)
 deltas = tf.abs(pred - Y)
 loss = tf.reduce_sum(deltas)/batch_size
optimiser = tf.train.AdamOptimizer(learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 total_batch = int(len(X_trainRanked)/ batch_size)
 val_total_batch = int(len(X_valRanked)/ batch_size)
 for epoch in range(epochs):
avg_cost = 0
avg_acc = 0
avg_val_cost = 0
avg_val_acc = 0
for i in range(total_batch):
batch_x, batch_y = next_batch(num=batch_size,data=X_trainRanked,labels=y_trainRanked)
 _, c = sess.run([optimiser, loss],
feed_dict={X: batch_x, Y: batch_y, keep_prob : 0.90, is_training: True},)
 train_pred=sess.run(pred, feed_dict={X: batch_x,keep_prob : 1.0, is_training: False})
 c= rmse(batch_y,train_pred)
 a= r2_score(batch_y, train_pred, multioutput='variance_weighted')
 avg_cost += c / total_batch
avg_acc += a / total_batch
val_pred=sess.run(pred, feed_dict={X: X_valRanked,keep_prob : 1.0, is_training: False})
 val_c= rmse(y_valRanked,val_pred)
 val_a= r2_score(y_valRanked, val_pred, multioutput='variance_weighted')
 avg_val_cost = val_c
avg_val_acc = val_a
print("Epoch:",(epoch + 1), "cost =", "{:.3f}".format(avg_cost))
lossArr = np.append(lossArr, [avg_cost])
 accArr= np.append(accArr, [avg_acc])
 val_lossArr = np.append(val_lossArr, [avg_val_cost])
 val_accArr= np.append(val_accArr, [avg_val_acc])
 epochArr= np.append(epochArr, [epoch])
 pred_200X200_kp90['SalePrice'] = sess.run(pred, feed_dict={X: train_df_munged,keep_prob : 1.0, is_training: False})
 lcurves()<prepare_x_and_y>
import numpy as np
import pandas as pd
import zipfile
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import log_loss, roc_auc_score
import gc
import os
lossArr = np.array([])
 accArr= np.array([])
 val_lossArr = np.array([])
 val_accArr= np.array([])
 epochArr= np.array([])
 pred_50X50_kp90 = pd.DataFrame()
pred_50X50_kp90['Id']= train_df['Id']
learning_rate = 0.001
epochs = 4
batch_size = 80
tf.reset_default_graph()
X = tf.placeholder(dtype=tf.float32, shape = [None, 403])
 Y = tf.placeholder(dtype=tf.float32, shape = [None, 1])
 keep_prob = tf.placeholder(tf.float32)
 is_training = tf.placeholder(tf.bool)
 n_samples = y_trainRanked.shape[0]
W1 = tf.Variable(tf.ones([403, 50]), name='W1',dtype=tf.float32)
 b1 = tf.Variable(tf.zeros([50]), name='b1',dtype=tf.float32)
 W2 = tf.Variable(tf.ones([50, 50]), name='W2',dtype=tf.float32)
 b2 = tf.Variable(tf.zeros([50]), name='b2',dtype=tf.float32)
 W3 = tf.Variable(tf.ones([50, 1]), name='W3',dtype=tf.float32)
 b3 = tf.Variable(tf.zeros([1]), name='b3',dtype=tf.float32)
 hidden_out = tf.add(tf.matmul(X, W1), b1)
 hidden_out = bn_layer_top(hidden_out,'hidden_out_bn', is_training=is_training)
 hidden_out = tf.nn.relu(hidden_out)
 drop_out = tf.nn.dropout(hidden_out, keep_prob)
 hidden_out2 = tf.add(tf.matmul(drop_out, W2), b2)
 hidden_out2 = bn_layer_top(hidden_out2,'hidden_out_bn2', is_training=is_training)
 hidden_out2 = tf.nn.relu(hidden_out2)
 drop_out2 = tf.nn.dropout(hidden_out2, keep_prob)
 pred = tf.add(tf.matmul(drop_out2, W3), b3)
 deltas = tf.abs(pred - Y)
 loss = tf.reduce_sum(deltas)/batch_size
optimiser = tf.train.AdamOptimizer(learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 total_batch = int(len(X_trainRanked)/ batch_size)
 val_total_batch = int(len(X_valRanked)/ batch_size)
 for epoch in range(epochs):
avg_cost = 0
avg_acc = 0
avg_val_cost = 0
avg_val_acc = 0
for i in range(total_batch):
batch_x, batch_y = next_batch(num=batch_size,data=X_trainRanked,labels=y_trainRanked)
 _, c = sess.run([optimiser, loss],
feed_dict={X: batch_x, Y: batch_y, keep_prob : 0.90, is_training: True},)
 train_pred=sess.run(pred, feed_dict={X: batch_x,keep_prob : 1.0, is_training: False})
 c= rmse(batch_y,train_pred)
 a= r2_score(batch_y, train_pred, multioutput='variance_weighted')
 avg_cost += c / total_batch
avg_acc += a / total_batch
val_pred=sess.run(pred, feed_dict={X: X_valRanked,keep_prob : 1.0, is_training: False})
 val_c= rmse(y_valRanked,val_pred)
 val_a= r2_score(y_valRanked, val_pred, multioutput='variance_weighted')
 avg_val_cost = val_c
avg_val_acc = val_a
print("Epoch:",(epoch + 1), "cost =", "{:.3f}".format(avg_cost))
lossArr = np.append(lossArr, [avg_cost])
 accArr= np.append(accArr, [avg_acc])
 val_lossArr = np.append(val_lossArr, [avg_val_cost])
 val_accArr= np.append(val_accArr, [avg_val_acc])
 epochArr= np.append(epochArr, [epoch])
 pred_50X50_kp90['SalePrice'] = sess.run(pred, feed_dict={X: train_df_munged,keep_prob : 1.0, is_training: False})
 lcurves()<prepare_x_and_y>
sample_submission = pd.read_csv('/kaggle/input/dogs-vs-cats-redux-kernels-edition/sample_submission.csv')
 sample_submission.tail()
lossArr = np.array([])
 accArr= np.array([])
 val_lossArr = np.array([])
 val_accArr= np.array([])
 epochArr= np.array([])
 pred_500_kp90 = pd.DataFrame()
pred_500_kp90['Id']= train_df['Id']
learning_rate = 0.001
epochs = 4
batch_size = 80
tf.reset_default_graph()
X = tf.placeholder(dtype=tf.float32, shape = [None, 403])
 Y = tf.placeholder(dtype=tf.float32, shape = [None, 1])
 keep_prob = tf.placeholder(tf.float32)
 is_training = tf.placeholder(tf.bool)
 n_samples = y_trainRanked.shape[0]
W1 = tf.Variable(tf.ones([403, 500]), name='W1',dtype=tf.float32)
 b1 = tf.Variable(tf.zeros([500]), name='b1',dtype=tf.float32)
 W2 = tf.Variable(tf.ones([500, 1]), name='W2',dtype=tf.float32)
 b2 = tf.Variable(tf.zeros([1]), name='b2',dtype=tf.float32)
 hidden_out = tf.add(tf.matmul(X, W1), b1)
 hidden_out = bn_layer_top(hidden_out,'hidden_out_bn', is_training=is_training)
 hidden_out = tf.nn.relu(hidden_out)
 drop_out = tf.nn.dropout(hidden_out, keep_prob)
 pred = tf.add(tf.matmul(drop_out, W2), b2)
 deltas = tf.abs(pred - Y)
 loss = tf.reduce_sum(deltas)/batch_size
optimiser = tf.train.AdamOptimizer(learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 total_batch = int(len(X_trainRanked)/ batch_size)
 val_total_batch = int(len(X_valRanked)/ batch_size)
 for epoch in range(epochs):
avg_cost = 0
avg_acc = 0
avg_val_cost = 0
avg_val_acc = 0
for i in range(total_batch):
batch_x, batch_y = next_batch(num=batch_size,data=X_trainRanked,labels=y_trainRanked)
 _, c = sess.run([optimiser, loss],
feed_dict={X: batch_x, Y: batch_y, keep_prob : 0.90, is_training: True},)
 train_pred=sess.run(pred, feed_dict={X: batch_x,keep_prob : 1.0, is_training: False})
 c= rmse(batch_y,train_pred)
 a= r2_score(batch_y, train_pred, multioutput='variance_weighted')
 avg_cost += c / total_batch
avg_acc += a / total_batch
val_pred=sess.run(pred, feed_dict={X: X_valRanked,keep_prob : 1.0, is_training: False})
 val_c= rmse(y_valRanked,val_pred)
 val_a= r2_score(y_valRanked, val_pred, multioutput='variance_weighted')
 avg_val_cost = val_c
avg_val_acc = val_a
print("Epoch:",(epoch + 1), "cost =", "{:.3f}".format(avg_cost))
lossArr = np.append(lossArr, [avg_cost])
 accArr= np.append(accArr, [avg_acc])
 val_lossArr = np.append(val_lossArr, [avg_val_cost])
 val_accArr= np.append(val_accArr, [avg_val_acc])
 epochArr= np.append(epochArr, [epoch])
 pred_500_kp90['SalePrice'] = sess.run(pred, feed_dict={X: train_df_munged,keep_prob : 1.0, is_training: False})
 lcurves(20,100,-100,100 )<prepare_x_and_y>
TRAIN_FOLDER = '.. /input/cats-and-dogs-embedded-data/train/train/'
TEST_FOLDER = '.. /input/cats-and-dogs-embedded-data/test/test/'
IMG_SIZE = 224
lossArr = np.array([])
 accArr= np.array([])
 val_lossArr = np.array([])
 val_accArr= np.array([])
 epochArr= np.array([])
 pred_500X300_kp50 = pd.DataFrame()
pred_500X300_kp50['Id']= train_df['Id']
learning_rate = 0.001
epochs = 1
batch_size = 80
tf.reset_default_graph()
X = tf.placeholder(dtype=tf.float32, shape = [None, 403])
 Y = tf.placeholder(dtype=tf.float32, shape = [None, 1])
 keep_prob = tf.placeholder(tf.float32)
 is_training = tf.placeholder(tf.bool)
 n_samples = y_trainRanked.shape[0]
W1 = tf.Variable(tf.ones([403, 500]), name='W1',dtype=tf.float32)
 b1 = tf.Variable(tf.zeros([500]), name='b1',dtype=tf.float32)
 W2 = tf.Variable(tf.ones([500, 300]), name='W2',dtype=tf.float32)
 b2 = tf.Variable(tf.zeros([300]), name='b2',dtype=tf.float32)
 W3 = tf.Variable(tf.ones([300, 1]), name='W3',dtype=tf.float32)
 b3 = tf.Variable(tf.zeros([1]), name='b3',dtype=tf.float32)
 hidden_out = tf.add(tf.matmul(X, W1), b1)
 hidden_out = bn_layer_top(hidden_out,'hidden_out_bn', is_training=is_training)
 hidden_out = tf.nn.relu(hidden_out)
 drop_out = tf.nn.dropout(hidden_out, keep_prob)
 hidden_out2 = tf.add(tf.matmul(drop_out, W2), b2)
 hidden_out2 = bn_layer_top(hidden_out2,'hidden_out_bn2', is_training=is_training)
 hidden_out2 = tf.nn.relu(hidden_out2)
 drop_out2 = tf.nn.dropout(hidden_out2, keep_prob)
 pred = tf.add(tf.matmul(drop_out2, W3), b3)
 deltas = tf.abs(pred - Y)
 loss = tf.reduce_sum(deltas)/batch_size
optimiser = tf.train.AdamOptimizer(learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 total_batch = int(len(X_trainRanked)/ batch_size)
 val_total_batch = int(len(X_valRanked)/ batch_size)
 for epoch in range(epochs):
avg_cost = 0
avg_acc = 0
avg_val_cost = 0
avg_val_acc = 0
for i in range(total_batch):
batch_x, batch_y = next_batch(num=batch_size,data=X_trainRanked,labels=y_trainRanked)
 _, c = sess.run([optimiser, loss],
feed_dict={X: batch_x, Y: batch_y, keep_prob : 0.50, is_training: True},)
 train_pred=sess.run(pred, feed_dict={X: batch_x,keep_prob : 1.0, is_training: False})
 c= rmse(batch_y,train_pred)
 a= r2_score(batch_y, train_pred, multioutput='variance_weighted')
 avg_cost += c / total_batch
avg_acc += a / total_batch
val_pred=sess.run(pred, feed_dict={X: X_valRanked,keep_prob : 1.0, is_training: False})
 val_c= rmse(y_valRanked,val_pred)
 val_a= r2_score(y_valRanked, val_pred, multioutput='variance_weighted')
 avg_val_cost = val_c
avg_val_acc = val_a
print("Epoch:",(epoch + 1), "cost =", "{:.3f}".format(avg_cost))
lossArr = np.append(lossArr, [avg_cost])
 accArr= np.append(accArr, [avg_acc])
 val_lossArr = np.append(val_lossArr, [avg_val_cost])
 val_accArr= np.append(val_accArr, [avg_val_acc])
 epochArr= np.append(epochArr, [epoch])
 pred_500X300_kp50['SalePrice'] = sess.run(pred, feed_dict={X: train_df_munged,keep_prob : 1.0, is_training: False})
 lcurves()<prepare_x_and_y>
train_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_image_list.npy')
 test_image_list = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_image_list.npy' )
lossArr = np.array([])
 accArr= np.array([])
 val_lossArr = np.array([])
 val_accArr= np.array([])
 epochArr= np.array([])
 pred_500X300_kp80 = pd.DataFrame()
pred_500X300_kp80['Id']= train_df['Id']
learning_rate = 0.001
epochs = 5000
batch_size = 80
tf.reset_default_graph()
X = tf.placeholder(dtype=tf.float32, shape = [None, 403])
 Y = tf.placeholder(dtype=tf.float32, shape = [None, 1])
 keep_prob = tf.placeholder(tf.float32)
 is_training = tf.placeholder(tf.bool)
 n_samples = y_trainRanked.shape[0]
W1 = tf.Variable(tf.ones([403, 500]), name='W1',dtype=tf.float32)
 b1 = tf.Variable(tf.zeros([500]), name='b1',dtype=tf.float32)
 W2 = tf.Variable(tf.ones([500, 300]), name='W2',dtype=tf.float32)
 b2 = tf.Variable(tf.zeros([300]), name='b2',dtype=tf.float32)
 W3 = tf.Variable(tf.ones([300, 1]), name='W3',dtype=tf.float32)
 b3 = tf.Variable(tf.zeros([1]), name='b3',dtype=tf.float32)
 hidden_out = tf.add(tf.matmul(X, W1), b1)
 hidden_out = bn_layer_top(hidden_out,'hidden_out_bn', is_training=is_training)
 hidden_out = tf.nn.relu(hidden_out)
 drop_out = tf.nn.dropout(hidden_out, keep_prob)
 hidden_out2 = tf.add(tf.matmul(drop_out, W2), b2)
 hidden_out2 = bn_layer_top(hidden_out2,'hidden_out_bn2', is_training=is_training)
 hidden_out2 = tf.nn.relu(hidden_out2)
 drop_out2 = tf.nn.dropout(hidden_out2, keep_prob)
 pred = tf.add(tf.matmul(drop_out2, W3), b3)
 deltas = tf.abs(pred - Y)
 loss = tf.reduce_sum(deltas)/batch_size
optimiser = tf.train.AdamOptimizer(learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 total_batch = int(len(X_trainRanked)/ batch_size)
 val_total_batch = int(len(X_valRanked)/ batch_size)
 for epoch in range(epochs):
avg_cost = 0
avg_acc = 0
avg_val_cost = 0
avg_val_acc = 0
for i in range(total_batch):
batch_x, batch_y = next_batch(num=batch_size,data=X_trainRanked,labels=y_trainRanked)
 _, c = sess.run([optimiser, loss],
feed_dict={X: batch_x, Y: batch_y, keep_prob : 0.80, is_training: True},)
 train_pred=sess.run(pred, feed_dict={X: batch_x,keep_prob : 1.0, is_training: False})
 c= rmse(batch_y,train_pred)
 a= r2_score(batch_y, train_pred,multioutput='variance_weighted')
 avg_cost += c / total_batch
avg_acc += a / total_batch
for i in range(val_total_batch):
val_batch_x, val_batch_y = next_batch(num=batch_size,data=X_valRanked,labels=y_valRanked)
 val_pred=sess.run(pred, feed_dict={X: val_batch_x,keep_prob : 1.0, is_training: False})
 val_c= rmse(val_batch_y,val_pred)
 val_a= r2_score(val_batch_y, val_pred,multioutput='variance_weighted')
 avg_val_cost += val_c / val_total_batch
avg_val_acc += val_a / val_total_batch
epoch=epoch + 1
lossArr = np.append(lossArr, [avg_cost])
 accArr= np.append(accArr, [avg_acc])
 val_lossArr = np.append(val_lossArr, [avg_val_cost])
 val_accArr= np.append(val_accArr, [avg_val_acc])
 epochArr= np.append(epochArr, [epoch])
 pred_500X300_kp90['SalePrice'] = sess.run(pred, feed_dict={X: train_df_munged,keep_prob : 1.0, is_training: False})
 lcurves() 
<prepare_x_and_y>
ids = [int(x.split('.')[0])for x in test_image_list]
ids[:20]
ntrain = train_df_munged.shape[0]
y_trainRanked= label_df.values
y_trainRanked=np.reshape(y_trainRanked,(ntrain, 1))
X_testRanked= test_df_munged.values
test1X=X_testRanked
X_trainRanked= train_df_munged.values
yShape=y_trainRanked.shape
XShape=X_trainRanked.shape
print(XShape)
 print(y_trainRanked.shape)
 print(y_trainRanked[2])
 def SaveResult2CSV(pred4Kaggle, epochNo):
submission=pd.DataFrame()
submission['Id']= test2.Id
pred_TestKaggle = pred4Kaggle
final_predictions= np.exp(pred_TestKaggle)
 submission['SalePrice']= final_predictions
q1 = submission['SalePrice'].quantile(0.0042)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.head()
create_download_link(submission)
 filename = "DNNKagglePredsMay{}.csv".format(epochNo)
 submission.to_csv(filename,index=False)
 print('Saved file: ' + filename)
 learning_rate = 0.0001
epochs = 2
batch_size = 80
tf.reset_default_graph()
X = tf.placeholder(dtype=tf.float32, shape = [None, 403])
 Y = tf.placeholder(dtype=tf.float32, shape = [None, 1])
 keep_prob = tf.placeholder(tf.float32)
 is_training = tf.placeholder(tf.bool)
 n_samples = y_trainRanked.shape[0]
W1 = tf.Variable(tf.ones([403, 500]), name='W1',dtype=tf.float32)
 b1 = tf.Variable(tf.zeros([500]), name='b1',dtype=tf.float32)
 W2 = tf.Variable(tf.ones([500, 300]), name='W2',dtype=tf.float32)
 b2 = tf.Variable(tf.zeros([300]), name='b2',dtype=tf.float32)
 W3 = tf.Variable(tf.ones([300, 1]), name='W3',dtype=tf.float32)
 b3 = tf.Variable(tf.zeros([1]), name='b3',dtype=tf.float32)
 hidden_out = tf.add(tf.matmul(X, W1), b1)
 hidden_out = bn_layer_top(hidden_out,'hidden_out_bn', is_training=is_training)
 hidden_out = tf.nn.relu(hidden_out)
 drop_out = tf.nn.dropout(hidden_out, keep_prob)
 hidden_out2 = tf.add(tf.matmul(drop_out, W2), b2)
 hidden_out2 = bn_layer_top(hidden_out2,'hidden_out_bn2', is_training=is_training)
 hidden_out2 = tf.nn.relu(hidden_out2)
 drop_out2 = tf.nn.dropout(hidden_out2, keep_prob)
 pred = tf.add(tf.matmul(drop_out2, W3), b3)
 deltas = tf.abs(pred - Y)
 loss = tf.reduce_sum(deltas)/batch_size
cost = loss
optimiser = tf.train.AdamOptimizer(learning_rate ).minimize(loss)
 init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
 total_batch = int(len(X_trainRanked)/ batch_size)
 for epoch in range(epochs):
avg_cost = 0
for i in range(total_batch):
batch_x, batch_y = next_batch(num=batch_size,data=X_trainRanked,labels=y_trainRanked)
 _, c = sess.run([optimiser, loss],
feed_dict={X: batch_x, Y: batch_y, keep_prob : 0.80, is_training: True},)
 avg_cost += c / total_batch
print("Epoch:",(epoch + 1), "cost =", "{:.3f}".format(avg_cost))
if(epoch==11300 or epoch==14400 or epoch==16500 or epoch==18800):
pred_TestKaggle = sess.run(pred, feed_dict={X: test1X,keep_prob : 1.0, is_training: False})
 SaveResult2CSV(pred_TestKaggle,epoch)
 submission=pd.DataFrame()
submission['Id']= test2.Id
pred_TestKaggle = sess.run(pred, feed_dict={X: test1X,keep_prob : 1.0, is_training: False})
 final_predictions= np.exp(pred_TestKaggle)
 submission['SalePrice']= final_predictions
q1 = submission['SalePrice'].quantile(0.0042)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.head()
create_download_link(submission)
 filename = 'DNNRegPredictionsMaykp80.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename)
<set_options>
target = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/target.npy' )
warnings.filterwarnings(category=FutureWarning, action="ignore")
 %matplotlib inline
backend.set_image_data_format('channels_last')
 DATA_PATH = '.. /input/'
SERIES = 'A'
VERSION = 1
print('{:s}{:d}'.format(SERIES, VERSION)) 
<load_from_csv>
train_InceptionResNetV2 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/train_InceptionResNetV2.npy')
 test_InceptionResNetV2 = np.load('.. /input/cats-and-dogs-embedded-data/cats_and_dogs_1/test_InceptionResNetV2.npy' )
train_data = pd.read_csv(DATA_PATH+'train.csv')
 train_data.head()<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(train_InceptionResNetV2, target, test_size=0.1, random_state=42 )
test_data = pd.read_csv(DATA_PATH+'test.csv')
 test_data.index =([x+1 for x in range(test_data.shape[0])])
 print(test_data.shape)
 test_data.head() 
<categorify>
xgb_params = {
'eta': 0.05,
'max_depth': 7,
'subsample': 0.85,
'colsample_bytree': 0.7,
'objective': 'reg:logistic',
'eval_metric': 'auc',
'verbosity': 0,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor'
}
dtrain = xgb.DMatrix(X_train, y_train)
 dval = xgb.DMatrix(X_val, y_val)
 dtest = xgb.DMatrix(test_InceptionResNetV2 )
def get_model_params(layers)-> str:
res = {}
for layer in layers:
lres = {}
config = layer.get_config()
for key in ['filters', 'kernel_size', 'activation', 'pool_size',
'padding', 'strides', 'rate', 'units', 'kernel_regularizer',
'batch_input_shape']:
if key in config.keys() :
lres[key] = config[key]
res[layer.get_config() ['name']] = lres
rep = ''
for elem in res:
rep += elem+': '
rep += str(res[elem])+'
'
return rep 
<prepare_x_and_y>
del train_InceptionResNetV2, test_InceptionResNetV2, X_train, X_val
gc.collect()
x_train = train_data.iloc[:, 1:].values.reshape(
(train_data.shape[0], 28, 28, 1)).astype('float32')
 x_train = x_train / 255.0
x_test = test_data.values.reshape(
(test_data.shape[0], 28, 28, 1)).astype('float32')
 x_test = x_test / 255.0
lb = LabelBinarizer()
y_train_ = lb.fit_transform(train_data.iloc[:, 0])
<train_model>
%%time
model = xgb.train(xgb_params, dtrain, num_boost_round=100 )
train_gen = ImageDataGenerator(
 rotation_range=9,
zoom_range=0.09,
width_shift_range=0.09,
height_shift_range=0.11,
validation_split=0.05 
)
 train_gen.fit(x_train)
 train_iterator = train_gen.flow(
 x=x_train, y=y_train_, batch_size=256, subset='training')
 val_iterator = train_gen.flow(
 x=x_train, y=y_train_, batch_size=256, subset='validation')
 train_x, train_y = train_iterator.next() 
<choose_model_class>
val_preds = model.predict(dval )
model = Sequential([
Conv2D(filters=128, kernel_size=(3, 3),
activation='relu', input_shape=(28, 28, 1)) ,
MaxPooling2D(pool_size=(2, 2)) ,
Conv2D(filters=256, kernel_size=(3, 3), activation='relu'),
MaxPooling2D(pool_size=(2, 2)) ,
Conv2D(filters=512, kernel_size=(4, 4), activation='relu'),
MaxPooling2D(pool_size=(2, 2)) ,
Flatten() ,
Dense(units=256, activation='relu'),
Dropout(rate=0.35),
Dense(units=128, activation='relu'),
Dense(units=64, activation='relu'),
Dense(10, activation='softmax')
 ])
 print(model.summary())
 plot_model(model, show_shapes=True, show_layer_names=True,)
<choose_model_class>
roc_auc_score(y_val, val_preds )
NEPOCHS = 300
early_stopping_cb = EarlyStopping(monitor='val_acc', min_delta=1e-5,
patience=15, restore_best_weights=True)
 rl_reduce = ReduceLROnPlateau(monitor='val_loss', patience=10,factor=0.25,verbose=1,min_delta=1e-5)
 opt_rms = RMSprop(learning_rate=1e-3, centered=False)
 model.compile(loss='categorical_crossentropy', optimizer=opt_rms,
metrics=['accuracy'])
 start = dt.datetime.now()
history = model.fit_generator(
 generator=train_iterator,
verbose=1,
epochs=NEPOCHS,
max_queue_size=10,
validation_data=val_iterator,
callbacks=[
early_stopping_cb,
rl_reduce
] 
)
 print('
Evaluation :')
 model.evaluate_generator(generator=val_iterator,
verbose=1, 
)
 print('Finished in: ', dt.datetime.now() -start)
<save_to_csv>
log_loss(y_val, val_preds )
test_gen = ImageDataGenerator(
 rotation_range=9,
zoom_range=0.09,
width_shift_range=0.09,
height_shift_range=0.11, 
)
 test_gen.fit(x_test)
 test_iterator = test_gen.flow(x=x_test, batch_size=len(x_test), shuffle=False)
 test_x = test_iterator.next()
test_x
res = model.predict(test_x)
 y_pred = pd.DataFrame([test_data.index, [x.argmax() for x in res]] ).T
y_pred.columns = ['ImageId', 'Label']
y_pred.to_csv('digit_submission_2.csv', index=False)
<set_options>
%%time
val_shap_preds = model.predict(dval, pred_contribs=True )
%matplotlib inline<define_variables>
%%time
test_shap_preds = model.predict(dtest, pred_contribs=True)
path = Path('.. /input/')
 !ls.. /input<load_from_csv>
%%time
train_shap_preds = model.predict(dtrain, pred_contribs=True )
class CustomImageItemList(ImageList):
def open(self, fn):
img = fn.reshape(28, 28)
 img = np.stack(( img,)*3, axis=-1)
 return Image(pil2tensor(img, dtype=np.float32))
@classmethod
def from_csv_custom(cls, path:PathOrStr, csv_name:str, imgIdx:int=1, header:str='infer', **kwargs)-> 'ItemList':
df = pd.read_csv(Path(path)/csv_name, header=header)
 res = super().from_df(df, path=path, cols=0, **kwargs)
 res.items = df.iloc[:,imgIdx:].apply(lambda x: x.values / 783.0, axis=1 ).values
return res<load_from_csv>
test_preds = model.predict(dtest )
<define_variables><EOS> 
sample_submission['id'] = ids
sample_submission['label'] = test_preds
sample_submission.to_csv('submission.csv', index=False)
 sample_submission.head()