Kaggle competition: SIIM-ISIC Melanoma Classification
<SOS> metric: MCAUC Kaggle data source: siim-isic-melanoma-classification<define_variables>
DEVICE = "TPU"
CFG = dict(
 net_count = 7,
batch_size = 32,
read_size = 512,
crop_size = 512,
net_size = 512,
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 7,
LR_SUSTAIN_EPOCHS = 0,
LR_EXP_DECAY = 0.8,
epochs = 25,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)
train_data = train_images
train_targets = train_labels
k = 4
num_val_samples = len(train_data)// k
all_mae_histories = []
for i in range(k):
print('processing fold
val_data = train_data[i * num_val_samples:(i + 1)* num_val_samples]
val_targets = train_targets[i * num_val_samples:(i + 1)* num_val_samples]
partial_train_data = np.concatenate(
 [train_data[:i * num_val_samples],
train_data[(i + 1)* num_val_samples:]],
axis=0)
 partial_train_targets = np.concatenate(
 [train_targets[:i * num_val_samples],
train_targets[(i + 1)* num_val_samples:]],
axis=0)
 model = build_model()
history = model.fit(partial_train_data, partial_train_targets,
validation_data=(val_data, val_targets),
epochs=num_epochs, batch_size=BatchSize, verbose=0)
 mae_history = history.history['acc']
all_mae_histories.append(mae_history)
 print("Done CV k-fold" )<concatenate>
!pip install -q efficientnet
train_imagesFin = np.concatenate(( train_images,test_imagesKeras), axis=0)
 print("train_imagesFin ", train_imagesFin.shape)
 print("_"*50)
 train_labelsFin = np.concatenate(( train_labels,test_labelsKeras), axis=0)
 print("train_labelsFin ", train_labelsFin.shape )<train_model>
random.seed(a=42)
model = build_model()
model.fit(train_imagesFin, train_labelsFin, epochs=num_epochs, batch_size=BatchSize )<predict_on_test>
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 GCS_PATH_2019 = KaggleDatasets().get_gcs_path('isic2019-512x512')
 files_train_2019 = np.array(tf.io.gfile.glob(GCS_PATH_2019 + '/train*.tfrec'))
files_train_2020 = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
files_train = np.sort(np.concatenate(( files_train_2020, files_train_2019)))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )
RawPred = model.predict(test_imagesKaggle)
 pred = []
numTest = RawPred.shape[0]
for i in range(numTest):
pred.append(np.argmax(RawPred[i]))
predictions = np.array(pred )<save_to_csv>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 result=pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':predictions})
 result.to_csv("submission.csv",index=False)
 print(result )<load_from_csv>
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )
training_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 training_data.head()<prepare_x_and_y>
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']), example['target'])
 def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']),(example['image_name'] if return_image_name else 0))
def prepare_data(data, cfg=None, augment=True):
img = data[0]
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 sex_oh = tf.one_hot(data[1][0], 2)
 age_aprox = tf.dtypes.cast(tf.reshape(data[1][1], [1]), tf.float32)
 anatom_site_general_challenge = tf.one_hot(data[1][2], 7)
 dense = tf.concat([sex_oh, age_aprox, anatom_site_general_challenge], axis=0)
 return(img, dense)
 def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )
x_train = training_data.drop('label', axis = 1)
 y_train = pd.DataFrame(data=training_data['label'])
 display(y_train.head())
 display(x_train.head() )<set_options>
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, imgname_or_label:(prepare_data(( img, dense), augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds
%matplotlib inline
i= 1
imshow(x_train.iloc[i].values.reshape(( 28, 28)))
 print('This image corresponds to ', y_train.iloc[i] )<choose_model_class>
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data[0][0], data[1]
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
cnn_model = Sequential()
cnn_model.add(Conv2D(128,(3,3), padding='same', input_shape=(28,28,1), data_format='channels_last', activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.2))
cnn_model.add(Conv2D(128,(3,3), padding='same', activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.2))
cnn_model.add(Conv2D(256,(3,3), padding='same', activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.2))
cnn_model.add(Conv2D(256,(3,3), padding='valid', activation='relu'))
cnn_model.add(Dropout(0.2))
cnn_model.add(Flatten())
 cnn_model.add(Dense(units=10, activation='softmax'))
cnn_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 print('The model was successfully created and compiled.')
 print('Shape of Layer Outputs:')
 for layer in cnn_model.layers:
print(layer.name,': ',layer.output_shape )<train_model>
len(df_train.diagnosis.unique() )
y_train_categorical = to_categorical(y_train, num_classes=10)
 reshaped_x = x_train.values.reshape(x_train.shape[0],28,28,1)/ 255
print(reshaped_x.shape)
 print(y_train_categorical.shape)
 cnn_model.fit(x=reshaped_x, y=y_train_categorical, batch_size=1000, epochs=32, verbose=1, validation_split=0.2 )<predict_on_test>
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, target:(prepare_data(( img, dense), cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )
reshaped_test_data = test_data.values.reshape(test_data.shape[0],28,28,1)/ 255
predictions = cnn_model.predict(reshaped_test_data)
 display(predictions )<prepare_output>
ds = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )
predictions_formatted = np.argmax(predictions, axis=1)
 display(predictions_formatted )<save_to_csv>
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback
submission = pd.DataFrame({'ImageId': np.arange(1,28001), 'Label': predictions_formatted})
 submission.to_csv('submission_4.csv', index=False)
 print('Done' )<set_options>
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 model_input_data = tf.keras.Input(shape=(10,), name='dataIn')
 dense = tf.keras.layers.Dense(256, activation='relu' )(model_input_data)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 dense = tf.keras.layers.Dense(64, activation='relu' )(dense)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 constructor = getattr(efn, f'EfficientNetB6')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(model_input)
 x = tf.keras.layers.Concatenate()([x, dense])
 x = tf.keras.layers.Dense(256, activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.5 )(x)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 model = tf.keras.Model([model_input, model_input_data], x, name='aNetwork')
 model.summary()
return model
%matplotlib inline
print(os.path.dirname(os.getcwd())+':', os.listdir(os.path.dirname(os.getcwd())));
print(os.getcwd() +':', os.listdir(os.getcwd())) ;<load_from_csv>
def compile_new_model(cfg, old_model=None):
with strategy.scope() :
if(old_model is None):
model = get_model(cfg)
 else:
model = old_model
model.compile(
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.1),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac']),
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model
if os.path.isfile('.. /input/train.csv'):
data_df = pd.read_csv('.. /input/train.csv')
 print('train.csv loaded: data_df({0[0]},{0[1]})'.format(data_df.shape))
elif os.path.isfile('data/train.csv'):
data_df = pd.read_csv('data/train.csv')
 print('train.csv loaded: data_df({0[0]},{0[1]})'.format(data_df.shape))
else:
print('Error: train.csv not found')
 print('')
 print(data_df.isnull().any().describe())
 print('')
 print('distinct labels ', data_df['label'].unique())
 print('')
 print(data_df['label'].value_counts() )<categorify>
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 class_weights = class_weight.compute_class_weight('balanced',
np.unique(train.target),
train.target)
 class_weights = dict(enumerate(class_weights))
def normalize_data(data):
data = data / data.max()
return data
def dense_to_one_hot(labels_dense, num_classes):
num_labels = labels_dense.shape[0]
index_offset = np.arange(num_labels)* num_classes
labels_one_hot = np.zeros(( num_labels, num_classes))
labels_one_hot.flat[index_offset + labels_dense.ravel() ] = 1
return labels_one_hot
def one_hot_to_dense(labels_one_hot):
return np.argmax(labels_one_hot,1)
 def accuracy_from_dense_labels(y_target, y_pred):
y_target = y_target.reshape(-1,)
 y_pred = y_pred.reshape(-1,)
 return np.mean(y_target == y_pred)
 def accuracy_from_one_hot_labels(y_target, y_pred):
y_target = one_hot_to_dense(y_target ).reshape(-1,)
 y_pred = one_hot_to_dense(y_pred ).reshape(-1,)
 return np.mean(y_target == y_pred)
 x_train_valid = data_df.iloc[:,1:].values.reshape(-1,28,28,1)
 x_train_valid = x_train_valid.astype(np.float)
 x_train_valid = normalize_data(x_train_valid)
 image_width = image_height = 28
image_size = 784
y_train_valid_labels = data_df.iloc[:,0].values
labels_count = np.unique(y_train_valid_labels ).shape[0];
plt.figure(figsize=(15,9))
for i in range(50):
plt.subplot(5,10,1+i)
 plt.title(y_train_valid_labels[i])
 plt.imshow(x_train_valid[i].reshape(28,28), cmap=cm.binary)
 y_train_valid = dense_to_one_hot(y_train_valid_labels, labels_count ).astype(np.uint8)
 y_valid_pred = {}
y_train_pred = {}
y_test_pred = {}
train_loss, valid_loss = {}, {}
train_acc, valid_acc = {}, {}
print('x_train_valid.shape = ', x_train_valid.shape)
 print('y_train_valid_labels.shape = ', y_train_valid_labels.shape)
 print('image_size = ', image_size)
 print('image_width = ', image_width)
 print('image_height = ', image_height)
 print('labels_count = ', labels_count)
<define_variables>
file_path="/kaggle/working/effnet4_weights.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='auc', verbose=1, save_best_only=True, mode='max' )
def generate_images(imgs):
image_generator = keras.preprocessing.image.ImageDataGenerator(
 rotation_range = 10, width_shift_range = 0.1 , height_shift_range = 0.1,
zoom_range = 0.1)
 imgs = image_generator.flow(imgs.copy() , np.zeros(len(imgs)) ,
batch_size=len(imgs), shuffle = False ).next()
return imgs[0]
fig,axs = plt.subplots(5,10, figsize=(15,9))
for i in range(5):
n = np.random.randint(0,x_train_valid.shape[0]-2)
 axs[i,0].imshow(x_train_valid[n:n+1].reshape(28,28),cmap=cm.binary)
 axs[i,1].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,2].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,3].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,4].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,5].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,6].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,7].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,8].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary)
 axs[i,9].imshow(generate_images(x_train_valid[n:n+1] ).reshape(28,28), cmap=cm.binary )<choose_model_class>
kf = KFold(5, shuffle=True)
 train_index, test_index = next(kf.split(files_train))
train = files_train[train_index]
ds_train = get_dataset(files_train[train_index], CFG, augment=True, shuffle=True, repeat=True)
 ds_val = get_dataset(files_train[test_index], CFG, augment=True, shuffle=True, repeat=False)
 steps_train = count_data_items(files_train[train_index])/(CFG['batch_size'] * REPLICAS )
logreg = sklearn.linear_model.LogisticRegression(verbose=0, solver='lbfgs',
multi_class='multinomial')
 decision_tree = sklearn.tree.DecisionTreeClassifier()
extra_trees = sklearn.ensemble.ExtraTreesClassifier(verbose=0)
 gradient_boost = sklearn.ensemble.GradientBoostingClassifier(verbose=0)
 random_forest = sklearn.ensemble.RandomForestClassifier(verbose=0)
 gaussianNB = sklearn.naive_bayes.GaussianNB()
base_models = {'logreg': logreg, 'extra_trees': extra_trees,
'gradient_boost': gradient_boost, 'random_forest': random_forest,
'decision_tree': decision_tree, 'gaussianNB': gaussianNB}
take_models = ['logreg','random_forest','extra_trees']
for mn in take_models:
train_acc[mn] = []
valid_acc[mn] = []
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True, random_state=123)
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
start = datetime.datetime.now() ;
x_train = x_train_valid[train_index].reshape(-1,784)
 y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index].reshape(-1,784)
 y_valid = y_train_valid[valid_index]
for mn in take_models:
model = sklearn.base.clone(base_models[mn])
 model.fit(x_train, one_hot_to_dense(y_train))
y_train_pred[mn] = model.predict_proba(x_train)
 y_valid_pred[mn] = model.predict_proba(x_valid)
 train_acc[mn].append(accuracy_from_one_hot_labels(y_train_pred[mn], y_train))
valid_acc[mn].append(accuracy_from_one_hot_labels(y_valid_pred[mn], y_valid))
print(i,': '+mn+' train/valid accuracy = %.3f/%.3f'%(train_acc[mn][-1],
valid_acc[mn][-1]))
if False:
break;
print(mn+': averaged train/valid accuracy = %.3f/%.3f'%(np.mean(train_acc[mn]),
np.mean(valid_acc[mn])))
<init_hyperparams>
print("Buidling model...")
 model = compile_new_model(CFG)
 model.load_weights('.. /input/effnet-pretrained-25-epochs/effnet4_weights.best.hdf5' )
class nn_class:
def __init__(self, nn_name = 'nn_1'):
self.s_f_conv1 = 3;
self.n_f_conv1 = 36;
self.s_f_conv2 = 3;
self.n_f_conv2 = 36;
self.s_f_conv3 = 3;
self.n_f_conv3 = 36;
self.n_n_fc1 = 576;
self.mb_size = 50
self.keep_prob = 0.33
self.learn_rate_array = [10*1e-4, 7.5*1e-4, 5*1e-4, 2.5*1e-4, 1*1e-4, 1*1e-4,
1*1e-4,0.75*1e-4, 0.5*1e-4, 0.25*1e-4, 0.1*1e-4,
0.1*1e-4, 0.075*1e-4,0.050*1e-4, 0.025*1e-4, 0.01*1e-4,
0.0075*1e-4, 0.0050*1e-4,0.0025*1e-4,0.001*1e-4]
self.learn_rate_step_size = 3
self.learn_rate = self.learn_rate_array[0]
self.learn_rate_pos = 0
self.index_in_epoch = 0
self.current_epoch = 0
self.log_step = 0.2
self.n_log_step = 0
self.use_tb_summary = False
self.use_tf_saver = False
self.nn_name = nn_name
self.perm_array = np.array([])
 def next_mini_batch(self):
start = self.index_in_epoch
self.index_in_epoch += self.mb_size
self.current_epoch += self.mb_size/len(self.x_train)
 if not len(self.perm_array)== len(self.x_train):
self.perm_array = np.arange(len(self.x_train))
if start == 0:
np.random.shuffle(self.perm_array)
 if self.index_in_epoch > self.x_train.shape[0]:
np.random.shuffle(self.perm_array)
 start = 0
self.index_in_epoch = self.mb_size
if self.train_on_augmented_data:
self.x_train_aug = normalize_data(self.generate_images(self.x_train))
self.y_train_aug = self.y_train
end = self.index_in_epoch
if self.train_on_augmented_data:
x_tr = self.x_train_aug[self.perm_array[start:end]]
y_tr = self.y_train_aug[self.perm_array[start:end]]
else:
x_tr = self.x_train[self.perm_array[start:end]]
y_tr = self.y_train[self.perm_array[start:end]]
return x_tr, y_tr
def generate_images(self, imgs):
print('generate new set of images')
 image_generator = keras.preprocessing.image.ImageDataGenerator(
 rotation_range = 10, width_shift_range = 0.1 , height_shift_range = 0.1,
zoom_range = 0.1)
 imgs = image_generator.flow(imgs.copy() , np.zeros(len(imgs)) ,
batch_size=len(imgs), shuffle = False ).next()
return imgs[0]
def weight_variable(self, shape, name = None):
initial = tf.truncated_normal(shape, stddev=0.1)
 return tf.Variable(initial, name = name)
 def bias_variable(self, shape, name = None):
initial = tf.constant(0.1, shape=shape)
 return tf.Variable(initial, name = name)
 def conv2d(self, x, W, name = None):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME', name = name)
 def max_pool_2x2(self, x, name = None):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding='SAME', name = name)
 def summary_variable(self, var, var_name):
with tf.name_scope(var_name):
mean = tf.reduce_mean(var)
 stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
 tf.summary.scalar('mean', mean)
 tf.summary.scalar('stddev', stddev)
 tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
 def create_graph(self):
tf.reset_default_graph()
self.x_data_tf = tf.placeholder(dtype=tf.float32, shape=[None,28,28,1],
name='x_data_tf')
 self.y_data_tf = tf.placeholder(dtype=tf.float32, shape=[None,10], name='y_data_tf')
 self.W_conv1_tf = self.weight_variable([self.s_f_conv1, self.s_f_conv1, 1,
self.n_f_conv1],
name = 'W_conv1_tf')
 self.b_conv1_tf = self.bias_variable([self.n_f_conv1], name = 'b_conv1_tf')
 self.h_conv1_tf = tf.nn.relu(self.conv2d(self.x_data_tf,
self.W_conv1_tf)+ self.b_conv1_tf,
name = 'h_conv1_tf')
 self.h_pool1_tf = self.max_pool_2x2(self.h_conv1_tf,
name = 'h_pool1_tf')
 self.W_conv2_tf = self.weight_variable([self.s_f_conv2, self.s_f_conv2,
self.n_f_conv1, self.n_f_conv2],
name = 'W_conv2_tf')
 self.b_conv2_tf = self.bias_variable([self.n_f_conv2], name = 'b_conv2_tf')
 self.h_conv2_tf = tf.nn.relu(self.conv2d(self.h_pool1_tf,
self.W_conv2_tf)+ self.b_conv2_tf,
name ='h_conv2_tf')
 self.h_pool2_tf = self.max_pool_2x2(self.h_conv2_tf, name = 'h_pool2_tf')
 self.W_conv3_tf = self.weight_variable([self.s_f_conv3, self.s_f_conv3,
self.n_f_conv2, self.n_f_conv3],
name = 'W_conv3_tf')
 self.b_conv3_tf = self.bias_variable([self.n_f_conv3], name = 'b_conv3_tf')
 self.h_conv3_tf = tf.nn.relu(self.conv2d(self.h_pool2_tf,
self.W_conv3_tf)+ self.b_conv3_tf,
name = 'h_conv3_tf')
 self.h_pool3_tf = self.max_pool_2x2(self.h_conv3_tf,
name = 'h_pool3_tf')
 self.W_fc1_tf = self.weight_variable([4*4*self.n_f_conv3,self.n_n_fc1],
name = 'W_fc1_tf')
 self.b_fc1_tf = self.bias_variable([self.n_n_fc1], name = 'b_fc1_tf')
 self.h_pool3_flat_tf = tf.reshape(self.h_pool3_tf, [-1,4*4*self.n_f_conv3],
name = 'h_pool3_flat_tf')
 self.h_fc1_tf = tf.nn.relu(tf.matmul(self.h_pool3_flat_tf,
self.W_fc1_tf)+ self.b_fc1_tf,
name = 'h_fc1_tf')
 self.keep_prob_tf = tf.placeholder(dtype=tf.float32, name = 'keep_prob_tf')
 self.h_fc1_drop_tf = tf.nn.dropout(self.h_fc1_tf, self.keep_prob_tf,
name = 'h_fc1_drop_tf')
 self.W_fc2_tf = self.weight_variable([self.n_n_fc1, 10], name = 'W_fc2_tf')
 self.b_fc2_tf = self.bias_variable([10], name = 'b_fc2_tf')
 self.z_pred_tf = tf.add(tf.matmul(self.h_fc1_drop_tf, self.W_fc2_tf),
self.b_fc2_tf, name = 'z_pred_tf')
 self.cross_entropy_tf = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
 labels=self.y_data_tf, logits=self.z_pred_tf), name = 'cross_entropy_tf')
 self.learn_rate_tf = tf.placeholder(dtype=tf.float32, name="learn_rate_tf")
 self.train_step_tf = tf.train.AdamOptimizer(self.learn_rate_tf ).minimize(
 self.cross_entropy_tf, name = 'train_step_tf')
 self.y_pred_proba_tf = tf.nn.softmax(self.z_pred_tf, name='y_pred_proba_tf')
 self.y_pred_correct_tf = tf.equal(tf.argmax(self.y_pred_proba_tf, 1),
tf.argmax(self.y_data_tf, 1),
name = 'y_pred_correct_tf')
 self.accuracy_tf = tf.reduce_mean(tf.cast(self.y_pred_correct_tf, dtype=tf.float32),
name = 'accuracy_tf')
 self.train_loss_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='train_loss_tf', validate_shape = False)
 self.valid_loss_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='valid_loss_tf', validate_shape = False)
 self.train_acc_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='train_acc_tf', validate_shape = False)
 self.valid_acc_tf = tf.Variable(np.array([]), dtype=tf.float32,
name='valid_acc_tf', validate_shape = False)
 num_weights =(self.s_f_conv1**2*self.n_f_conv1
+ self.s_f_conv2**2*self.n_f_conv1*self.n_f_conv2
+ self.s_f_conv3**2*self.n_f_conv2*self.n_f_conv3
+ 4*4*self.n_f_conv3*self.n_n_fc1 + self.n_n_fc1*10)
 num_biases = self.n_f_conv1 + self.n_f_conv2 + self.n_f_conv3 + self.n_n_fc1
print('num_weights =', num_weights)
 print('num_biases =', num_biases)
 return None
def attach_summary(self, sess):
self.use_tb_summary = True
self.summary_variable(self.W_conv1_tf, 'W_conv1_tf')
 self.summary_variable(self.b_conv1_tf, 'b_conv1_tf')
 self.summary_variable(self.W_conv2_tf, 'W_conv2_tf')
 self.summary_variable(self.b_conv2_tf, 'b_conv2_tf')
 self.summary_variable(self.W_conv3_tf, 'W_conv3_tf')
 self.summary_variable(self.b_conv3_tf, 'b_conv3_tf')
 self.summary_variable(self.W_fc1_tf, 'W_fc1_tf')
 self.summary_variable(self.b_fc1_tf, 'b_fc1_tf')
 self.summary_variable(self.W_fc2_tf, 'W_fc2_tf')
 self.summary_variable(self.b_fc2_tf, 'b_fc2_tf')
 tf.summary.scalar('cross_entropy_tf', self.cross_entropy_tf)
 tf.summary.scalar('accuracy_tf', self.accuracy_tf)
 self.merged = tf.summary.merge_all()
timestamp = datetime.datetime.now().strftime('%d-%m-%Y_%H-%M-%S')
 filepath = os.path.join(os.getcwd() , 'logs',(self.nn_name+'_'+timestamp))
self.train_writer = tf.summary.FileWriter(os.path.join(filepath,'train'), sess.graph)
 self.valid_writer = tf.summary.FileWriter(os.path.join(filepath,'valid'), sess.graph)
 def attach_saver(self):
self.use_tf_saver = True
self.saver_tf = tf.train.Saver()
def train_graph(self, sess, x_train, y_train, x_valid, y_valid, n_epoch = 1,
train_on_augmented_data = False):
self.train_on_augmented_data = train_on_augmented_data
self.x_train = x_train
self.y_train = y_train
self.x_valid = x_valid
self.y_valid = y_valid
if self.train_on_augmented_data:
print('generate new set of images')
 self.x_train_aug = normalize_data(self.generate_images(self.x_train))
self.y_train_aug = self.y_train
mb_per_epoch = self.x_train.shape[0]/self.mb_size
train_loss, train_acc, valid_loss, valid_acc = [],[],[],[]
start = datetime.datetime.now() ;
print(datetime.datetime.now().strftime('%d-%m-%Y %H:%M:%S'),': start training')
 print('learnrate = ',self.learn_rate,', n_epoch = ', n_epoch,
', mb_size = ', self.mb_size)
 for i in range(int(n_epoch*mb_per_epoch)+1):
self.learn_rate_pos = int(self.current_epoch // self.learn_rate_step_size)
 if not self.learn_rate == self.learn_rate_array[self.learn_rate_pos]:
self.learn_rate = self.learn_rate_array[self.learn_rate_pos]
print(datetime.datetime.now() -start,': set learn rate to %.6f'%self.learn_rate)
 x_batch, y_batch = self.next_mini_batch()
sess.run(self.train_step_tf, feed_dict={self.x_data_tf: x_batch,
self.y_data_tf: y_batch,
self.keep_prob_tf: self.keep_prob,
self.learn_rate_tf: self.learn_rate})
 if i%int(self.log_step*mb_per_epoch)== 0 or i == int(n_epoch*mb_per_epoch):
self.n_log_step += 1
feed_dict_train = {
self.x_data_tf: self.x_train[self.perm_array[:len(self.x_valid)]],
self.y_data_tf: self.y_train[self.perm_array[:len(self.y_valid)]],
self.keep_prob_tf: 1.0}
feed_dict_valid = {self.x_data_tf: self.x_valid,
self.y_data_tf: self.y_valid,
self.keep_prob_tf: 1.0}
if self.use_tb_summary:
train_summary = sess.run(self.merged, feed_dict = feed_dict_train)
 valid_summary = sess.run(self.merged, feed_dict = feed_dict_valid)
 self.train_writer.add_summary(train_summary, self.n_log_step)
 self.valid_writer.add_summary(valid_summary, self.n_log_step)
 train_loss.append(sess.run(self.cross_entropy_tf,
feed_dict = feed_dict_train))
train_acc.append(self.accuracy_tf.eval(session = sess,
feed_dict = feed_dict_train))
valid_loss.append(sess.run(self.cross_entropy_tf,
feed_dict = feed_dict_valid))
valid_acc.append(self.accuracy_tf.eval(session = sess,
feed_dict = feed_dict_valid))
print('%.2f epoch: train/val loss = %.4f/%.4f, train/val acc = %.4f/%.4f'%(
 self.current_epoch, train_loss[-1], valid_loss[-1],
train_acc[-1], valid_acc[-1]))
tl_c = np.concatenate([self.train_loss_tf.eval(session=sess), train_loss], axis = 0)
 vl_c = np.concatenate([self.valid_loss_tf.eval(session=sess), valid_loss], axis = 0)
 ta_c = np.concatenate([self.train_acc_tf.eval(session=sess), train_acc], axis = 0)
 va_c = np.concatenate([self.valid_acc_tf.eval(session=sess), valid_acc], axis = 0)
 sess.run(tf.assign(self.train_loss_tf, tl_c, validate_shape = False))
sess.run(tf.assign(self.valid_loss_tf, vl_c , validate_shape = False))
sess.run(tf.assign(self.train_acc_tf, ta_c , validate_shape = False))
sess.run(tf.assign(self.valid_acc_tf, va_c , validate_shape = False))
print('running time for training: ', datetime.datetime.now() - start)
 return None
def save_model(self, sess):
if self.use_tf_saver:
filepath = os.path.join(os.getcwd() , self.nn_name)
 self.saver_tf.save(sess, filepath)
 if self.use_tb_summary:
self.train_writer.close()
self.valid_writer.close()
return None
def forward(self, sess, x_data):
y_pred_proba = self.y_pred_proba_tf.eval(session = sess,
feed_dict = {self.x_data_tf: x_data,
self.keep_prob_tf: 1.0})
 return y_pred_proba
def load_tensors(self, graph):
self.x_data_tf = graph.get_tensor_by_name("x_data_tf:0")
 self.y_data_tf = graph.get_tensor_by_name("y_data_tf:0")
 self.W_conv1_tf = graph.get_tensor_by_name("W_conv1_tf:0")
 self.W_conv2_tf = graph.get_tensor_by_name("W_conv2_tf:0")
 self.W_conv3_tf = graph.get_tensor_by_name("W_conv3_tf:0")
 self.W_fc1_tf = graph.get_tensor_by_name("W_fc1_tf:0")
 self.W_fc2_tf = graph.get_tensor_by_name("W_fc2_tf:0")
 self.b_conv1_tf = graph.get_tensor_by_name("b_conv1_tf:0")
 self.b_conv2_tf = graph.get_tensor_by_name("b_conv2_tf:0")
 self.b_conv3_tf = graph.get_tensor_by_name("b_conv3_tf:0")
 self.b_fc1_tf = graph.get_tensor_by_name("b_fc1_tf:0")
 self.b_fc2_tf = graph.get_tensor_by_name("b_fc2_tf:0")
 self.h_conv1_tf = graph.get_tensor_by_name('h_conv1_tf:0')
 self.h_pool1_tf = graph.get_tensor_by_name('h_pool1_tf:0')
 self.h_conv2_tf = graph.get_tensor_by_name('h_conv2_tf:0')
 self.h_pool2_tf = graph.get_tensor_by_name('h_pool2_tf:0')
 self.h_conv3_tf = graph.get_tensor_by_name('h_conv3_tf:0')
 self.h_pool3_tf = graph.get_tensor_by_name('h_pool3_tf:0')
 self.h_fc1_tf = graph.get_tensor_by_name('h_fc1_tf:0')
 self.z_pred_tf = graph.get_tensor_by_name('z_pred_tf:0')
 self.learn_rate_tf = graph.get_tensor_by_name("learn_rate_tf:0")
 self.keep_prob_tf = graph.get_tensor_by_name("keep_prob_tf:0")
 self.cross_entropy_tf = graph.get_tensor_by_name('cross_entropy_tf:0')
 self.train_step_tf = graph.get_operation_by_name('train_step_tf')
 self.z_pred_tf = graph.get_tensor_by_name('z_pred_tf:0')
 self.y_pred_proba_tf = graph.get_tensor_by_name("y_pred_proba_tf:0")
 self.y_pred_correct_tf = graph.get_tensor_by_name('y_pred_correct_tf:0')
 self.accuracy_tf = graph.get_tensor_by_name('accuracy_tf:0')
 self.train_loss_tf = graph.get_tensor_by_name("train_loss_tf:0")
 self.train_acc_tf = graph.get_tensor_by_name("train_acc_tf:0")
 self.valid_loss_tf = graph.get_tensor_by_name("valid_loss_tf:0")
 self.valid_acc_tf = graph.get_tensor_by_name("valid_acc_tf:0")
 return None
def get_loss(self, sess):
train_loss = self.train_loss_tf.eval(session = sess)
 valid_loss = self.valid_loss_tf.eval(session = sess)
 return train_loss, valid_loss
def get_accuracy(self, sess):
train_acc = self.train_acc_tf.eval(session = sess)
 valid_acc = self.valid_acc_tf.eval(session = sess)
 return train_acc, valid_acc
def get_weights(self, sess):
W_conv1 = self.W_conv1_tf.eval(session = sess)
 W_conv2 = self.W_conv2_tf.eval(session = sess)
 W_conv3 = self.W_conv3_tf.eval(session = sess)
 W_fc1_tf = self.W_fc1_tf.eval(session = sess)
 W_fc2_tf = self.W_fc2_tf.eval(session = sess)
 return W_conv1, W_conv2, W_conv3, W_fc1_tf, W_fc2_tf
def get_biases(self, sess):
b_conv1 = self.b_conv1_tf.eval(session = sess)
 b_conv2 = self.b_conv2_tf.eval(session = sess)
 b_conv3 = self.b_conv3_tf.eval(session = sess)
 b_fc1_tf = self.b_fc1_tf.eval(session = sess)
 b_fc2_tf = self.b_fc2_tf.eval(session = sess)
 return b_conv1, b_conv2, b_conv3, b_fc1_tf, b_fc2_tf
def load_session_from_file(self, filename):
tf.reset_default_graph()
filepath = os.path.join(os.getcwd() , filename + '.meta')
 saver = tf.train.import_meta_graph(filepath)
 print(filepath)
 sess = tf.Session()
saver.restore(sess, mn)
 graph = tf.get_default_graph()
self.load_tensors(graph)
 return sess
def get_activations(self, sess, x_data):
feed_dict = {self.x_data_tf: x_data, self.keep_prob_tf: 1.0}
h_conv1 = self.h_conv1_tf.eval(session = sess, feed_dict = feed_dict)
 h_pool1 = self.h_pool1_tf.eval(session = sess, feed_dict = feed_dict)
 h_conv2 = self.h_conv2_tf.eval(session = sess, feed_dict = feed_dict)
 h_pool2 = self.h_pool2_tf.eval(session = sess, feed_dict = feed_dict)
 h_conv3 = self.h_conv3_tf.eval(session = sess, feed_dict = feed_dict)
 h_pool3 = self.h_pool3_tf.eval(session = sess, feed_dict = feed_dict)
 h_fc1 = self.h_fc1_tf.eval(session = sess, feed_dict = feed_dict)
 h_fc2 = self.z_pred_tf.eval(session = sess, feed_dict = feed_dict)
 return h_conv1,h_pool1,h_conv2,h_pool2,h_conv3,h_pool3,h_fc1,h_fc2 
<split>
history = model.fit(ds_train,
validation_data = ds_val,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [checkpoint, get_lr_callback(CFG)],
class_weight=class_weights )
nn_name = ['tmp']
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True, random_state=123)
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
start = datetime.datetime.now() ;
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index]
nn_graph = nn_class(nn_name = nn_name[i])
 nn_graph.create_graph()
nn_graph.attach_saver()
with tf.Session() as sess:
nn_graph.attach_summary(sess)
 sess.run(tf.global_variables_initializer())
 nn_graph.train_graph(sess, x_train, y_train, x_valid, y_valid, n_epoch = 1.0)
 nn_graph.train_graph(sess, x_train, y_train, x_valid, y_valid, n_epoch = 14.0,
train_on_augmented_data = True)
 nn_graph.save_model(sess)
 if True:
break;
print('total running time for training: ', datetime.datetime.now() - start)
<set_options>
model = compile_new_model(CFG)
 model.load_weights(file_path )
if False:
!tensorboard --logdir=./logs<normalization>
CFG['batch_size'] = 128
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_testAug, verbose=1, steps=steps)
 probs = np.stack(probs)
 probs = probs2 = probs[:cnt_test * CFG['tta_steps'],0]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=0), axis=1)
 probs = np.mean(probs, axis=1 )
mn = nn_name[0]
nn_graph = nn_class()
sess = nn_graph.load_session_from_file(mn)
 W_conv1, W_conv2, W_conv3, _, _ = nn_graph.get_weights(sess)
 sess.close()
print('W_conv1: min = ' + str(np.min(W_conv1)) + ' max = ' + str(np.max(W_conv1))
+ ' mean = ' + str(np.mean(W_conv1)) + ' std = ' + str(np.std(W_conv1)))
 print('W_conv2: min = ' + str(np.min(W_conv2)) + ' max = ' + str(np.max(W_conv2))
+ ' mean = ' + str(np.mean(W_conv2)) + ' std = ' + str(np.std(W_conv2)))
 print('W_conv3: min = ' + str(np.min(W_conv3)) + ' max = ' + str(np.max(W_conv3))
+ ' mean = ' + str(np.mean(W_conv3)) + ' std = ' + str(np.std(W_conv3)))
 s_f_conv1 = nn_graph.s_f_conv1
s_f_conv2 = nn_graph.s_f_conv2
s_f_conv3 = nn_graph.s_f_conv3
W_conv1 = np.reshape(W_conv1,(s_f_conv1,s_f_conv1,1,6,6))
W_conv1 = np.transpose(W_conv1,(3,0,4,1,2))
W_conv1 = np.reshape(W_conv1,(s_f_conv1*6,s_f_conv1*6,1))
W_conv2 = np.reshape(W_conv2,(s_f_conv2,s_f_conv2,6,6,36))
W_conv2 = np.transpose(W_conv2,(2,0,3,1,4))
W_conv2 = np.reshape(W_conv2,(6*s_f_conv2,6*s_f_conv2,6,6))
W_conv2 = np.transpose(W_conv2,(2,0,3,1))
W_conv2 = np.reshape(W_conv2,(6*6*s_f_conv2,6*6*s_f_conv2))
W_conv3 = np.reshape(W_conv3,(s_f_conv3,s_f_conv3,6,6,36))
W_conv3 = np.transpose(W_conv3,(2,0,3,1,4))
W_conv3 = np.reshape(W_conv3,(6*s_f_conv3,6*s_f_conv3,6,6))
W_conv3 = np.transpose(W_conv3,(2,0,3,1))
W_conv3 = np.reshape(W_conv3,(6*6*s_f_conv3,6*6*s_f_conv3))
plt.figure(figsize=(15,5))
plt.subplot(1,3,1)
 plt.gca().set_xticks(np.arange(-0.5, s_f_conv1*6, s_f_conv1), minor = False);
plt.gca().set_yticks(np.arange(-0.5, s_f_conv1*6, s_f_conv1), minor = False);
plt.grid(which = 'minor', color='b', linestyle='-', linewidth=1)
 plt.title('W_conv1 ' + str(W_conv1.shape))
plt.colorbar(plt.imshow(W_conv1[:,:,0], cmap=cm.binary)) ;
plt.subplot(1,3,2)
 plt.gca().set_xticks(np.arange(-0.5, 6*6*s_f_conv2, 6*s_f_conv2), minor = False);
plt.gca().set_yticks(np.arange(-0.5, 6*6*s_f_conv2, 6*s_f_conv2), minor = False);
plt.grid(which = 'minor', color='b', linestyle='-', linewidth=1)
 plt.title('W_conv2 ' + str(W_conv2.shape))
plt.colorbar(plt.imshow(W_conv2[:,:], cmap=cm.binary)) ;
plt.subplot(1,3,3)
 plt.gca().set_xticks(np.arange(-0.5, 6*6*s_f_conv3, 6*s_f_conv3), minor = False);
plt.gca().set_yticks(np.arange(-0.5, 6*6*s_f_conv3, 6*s_f_conv3), minor = False);
plt.grid(which = 'minor', color='b', linestyle='-', linewidth=1)
 plt.title('W_conv3 ' + str(W_conv3.shape))
plt.colorbar(plt.imshow(W_conv3[:,:], cmap=cm.binary)) ; 
<categorify>
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )
<load_from_csv><EOS> 
submission = pd.DataFrame(dict(
 image_name = image_names,
target = probs[:]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission_model.csv', index=False )