Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<compute_test_metric>
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *
<drop_column>
N_SPLITS = 5
sample_size = 800000
Index=X_test['ID_code']
X_test=X_test.drop('ID_code', axis=1 )<create_dataframe>
epsilon = K.epsilon()
def matthews_correlation(y_true, y_pred, K=K):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + epsilon )
submissions=pd.DataFrame({'ID_code':Index, 'target': sub_preds})
<save_to_csv>
df_train = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
submissions.to_csv('submissions_lgb_samplingFEtuningbest2.csv', index=False )<drop_column>
max_num = 127
min_num = -128
df_sampled2=df_sampled1.drop('ID_code', axis=1)
 df_sampled2.head(2 )<load_pretrained>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
boosting = ["goss","dart"]
def kfold_lightgbm(train_df, test_df, num_folds, stratified = False, boosting = boosting[0]):
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=326)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=2045)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, df_sampled["target"])) :
train_x, train_y = train_df.iloc[train_idx], df_sampled["target"].iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], df_sampled["target"].iloc[valid_idx]
lgb_train = lgb.Dataset(train_x,label=train_y,free_raw_data=False)
 lgb_test = lgb.Dataset(valid_x,label=valid_y,free_raw_data=False)
 params ={
'task': 'train',
'boosting': 'goss',
'objective': 'binary',
'metric': 'auc',
'learning_rate': 0.01,
'subsample': 0.8,
'max_depth': -1,
'top_rate': 0.9064148448434349,
'num_leaves': 32,
'min_child_weight': 41.9612869171337,
'other_rate': 0.0721768246018207,
'reg_alpha': 9.677537745007898,
'colsample_bytree': 0.5665320670155495,
'min_split_gain': 9.820197773625843,
'reg_lambda': 8.2532317400459,
'min_data_in_leaf': 21,
'verbose': -1,
'seed':int(2**n_fold),
'bagging_seed':int(2**n_fold),
'drop_seed':int(2**n_fold)
 }
reg = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train, lgb_test],
valid_names=['train', 'test'],
num_boost_round=7000,early_stopping_rounds= 200,
verbose_eval=100, 
)
 oof_preds[valid_idx] = reg.predict(valid_x, num_iteration=reg.best_iteration)
 sub_preds += reg.predict(test_df, num_iteration=reg.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_x.columns
fold_importance_df["importance"] = np.log1p(reg.feature_importance(importance_type='gain', iteration=reg.best_iteration))
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d roc_auc_score : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del reg, train_x, train_y, valid_x, valid_y
display_importances(feature_importance_df)
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission['target'] = sub_preds
submission.to_csv(boosting+".csv", index=False)
 display(submission.head())
 return(submission )<compute_train_metric>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
kfold_lightgbm(df_sampled2, X_test, num_folds=5, stratified = True, boosting = boosting[0] )<load_pretrained>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
joblib.dump(lgb, 'lgb_train model' )<load_from_csv>
X = np.load(".. /input/5-fold-lstm-attention-fully-commented-0-694/X.npy")
 y = np.load(".. /input/5-fold-lstm-attention-fully-commented-0-694/y.npy" )
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<install_modules>
try:
except: pass
class LayerNormalization(Layer):
def __init__(self, eps=1e-6, **kwargs):
self.eps = eps
super(LayerNormalization, self ).__init__(**kwargs)
 def build(self, input_shape):
self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],
initializer=Ones() , trainable=True)
 self.beta = self.add_weight(name='beta', shape=input_shape[-1:],
initializer=Zeros() , trainable=True)
 super(LayerNormalization, self ).build(input_shape)
 def call(self, x):
mean = K.mean(x, axis=-1, keepdims=True)
 std = K.std(x, axis=-1, keepdims=True)
 return self.gamma *(x - mean)/(std + self.eps)+ self.beta
def compute_output_shape(self, input_shape):
return input_shape
class ScaledDotProductAttention() :
def __init__(self, d_model, attn_dropout=0.1):
self.temper = np.sqrt(d_model)
 self.dropout = Dropout(attn_dropout)
 def __call__(self, q, k, v, mask):
attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper )([q, k])
 if mask is not None:
mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)
 attn = Add()([attn, mmask])
 attn = Activation('softmax' )(attn)
 attn = self.dropout(attn)
 output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])
 return output, attn
class MultiHeadAttention() :
def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):
self.mode = mode
self.n_head = n_head
self.d_k = d_k
self.d_v = d_v
self.dropout = dropout
if mode == 0:
self.qs_layer = Dense(n_head*d_k, use_bias=False)
 self.ks_layer = Dense(n_head*d_k, use_bias=False)
 self.vs_layer = Dense(n_head*d_v, use_bias=False)
 elif mode == 1:
self.qs_layers = []
self.ks_layers = []
self.vs_layers = []
for _ in range(n_head):
self.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))
 self.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))
 self.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))
 self.attention = ScaledDotProductAttention(d_model)
 self.layer_norm = LayerNormalization() if use_norm else None
self.w_o = TimeDistributed(Dense(d_model))
def __call__(self, q, k, v, mask=None):
d_k, d_v = self.d_k, self.d_v
n_head = self.n_head
if self.mode == 0:
qs = self.qs_layer(q)
 ks = self.ks_layer(k)
 vs = self.vs_layer(v)
 def reshape1(x):
s = tf.shape(x)
 x = tf.reshape(x, [s[0], s[1], n_head, d_k])
 x = tf.transpose(x, [2, 0, 1, 3])
 x = tf.reshape(x, [-1, s[1], d_k])
 return x
qs = Lambda(reshape1 )(qs)
 ks = Lambda(reshape1 )(ks)
 vs = Lambda(reshape1 )(vs)
 if mask is not None:
mask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)
 head, attn = self.attention(qs, ks, vs, mask=mask)
 def reshape2(x):
s = tf.shape(x)
 x = tf.reshape(x, [n_head, -1, s[1], s[2]])
 x = tf.transpose(x, [1, 2, 0, 3])
 x = tf.reshape(x, [-1, s[1], n_head*d_v])
 return x
head = Lambda(reshape2 )(head)
 elif self.mode == 1:
heads = []; attns = []
for i in range(n_head):
qs = self.qs_layers[i](q)
 ks = self.ks_layers[i](k)
 vs = self.vs_layers[i](v)
 head, attn = self.attention(qs, ks, vs, mask)
 heads.append(head); attns.append(attn)
 head = Concatenate()(heads)if n_head > 1 else heads[0]
attn = Concatenate()(attns)if n_head > 1 else attns[0]
outputs = self.w_o(head)
 outputs = Dropout(self.dropout )(outputs)
 if not self.layer_norm: return outputs, attn
outputs = Add()([outputs, q])
 return self.layer_norm(outputs), attn
class PositionwiseFeedForward() :
def __init__(self, d_hid, d_inner_hid, dropout=0.1):
self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')
 self.w_2 = Conv1D(d_hid, 1)
 self.layer_norm = LayerNormalization()
self.dropout = Dropout(dropout)
 def __call__(self, x):
output = self.w_1(x)
 output = self.w_2(output)
 output = self.dropout(output)
 output = Add()([output, x])
 return self.layer_norm(output)
 class EncoderLayer() :
def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):
self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
 self.pos_ffn_layer = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)
 def __call__(self, enc_input, mask=None):
output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)
 output = self.pos_ffn_layer(output)
 return output, slf_attn
def GetPosEncodingMatrix(max_len, d_emb):
pos_enc = np.array([
[pos / np.power(10000, 2 *(j // 2)/ d_emb)for j in range(d_emb)]
if pos != 0 else np.zeros(d_emb)
 for pos in range(max_len)
 ])
 pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2])
 pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2])
 return pos_enc
def GetPadMask(q, k):
ones = K.expand_dims(K.ones_like(q, 'float32'), -1)
 mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')
 mask = K.batch_dot(ones, mask, axes=[2,1])
 return mask
def GetSubMask(s):
len_s = tf.shape(s)[1]
bs = tf.shape(s)[:1]
mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)
 return mask
class Encoder() :
def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, \
layers=6, dropout=0.1, pos_emb=None):
self.pos_layer = pos_emb
self.dropout = Dropout(dropout)
 self.layers = [EncoderLayer(d_model, d_inner_hid, n_head, d_k, d_v, dropout)for _ in range(layers)]
def __call__(self, src_seq, src_pos, return_att=False, active_layers=999):
x = src_seq
if src_pos is not None:
pos = self.pos_layer(src_pos)
 x = Add()([x, pos])
 x = self.dropout(x)
 if return_att: atts = []
mask = None
for enc_layer in self.layers[:active_layers]:
x, att = enc_layer(x, mask)
 if return_att: atts.append(att)
 return(x, atts)if return_att else x
class Transformer() :
def __init__(self, len_limit, d_model=X.shape[-1], \
d_inner_hid=512, n_head=8, d_k=64, d_v=64, layers=2, dropout=0.1, \
share_word_emb=False, **kwargs):
self.name = 'Transformer'
self.len_limit = len_limit
self.src_loc_info = True
self.d_model = d_model
d_emb = d_model
pos_emb = Embedding(len_limit, d_emb, trainable=False, \
weights=[GetPosEncodingMatrix(len_limit, d_emb)])
 self.encoder = Encoder(d_model, d_inner_hid, n_head, d_k, d_v, layers, dropout, \
pos_emb=pos_emb)
 def get_pos_seq(self, x):
pos = K.cumsum(K.ones_like(x[...,0], 'int32'), 1)
 return pos
def compile(self, active_layers=999):
src_seq = Input(shape=(X.shape[1],X.shape[2]))
src_pos = Lambda(self.get_pos_seq )(src_seq)
 if not self.src_loc_info: src_pos = None
x = self.encoder(src_seq, src_pos, active_layers=active_layers)
 x = GlobalAveragePooling1D()(x)
 outp = Dense(1, activation="sigmoid" )(x)
 self.model = Model(inputs=src_seq, outputs=outp)
 self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation] )
!pip install -q efficientnet >> /dev/null<import_modules>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = Transformer(train_X.shape[1], layers=1,d_inner_hid=256)
 model.compile()
model = model.model
ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=100, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
%matplotlib inline<define_variables>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = matthews_correlation(y_true,(y_proba > threshold ).astype(np.float64),K=np)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
print(search_result)
 return search_result
DEVICE = "TPU"
SEED = 2019
FOLDS = 2
IMG_SIZES = [128,256]
INC2019 = [0,1]
INC2018 = [1,0]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,7]
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
best_threshold = threshold_search(y_val, preds_val)['threshold']
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
%%time
meta_test = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_test.csv' )
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
X_test_input = np.load(".. /input/5-fold-lstm-attention-fully-commented-0-694/X_test.npy" )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
submission = pd.read_csv('.. /input/vsb-power-line-fault-detection/sample_submission.csv')
 print(len(submission))
submission.head()
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['target']
def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, tfrec_format)
 return example['image'], example['image_name'] if return_image_name else 0
def prepare_image(img, augment=True, dim=256):
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img,DIM=dim)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.reshape(img, [dim,dim, 3])
 return img
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<create_dataframe>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
def get_dataset(files, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True, batch_size=16, dim=256):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, imgname_or_label:(prepare_image(img, augment=augment, dim=dim),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(batch_size * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<choose_model_class>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<choose_model_class><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()